From 3c732f2834cba21d18bf9a84d61448d8127f6f2c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adis=20Halilovi=C4=87?= Date: Wed, 24 Jun 2026 19:05:58 +0200 Subject: [PATCH 01/71] feat: transformer state export, summary and vcs triggers - pull workspace state from the TFC API instead of per-workspace init - write migration-summary.{md,json}; replace workspace.tmpl with summary.tmpl - map TFC vcs settings to SG VCSTriggers (push, PR speculative, file filters) - add SGDefault*/workspaceOverrides for vcs auth, source kind and triggers --- transformer/terraform-cloud/data.tf | 4 + .../terraform-cloud/example_payload.jsonc | 21 +- transformer/terraform-cloud/locals.tf | 245 +++++++++++++----- transformer/terraform-cloud/main.tf | 4 +- transformer/terraform-cloud/resources.tf | 87 ++++--- transformer/terraform-cloud/summary.tmpl | 52 ++++ .../terraform-cloud/terraform.tfvars.example | 48 +++- transformer/terraform-cloud/variables.tf | 40 +++ transformer/terraform-cloud/workspace.tmpl | 8 - 9 files changed, 408 insertions(+), 101 deletions(-) create mode 100644 transformer/terraform-cloud/summary.tmpl delete mode 100644 transformer/terraform-cloud/workspace.tmpl diff --git a/transformer/terraform-cloud/data.tf b/transformer/terraform-cloud/data.tf index 8ea9d91..be241df 100644 --- a/transformer/terraform-cloud/data.tf +++ b/transformer/terraform-cloud/data.tf @@ -16,4 +16,8 @@ data "tfe_variables" "data" { for_each = toset(local.workflowIds) workspace_id = each.key +} + +data "tfe_projects" "data" { + organization = var.tfOrg } \ No newline at end of file diff --git a/transformer/terraform-cloud/example_payload.jsonc b/transformer/terraform-cloud/example_payload.jsonc index d7034a7..cdc11b0 100644 --- a/transformer/terraform-cloud/example_payload.jsonc +++ b/transformer/terraform-cloud/example_payload.jsonc @@ -53,5 +53,24 @@ "useMarketplaceTemplate": false } }, - "WfType": "TERRAFORM" // could be "TERRAFORM" or "CUSTOM" + "WfType": "TERRAFORM", // could be "TERRAFORM" or "CUSTOM" + "VCSTriggers": { + // Pre-configured VCS triggers, remapped from the workspace's TFC settings. + // Omit (or null) for no triggers. NOT applied by the bulk create API — the + // importer sets it in a second pass via POST .../wfs//webhooks/vcs_triggers/ + // which registers the repo webhook. Server-assigned fields (gh_webhook_url, + // *_hook_id, github_app_installation_id) are intentionally omitted. + "type": "GITHUB_COM", // GITHUB_COM, GITLAB_COM, BITBUCKET_ORG, AZURE_DEVOPS + "tracked_branch": "main", + "approval_pre_apply": true, // inverse of TFC auto_apply + "plan_only": false, + "gh_check": true, + "gl_pipeline": true, + "post_comments": true, + "push": { "createWfRun": { "enabled": true } }, // run on push to tracked branch + "pull_request_opened": { "createWfRun": { "enabled": true } }, // TFC speculative plan + "pull_request_modified": { "createWfRun": { "enabled": true } }, + "file_triggers_enabled": true, // from TFC file_triggers_enabled + "file_trigger_patterns": ["path/to/workdir/*"] // TFC trigger_patterns / trigger_prefixes / working_directory + } } diff --git a/transformer/terraform-cloud/locals.tf b/transformer/terraform-cloud/locals.tf index 356730c..48a6ebc 100644 --- a/transformer/terraform-cloud/locals.tf +++ b/transformer/terraform-cloud/locals.tf @@ -1,73 +1,200 @@ locals { - workflowIds = [for i, v in data.tfe_workspace_ids.data.ids : v] - workflowNames = [for i, v in data.tfe_workspace_ids.data.ids : i] - workflows = [for i, v in data.tfe_workspace_ids.data.ids : { - CLIConfiguration = { - "WorkflowGroup" : { - "name" : data.tfe_workspace.data[i].project_id - }, - "TfStateFilePath" : "${abspath(path.root)}/../../${var.exportPath}/states/${data.tfe_workspace.data[i].name}.tfstate" - } - ResourceName = data.tfe_workspace.data[i].name - Description = "" - Tags = data.tfe_workspace.data[i].tag_names - EnvironmentVariables = [for i, v in data.tfe_variables.data[v].variables : - { "config" : { - "textValue" : v.value, - "varName" : v.name + # data.tfe_workspace_ids.data.ids is a map of workspace name => workspace id. + workflowIds = [for name, id in data.tfe_workspace_ids.data.ids : id] + workflowNames = [for name, id in data.tfe_workspace_ids.data.ids : name] + + # project id => project name, used to name the per-project payload files and + # to set a human-readable WorkflowGroup name in the payload. + projectNames = { for p in data.tfe_projects.data.projects : p.id => p.name } + + # SG workflow-name (ResourceName) sanitization. Per the SG OpenAPI spec, + # ResourceName must be 1-100 chars; SG's name convention is ^[-a-zA-Z0-9_]+$. + # TFC workspace names already satisfy both, so for normal inputs this is a + # no-op; the steps below defensively guarantee a valid, unique name and the + # summary reports any workspace that was actually renamed. + # 1. replace any disallowed character with "-" + nameCleaned = { for name in local.workflowNames : name => replace(name, "/[^-a-zA-Z0-9_]/", "-") } + # 2. enforce the 100-char maximum + nameTruncated = { for name, cleaned in local.nameCleaned : name => length(cleaned) > 100 ? substr(cleaned, 0, 100) : cleaned } + # 3. group originals by their sanitized name to detect collisions + sanitizedGroups = { for name, san in local.nameTruncated : san => name... } + # 4. disambiguate collisions with a short deterministic suffix (<=100 chars) + resourceNames = { + for name, san in local.nameTruncated : + name => length(local.sanitizedGroups[san]) > 1 ? "${length(san) > 93 ? substr(san, 0, 93) : san}-${substr(md5(name), 0, 6)}" : san + } + + # TFC never returns values for sensitive variables, so they cannot be + # migrated. Record them per workspace so the summary can flag them. + sensitiveVars = { + for name, id in data.tfe_workspace_ids.data.ids : + name => [for v in data.tfe_variables.data[id].variables : "${v.category}:${v.name}" if v.sensitive] + } + + # Workspaces whose terraform_version is not a pinned semver (e.g. "latest" or + # a constraint) and have no per-workspace override fall back to the default. + versionFallbacks = { + for name in local.workflowNames : + name => data.tfe_workspace.data[name].terraform_version + if !can(regex("^[0-9]+\\.[0-9]+\\.[0-9]+$", data.tfe_workspace.data[name].terraform_version)) && try(var.workspaceOverrides[name].terraformVersion, null) == null + } + + # Workspaces not using "remote" execution may not store their state in TFC, so + # the API state export can come back empty; flag them in the summary. + nonRemoteModes = { + for name in local.workflowNames : + name => data.tfe_workspace.data[name].execution_mode + if data.tfe_workspace.data[name].execution_mode != "remote" + } + + # Resolved SG VCS provider per workspace (override wins over the default). + # Drives both the source config kind and which workspaces get VCS triggers. + sourceKind = { + for name in local.workflowNames : + name => try(var.workspaceOverrides[name].sourceConfigDestKind, null) != null ? var.workspaceOverrides[name].sourceConfigDestKind : var.SGDefaultSourceConfigDestKind + } + + # One SG workflow payload per workspace. Per-workspace overrides win over the + # SGDefault* values; everything else is derived from the TFC workspace. + workflowPayload = { + for wsName, wsId in data.tfe_workspace_ids.data.ids : wsName => { + CLIConfiguration = { + "WorkflowGroup" : { + # SG workflow group per TFC project: tfc- (matches the group + # the importer creates/targets and the per-project payload filename). + "name" : "tfc-${local.projectFileSegment[data.tfe_workspace.data[wsName].project_id]}" }, - "kind" : "PLAIN_TEXT" } if v.category == "env" && v.sensitive == false] - - DeploymentPlatformConfig = var.SGDefaultDeploymentPlatformConfig - RunnerConstraints = { "type" : "shared" } - VCSConfig = { - "iacVCSConfig" : { - "useMarketplaceTemplate" : false, - "customSource" : { - "sourceConfigDestKind" : var.SGDefaultSourceConfigDestKind - "config" : { - "includeSubModule" : false, - "ref" : length(data.tfe_workspace.data[i].vcs_repo) > 0 ? data.tfe_workspace.data[i].vcs_repo[0].branch != "" ? data.tfe_workspace.data[i].vcs_repo[0].branch : "" : "", - "isPrivate" : length(data.tfe_workspace.data[i].vcs_repo) > 0 ? length(data.tfe_workspace.data[i].vcs_repo[0].oauth_token_id) > 0 || length(data.tfe_workspace.data[i].vcs_repo[0].github_app_installation_id) > 0 ? true : false : false, - "auth" : length(data.tfe_workspace.data[i].vcs_repo) > 0 ? length(data.tfe_workspace.data[i].vcs_repo[0].oauth_token_id) > 0 || length(data.tfe_workspace.data[i].vcs_repo[0].github_app_installation_id) > 0 ? var.SGDefaultVCSAuthIntegrationID : "" : "", - "workingDir" : data.tfe_workspace.data[i].working_directory, - "repo" : length(data.tfe_workspace.data[i].vcs_repo) > 0 ? format("%s/%s", var.SGDefaultIACVCSRepoPrefix, data.tfe_workspace.data[i].vcs_repo[0].identifier) : "" + "TfStateFilePath" : "${abspath(path.root)}/../../${var.exportPath}/states/${data.tfe_workspace.data[wsName].name}.tfstate" + } + ResourceName = local.resourceNames[wsName] + Description = "" + Tags = data.tfe_workspace.data[wsName].tag_names + EnvironmentVariables = concat( + [for v in data.tfe_variables.data[wsId].variables : + { "config" : { "textValue" : v.value, "varName" : v.name }, "kind" : "PLAIN_TEXT" } + if v.category == "env" && v.sensitive == false], + try(var.workspaceOverrides[wsName].extraEnvironmentVariables, []) + ) + + DeploymentPlatformConfig = try(var.workspaceOverrides[wsName].DeploymentPlatformConfig, null) != null ? var.workspaceOverrides[wsName].DeploymentPlatformConfig : var.SGDefaultDeploymentPlatformConfig + RunnerConstraints = try(var.workspaceOverrides[wsName].RunnerConstraints, null) != null ? var.workspaceOverrides[wsName].RunnerConstraints : { "type" : "shared" } + + VCSConfig = { + "iacVCSConfig" : { + "useMarketplaceTemplate" : false, + "customSource" : { + "sourceConfigDestKind" : local.sourceKind[wsName] + "config" : { + "includeSubModule" : false, + "ref" : length(data.tfe_workspace.data[wsName].vcs_repo) > 0 ? data.tfe_workspace.data[wsName].vcs_repo[0].branch : "", + "isPrivate" : length(data.tfe_workspace.data[wsName].vcs_repo) > 0 ? length(data.tfe_workspace.data[wsName].vcs_repo[0].oauth_token_id) > 0 || length(data.tfe_workspace.data[wsName].vcs_repo[0].github_app_installation_id) > 0 : false, + "auth" : length(data.tfe_workspace.data[wsName].vcs_repo) > 0 ? (length(data.tfe_workspace.data[wsName].vcs_repo[0].oauth_token_id) > 0 || length(data.tfe_workspace.data[wsName].vcs_repo[0].github_app_installation_id) > 0 ? (try(var.workspaceOverrides[wsName].vcsAuthIntegrationID, null) != null ? var.workspaceOverrides[wsName].vcsAuthIntegrationID : var.SGDefaultVCSAuthIntegrationID) : "") : "", + "workingDir" : data.tfe_workspace.data[wsName].working_directory, + "repo" : length(data.tfe_workspace.data[wsName].vcs_repo) > 0 ? format("%s/%s", try(var.workspaceOverrides[wsName].vcsRepoPrefix, null) != null ? var.workspaceOverrides[wsName].vcsRepoPrefix : var.SGDefaultIACVCSRepoPrefix, data.tfe_workspace.data[wsName].vcs_repo[0].identifier) : "" + } } + }, + "iacInputData" : { + "schemaType" : "RAW_JSON", + "data" : { for v in data.tfe_variables.data[wsId].variables : v.name => try(jsondecode(v.value), v.value) if v.category == "terraform" && v.sensitive == false } } - }, - "iacInputData" : { - "schemaType" : "RAW_JSON", - "data" : { for i, v in data.tfe_variables.data[v].variables : v.name => try(jsondecode(v.value), v.value) if v.category == "terraform" } } - } - MiniSteps = { - "wfChaining" : { - "ERRORED" : [], - "COMPLETED" : [] - }, - "notifications" : { - "email" : { + # VCS triggers, remapped 1:1 from the workspace's own TFC VCS settings. + # Only emitted for VCS-backed workspaces on a provider SG can webhook + # (GIT_OTHER has no webhook support); null otherwise. This block is NOT + # accepted by the bulk workflow-create API — the importer applies it in a + # second pass via POST .../wfs//webhooks/vcs_triggers/ (see migrate.sh + # cmd_triggers), which is what actually registers the repo webhook. Field + # shape matches a real SG workflow + the landfast set_vcs_triggers call; + # only the truly server-assigned fields (gh_webhook_url, *_hook_id, + # github_app_installation_id) are omitted. + VCSTriggers = try(var.workspaceOverrides[wsName].VCSTriggers, null) != null ? var.workspaceOverrides[wsName].VCSTriggers : ( + var.SGDefaultEnableVCSTriggers && + length(data.tfe_workspace.data[wsName].vcs_repo) > 0 && + contains(["GITHUB_COM", "GITLAB_COM", "BITBUCKET_ORG", "AZURE_DEVOPS"], local.sourceKind[wsName]) + ? { + "type" : local.sourceKind[wsName], + "tracked_branch" : data.tfe_workspace.data[wsName].vcs_repo[0].branch, + "approval_pre_apply" : !data.tfe_workspace.data[wsName].auto_apply, + "plan_only" : false, + "gh_check" : true, + "gl_pipeline" : true, + "post_comments" : true, + # TFC runs on every push to the tracked branch when VCS-connected. + "push" : { "createWfRun" : { "enabled" : true } }, + # TFC speculative plans on PRs map to the PR-open/update triggers. + "pull_request_opened" : { "createWfRun" : { "enabled" : data.tfe_workspace.data[wsName].speculative_enabled } }, + "pull_request_modified" : { "createWfRun" : { "enabled" : data.tfe_workspace.data[wsName].speculative_enabled } }, + "file_triggers_enabled" : data.tfe_workspace.data[wsName].file_triggers_enabled, + # Prefer TFC's glob trigger_patterns; fall back to legacy + # trigger_prefixes (prefix -> "/*"); else, when file triggers + # are on with a working dir, scope to that dir like TFC does. + "file_trigger_patterns" : ( + try(length(data.tfe_workspace.data[wsName].trigger_patterns), 0) > 0 ? data.tfe_workspace.data[wsName].trigger_patterns : + try(length(data.tfe_workspace.data[wsName].trigger_prefixes), 0) > 0 ? [for p in data.tfe_workspace.data[wsName].trigger_prefixes : "${p}/*"] : + data.tfe_workspace.data[wsName].file_triggers_enabled && data.tfe_workspace.data[wsName].working_directory != "" ? ["${data.tfe_workspace.data[wsName].working_directory}/*"] : [] + ) + } + : null + ) + + MiniSteps = { + "wfChaining" : { "ERRORED" : [], - "COMPLETED" : [], - "APPROVAL_REQUIRED" : [], - "CANCELLED" : [] + "COMPLETED" : [] + }, + "notifications" : { + "email" : { + "ERRORED" : [], + "COMPLETED" : [], + "APPROVAL_REQUIRED" : [], + "CANCELLED" : [] + } } } - } - Approvers = data.tfe_workspace.data[i].auto_apply == true ? [] : var.SGDefaultWfApprovers + Approvers = try(var.workspaceOverrides[wsName].Approvers, null) != null ? var.workspaceOverrides[wsName].Approvers : (data.tfe_workspace.data[wsName].auto_apply ? [] : var.SGDefaultWfApprovers) - TerraformConfig = { - "managedTerraformState" : true, - "terraformVersion" : "TERRAFORM-${data.tfe_workspace.data[i].terraform_version}", - "approvalPreApply" : !data.tfe_workspace.data[i].auto_apply + TerraformConfig = { + "managedTerraformState" : true, + "terraformVersion" : ( + try(var.workspaceOverrides[wsName].terraformVersion, null) != null ? var.workspaceOverrides[wsName].terraformVersion : + can(regex("^[0-9]+\\.[0-9]+\\.[0-9]+$", data.tfe_workspace.data[wsName].terraform_version)) ? "TERRAFORM-${data.tfe_workspace.data[wsName].terraform_version}" : var.SGDefaultTerraformVersion + ), + "approvalPreApply" : !data.tfe_workspace.data[wsName].auto_apply + } + + WfType = "TERRAFORM" + UserSchedules = [] } + } + + # Group payloads by TFC project so each project imports into its own SG + # workflow group (the bulk import takes a single --workflow-group per file). + workflowProject = { for wsName, wsId in data.tfe_workspace_ids.data.ids : wsName => data.tfe_workspace.data[wsName].project_id } + projectsUsed = toset(values(local.workflowProject)) + + payloadByProject = { + for pid in local.projectsUsed : + pid => [for name, payload in local.workflowPayload : payload if local.workflowProject[name] == pid] + } + + # project id => filesystem-safe segment for the per-project payload filename. + projectFileSegment = { + for pid in local.projectsUsed : + pid => replace(lower(try(local.projectNames[pid], pid)), "/[^a-z0-9-]+/", "-") + } - WfType = "TERRAFORM" - UserSchedules = [] - }] - data = jsonencode( - local.workflows - ) + # Machine-readable migration summary (also rendered to markdown). + summary = { + organization = var.tfOrg + workspaceCount = length(local.workflowNames) + projectWorkspaceCounts = { for pid in local.projectsUsed : try(local.projectNames[pid], pid) => length(local.payloadByProject[pid]) } + skippedSensitiveVars = { for name, vars in local.sensitiveVars : name => vars if length(vars) > 0 } + terraformVersionFallbacks = local.versionFallbacks + nonRemoteExecutionModes = local.nonRemoteModes + renamedWorkspaces = { for name in local.workflowNames : name => local.resourceNames[name] if local.resourceNames[name] != name } + variableSetsReminder = "TFC Variable Set variables are merged by the 'enrich' step (non-sensitive only). Sensitive set vars can't be read from the API — recreate them as SG secrets; see the enrich step output." + } } diff --git a/transformer/terraform-cloud/main.tf b/transformer/terraform-cloud/main.tf index 5ae44be..9a19179 100644 --- a/transformer/terraform-cloud/main.tf +++ b/transformer/terraform-cloud/main.tf @@ -1,5 +1,5 @@ terraform { - required_version = "~> 1.2" + required_version = ">= 1.3" required_providers { local = { @@ -8,7 +8,7 @@ terraform { } tfe = { source = "hashicorp/tfe" - version = "~> 0.48.0" + version = "~> 0.78" } null = { source = "hashicorp/null" diff --git a/transformer/terraform-cloud/resources.tf b/transformer/terraform-cloud/resources.tf index d53c3d5..fd58be0 100644 --- a/transformer/terraform-cloud/resources.tf +++ b/transformer/terraform-cloud/resources.tf @@ -1,40 +1,67 @@ +# One payload file per TFC project. Written directly (no -generated + mv dance) +# so re-applies always refresh the output. resource "local_file" "data" { - content = local.data - filename = "${path.module}/../../${var.exportPath}/sg-payload-generated.json" - provisioner "local-exec" { - command = "mv ${path.module}/../../${var.exportPath}/sg-payload-generated.json ${path.module}/../../${var.exportPath}/sg-payload.json" - } -} + for_each = local.payloadByProject -resource "local_file" "generateTempTfFiles" { - for_each = var.exportStateFiles ? toset(local.workflowNames) : [] - - content = templatefile("${path.module}/workspace.tmpl", { tfOrg = var.tfOrg, workspace = each.key }) - filename = "${path.module}/../../${var.exportPath}/tfDir/${each.key}/main.tf" + content = jsonencode(each.value) + filename = "${path.module}/../../${var.exportPath}/sg-payload.${local.projectFileSegment[each.key]}.json" } -resource "null_resource" "exportStateFiles" { - depends_on = [local_file.generateTempTfFiles] - triggers = { - always-update = timestamp() - } - for_each = var.exportStateFiles ? toset(local.workflowNames) : [] +resource "local_file" "summary" { + content = jsonencode(local.summary) + filename = "${path.module}/../../${var.exportPath}/migration-summary.json" +} - provisioner "local-exec" { - command = "mkdir -p ../../states && rm -rf .terraform .terraform.lock.hcl terraform.tfstate terraform.tfstate.backup && terraform init -input=false && terraform state pull > ../../states/'${each.key}.tfstate'" - working_dir = "${path.module}/../../${var.exportPath}/tfDir/${each.key}" - } +resource "local_file" "summaryMd" { + content = templatefile("${path.module}/summary.tmpl", { summary = local.summary }) + filename = "${path.module}/../../${var.exportPath}/migration-summary.md" } -resource "null_resource" "deleteTempTfFiles" { - count = var.exportStateFiles ? 1 : 0 - triggers = { - always-update = timestamp() - } - depends_on = [null_resource.exportStateFiles] +# Pull each workspace's current state directly from the TFC/TFE API — no +# `terraform init` or providers (avoids the plugin-cache concurrency issue and +# per-workspace provider downloads). The token is read at runtime from the +# `terraform login` credentials file or TFE_TOKEN, so it never enters TF state. +# Requires `curl` and `jq` on PATH (provided by the Docker image / orchestrator). +resource "null_resource" "exportState" { + for_each = var.exportStateFiles ? data.tfe_workspace_ids.data.ids : {} + + # Idempotent by default (keyed by stable workspace name/id); forceStateRefresh + # re-pulls every workspace. + triggers = merge( + { workspace = each.key, id = each.value }, + var.forceStateRefresh ? { refresh = timestamp() } : {} + ) provisioner "local-exec" { - command = "rm -rf tfDir" - working_dir = "${path.module}/../../${var.exportPath}/" + interpreter = ["/bin/bash", "-c"] + # SG_EXPORT is absolute and created in-command, so it works regardless of the + # provisioner's working directory or resource ordering. + environment = { + SG_TFC_HOST = var.tfHostname + SG_WS_ID = each.value + SG_WS_NAME = each.key + SG_EXPORT = "${abspath(path.module)}/../../${var.exportPath}" + } + # Failure isolation: missing token / no-state / download error is recorded + # in state-export-failures.log instead of aborting the whole apply. + command = <<-EOT + set -uo pipefail + mkdir -p "$SG_EXPORT/states" + creds="$HOME/.terraform.d/credentials.tfrc.json" + token="" + if [ -f "$creds" ] && command -v jq >/dev/null 2>&1; then + token="$(jq -r --arg h "$SG_TFC_HOST" '.credentials[$h].token // empty' "$creds" 2>/dev/null || true)" + fi + [ -z "$token" ] && token="$${TFE_TOKEN:-}" + if [ -z "$token" ]; then + echo "$SG_WS_NAME: no TFC token (set TFE_TOKEN or run 'terraform login')" >> "$SG_EXPORT/state-export-failures.log"; exit 0 + fi + url="$(curl -fsS -H "Authorization: Bearer $token" "https://$SG_TFC_HOST/api/v2/workspaces/$SG_WS_ID/current-state-version" | jq -r '.data.attributes."hosted-state-download-url" // empty' 2>/dev/null || true)" + if [ -z "$url" ]; then + echo "$SG_WS_NAME: no current state version" >> "$SG_EXPORT/state-export-failures.log"; exit 0 + fi + curl -fsSL -H "Authorization: Bearer $token" "$url" -o "$SG_EXPORT/states/$SG_WS_NAME.tfstate" \ + || echo "$SG_WS_NAME: state download failed" >> "$SG_EXPORT/state-export-failures.log" + EOT } -} \ No newline at end of file +} diff --git a/transformer/terraform-cloud/summary.tmpl b/transformer/terraform-cloud/summary.tmpl new file mode 100644 index 0000000..8dc1ba4 --- /dev/null +++ b/transformer/terraform-cloud/summary.tmpl @@ -0,0 +1,52 @@ +# StackGuardian migration summary + +- Organization: ${summary.organization} +- Workspaces processed: ${summary.workspaceCount} + +## Workflows per project +%{ for project, count in summary.projectWorkspaceCounts ~} +- ${project}: ${count} +%{ endfor ~} + +## Skipped sensitive variables +These are not migrated (TFC never returns sensitive values). Recreate them as SG secrets after import. +%{ if length(summary.skippedSensitiveVars) == 0 ~} +- None. +%{ else ~} +%{ for ws, vars in summary.skippedSensitiveVars ~} +- ${ws}: ${join(", ", vars)} +%{ endfor ~} +%{ endif ~} + +## Terraform version fallbacks +Workspaces whose version was not a pinned semver; the configured SGDefaultTerraformVersion was used. +%{ if length(summary.terraformVersionFallbacks) == 0 ~} +- None. +%{ else ~} +%{ for ws, ver in summary.terraformVersionFallbacks ~} +- ${ws}: reported "${ver}" +%{ endfor ~} +%{ endif ~} + +## Non-remote execution modes +TFC may not hold state for these (local/agent execution); state export can be empty. +%{ if length(summary.nonRemoteExecutionModes) == 0 ~} +- None. +%{ else ~} +%{ for ws, mode in summary.nonRemoteExecutionModes ~} +- ${ws}: ${mode} +%{ endfor ~} +%{ endif ~} + +## Renamed workspaces +%{ if length(summary.renamedWorkspaces) == 0 ~} +- None. +%{ else ~} +%{ for orig, new in summary.renamedWorkspaces ~} +- ${orig} -> ${new} +%{ endfor ~} +%{ endif ~} + +## Reminders +- ${summary.variableSetsReminder} +- If state export ran, check state-export-failures.log in the export directory for workspaces whose state could not be pulled. diff --git a/transformer/terraform-cloud/terraform.tfvars.example b/transformer/terraform-cloud/terraform.tfvars.example index 66bf280..9454029 100644 --- a/transformer/terraform-cloud/terraform.tfvars.example +++ b/transformer/terraform-cloud/terraform.tfvars.example @@ -20,7 +20,7 @@ exportPath = "export" SGDefaultWfApprovers = [] # Prefix for your repo URL -SGDefaultIACVCSRepoPrefix = "https://www.github.com" +SGDefaultIACVCSRepoPrefix = "https://github.com" # Provide an integration id like /integrations/aws-dev-account or /secrets/my-git-token SGDefaultVCSAuthIntegrationID = "/integrations/github_com" @@ -38,3 +38,49 @@ SGDefaultDeploymentPlatformConfig = [ # Choose from: GITHUB_COM, BITBUCKET_ORG, GITLAB_COM, AZURE_DEVOPS, GIT_OTHER SGDefaultSourceConfigDestKind = "GITHUB_COM" + +# SG Terraform version used when a workspace's terraform_version is not a pinned +# semver (e.g. "latest" or a constraint), or runs an engine SG cannot map. +SGDefaultTerraformVersion = "TERRAFORM-1.5.7" + +# Pre-configure VCS triggers on each VCS-backed workflow, remapped from the +# workspace's own TFC settings (tracked branch, push, PR speculative plans, file +# triggers). Set false to import workflows without any triggers. +SGDefaultEnableVCSTriggers = true + +# Re-pull state for every workspace on each apply. Leave false for idempotent runs. +forceStateRefresh = false + +# Per-workspace overrides, keyed by workspace name. Any field set here wins over +# the SGDefault* value above, for that workspace only. All fields are optional. +# workspaceOverrides = { +# "prod-networking" = { +# DeploymentPlatformConfig = [ +# { +# "kind" : "AWS_RBAC", +# "config" : { "integrationId" : "/integrations/aws-prod", "profileName" : "default" } +# } +# ] +# RunnerConstraints = { "type" : "private", "names" : ["sg-runner"] } +# Approvers = ["lead@example.com"] +# vcsAuthIntegrationID = "/integrations/github_prod" +# vcsRepoPrefix = "https://www.github.com" +# sourceConfigDestKind = "GITHUB_COM" +# terraformVersion = "TERRAFORM-1.7.5" +# extraEnvironmentVariables = [ +# { "config" : { "textValue" : "us-east-1", "varName" : "AWS_REGION" }, "kind" : "PLAIN_TEXT" } +# ] +# # Full override of the derived VCS triggers (replaces the remap entirely). +# VCSTriggers = { +# type = "GITHUB_COM" +# tracked_branch = "main" +# approval_pre_apply = true +# plan_only = false +# push = { createWfRun = { enabled = true } } +# pull_request_opened = { createWfRun = { enabled = true } } +# pull_request_modified = { createWfRun = { enabled = true } } +# file_triggers_enabled = true +# file_trigger_patterns = ["modules/networking/*"] +# } +# } +# } diff --git a/transformer/terraform-cloud/variables.tf b/transformer/terraform-cloud/variables.tf index 3e15899..52c6df6 100644 --- a/transformer/terraform-cloud/variables.tf +++ b/transformer/terraform-cloud/variables.tf @@ -15,6 +15,12 @@ variable "exportStateFiles" { type = bool } +variable "tfHostname" { + default = "app.terraform.io" + description = "TFC/TFE hostname used for the state-export API and the `terraform login` credential lookup." + type = string +} + variable "tfWorkspaceTags" { default = null description = "List of TFC/TFE workspace tags to include when exporting. Excluded tags take precedence over included ones. Wildcards are not supported." @@ -69,4 +75,38 @@ variable "SGDefaultSourceConfigDestKind" { default = "GIT_OTHER" description = "Choose from: GITHUB_COM, BITBUCKET_ORG, GITLAB_COM, AZURE_DEVOPS, GIT_OTHER" type = string +} + +variable "SGDefaultTerraformVersion" { + default = "TERRAFORM-1.5.7" + description = "SG Terraform version used when a workspace's terraform_version is not a pinned semver (e.g. 'latest' or a version constraint), or the workspace runs an engine SG cannot map. Use the SG-formatted value, e.g. TERRAFORM-1.5.7." + type = string +} + +variable "SGDefaultEnableVCSTriggers" { + default = true + description = "Pre-configure VCS triggers on each VCS-backed workflow, remapped from the workspace's own TFC settings (tracked branch, push, PR speculative plans, file triggers). Set false to import workflows without any triggers. Per-workspace overrides via workspaceOverrides[name].VCSTriggers." + type = bool +} + +variable "forceStateRefresh" { + default = false + description = "Re-pull Terraform state for every workspace on each apply. When false (default), state export is idempotent and only runs for workspaces it has not exported before." + type = bool +} + +variable "workspaceOverrides" { + default = {} + description = "Per-workspace overrides keyed by TFC/TFE workspace name. Any field set here takes precedence over the matching SGDefault* value for that workspace only." + type = map(object({ + DeploymentPlatformConfig = optional(list(any)) + RunnerConstraints = optional(any) + Approvers = optional(list(string)) + vcsAuthIntegrationID = optional(string) + vcsRepoPrefix = optional(string) + sourceConfigDestKind = optional(string) + terraformVersion = optional(string) + extraEnvironmentVariables = optional(list(any), []) + VCSTriggers = optional(any) + })) } \ No newline at end of file diff --git a/transformer/terraform-cloud/workspace.tmpl b/transformer/terraform-cloud/workspace.tmpl deleted file mode 100644 index 9aa95f4..0000000 --- a/transformer/terraform-cloud/workspace.tmpl +++ /dev/null @@ -1,8 +0,0 @@ -terraform { - cloud { - organization = "${tfOrg}" - workspaces { - name = "${workspace}" - } - } -} \ No newline at end of file From a0c0b177ec685945f43f88ee7a357e5f858fb9db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adis=20Halilovi=C4=87?= Date: Wed, 24 Jun 2026 19:06:05 +0200 Subject: [PATCH 02/71] feat: payload json schema and validator Validate generated payloads against schema/sg-payload.schema.json via yajsv. --- schema/sg-payload.schema.json | 118 ++++++++++++++++++++++++++++++++++ scripts/validate_payload.sh | 29 +++++++++ 2 files changed, 147 insertions(+) create mode 100644 schema/sg-payload.schema.json create mode 100755 scripts/validate_payload.sh diff --git a/schema/sg-payload.schema.json b/schema/sg-payload.schema.json new file mode 100644 index 0000000..c507be2 --- /dev/null +++ b/schema/sg-payload.schema.json @@ -0,0 +1,118 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "StackGuardian bulk workflow payload", + "description": "Validates the array of workflow objects produced by the transformer (sg-payload..json). Constraints (ResourceName length, kind/sourceConfigDestKind enums) are derived from schema/sg-openapi.json (#/components/schemas/Workflow and related). Lenient by design: only the fields the transformer emits are checked; unknown fields are allowed so the schema does not need to track every optional API field.", + "type": "array", + "items": { "$ref": "#/definitions/workflow" }, + "definitions": { + "workflow": { + "type": "object", + "required": ["ResourceName", "WfType", "VCSConfig"], + "properties": { + "ResourceName": { "type": "string", "minLength": 1, "maxLength": 100 }, + "WfType": { "type": "string", "minLength": 1 }, + "Description": { "type": "string" }, + "Tags": { "type": "array", "items": { "type": "string" } }, + "Approvers": { "type": "array", "items": { "type": "string" } }, + "UserSchedules": { "type": "array" }, + "EnvironmentVariables": { + "type": "array", + "items": { + "type": "object", + "required": ["kind", "config"], + "properties": { + "kind": { "type": "string" }, + "config": { + "type": "object", + "required": ["varName"], + "properties": { + "varName": { "type": "string", "minLength": 1 }, + "textValue": { "type": "string" } + } + } + } + } + }, + "DeploymentPlatformConfig": { + "type": "array", + "items": { + "type": "object", + "required": ["kind", "config"], + "properties": { + "kind": { + "enum": ["AWS_STATIC", "AWS_RBAC", "AWS_OIDC", "AZURE_STATIC", "AZURE_OIDC", "AZURE_MANAGED_ID_OIDC", "GCP_STATIC", "GCP_OIDC"] + }, + "config": { "type": "object" } + } + } + }, + "RunnerConstraints": { + "type": "object", + "required": ["type"], + "properties": { + "type": { "type": "string" }, + "names": { "type": "array", "items": { "type": "string" } } + } + }, + "VCSConfig": { + "type": "object", + "required": ["iacVCSConfig", "iacInputData"], + "properties": { + "iacVCSConfig": { + "type": "object", + "properties": { + "customSource": { + "type": "object", + "properties": { + "sourceConfigDestKind": { + "enum": ["GITHUB_COM", "GITHUB_APP_CUSTOM", "GIT_OTHER", "INLINE", "BITBUCKET_ORG", "GITLAB_COM", "AZURE_DEVOPS"] + } + } + } + } + }, + "iacInputData": { + "type": "object", + "required": ["schemaType", "data"], + "properties": { + "schemaType": { "type": "string" }, + "data": { "type": "object" } + } + } + } + }, + "TerraformConfig": { + "type": "object", + "properties": { + "terraformVersion": { "type": "string", "minLength": 1 }, + "managedTerraformState": { "type": "boolean" }, + "approvalPreApply": { "type": "boolean" } + } + }, + "VCSTriggers": { + "type": ["object", "null"], + "required": ["type"], + "properties": { + "type": { + "enum": ["GITHUB_COM", "GITHUB_APP_CUSTOM", "GITLAB_OAUTH_SSH", "BITBUCKET_ORG", "GITLAB_COM", "AZURE_DEVOPS", "AZURE_DEVOPS_SP"] + }, + "tracked_branch": { "type": ["string", "null"] }, + "approval_pre_apply": { "type": "boolean" }, + "plan_only": { "type": "boolean" }, + "gh_check": { "type": "boolean" }, + "gl_pipeline": { "type": "boolean" }, + "post_comments": { "type": "boolean" }, + "file_triggers_enabled": { "type": "boolean" }, + "file_trigger_patterns": { "type": "array", "items": { "type": "string" } }, + "tags_regex": { "type": ["string", "null"] }, + "push": { "type": "object" }, + "pull_request_opened": { "type": "object" }, + "pull_request_modified": { "type": "object" }, + "all_pull_requests": { "type": "object" }, + "create_tag": { "type": "object" } + } + } + } + } + } +} diff --git a/scripts/validate_payload.sh b/scripts/validate_payload.sh new file mode 100755 index 0000000..234638e --- /dev/null +++ b/scripts/validate_payload.sh @@ -0,0 +1,29 @@ +#!/bin/bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=tools.sh +source "$SCRIPT_DIR/tools.sh" + +SCHEMA="$SG_REPO_ROOT/schema/sg-payload.schema.json" + +if [ "$#" -eq 0 ]; then + echo "Usage: $0 [more.json ...]" >&2 + echo " e.g. $0 export/sg-payload.*.json" >&2 + exit 1 +fi +if [ ! -f "$SCHEMA" ]; then + echo "Schema not found: $SCHEMA" >&2 + exit 1 +fi + +# Resolve yajsv from PATH (Docker image) or download+cache (native). +YAJSV_BIN=$(sg_resolve yajsv sg_ensure_yajsv) + +sg_log "validating $# file(s) against schema/sg-payload.schema.json" +# yajsv prints ": valid" per file and exits non-zero if any file fails. +# Strip the repo-root prefix from its output for readable, relative paths +# (pipefail off so the pipeline's status is sed's; yajsv's status via PIPESTATUS). +set +o pipefail +"$YAJSV_BIN" -s "$SCHEMA" "$@" 2>&1 | sed "s#${SG_REPO_ROOT}/##g" +exit "${PIPESTATUS[0]}" From e2ec6840e384a8704f382ecac6a76cf617f1a0c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adis=20Halilovi=C4=87?= Date: Wed, 24 Jun 2026 19:06:13 +0200 Subject: [PATCH 03/71] feat: dockerized migration orchestrator - sg-migrate.sh + Dockerfile run the full pipeline inside a container - scripts/migrate.sh: init/apply/enrich/convert/validate/import/triggers/all - merge TFC variable sets, parallel convert/import, register VCS triggers - move convert_hcl_to_json.sh into scripts/ --- .dockerignore | 9 + .gitignore | 3 + Dockerfile | 53 +++ .../convert_hcl_to_json.sh | 64 +-- scripts/enrich_variable_sets.sh | 166 +++++++ scripts/migrate.sh | 443 ++++++++++++++++++ scripts/tools.sh | 162 +++++++ sg-migrate.sh | 69 +++ 8 files changed, 920 insertions(+), 49 deletions(-) create mode 100644 .dockerignore create mode 100644 Dockerfile rename convert_hcl_to_json.sh => scripts/convert_hcl_to_json.sh (66%) create mode 100755 scripts/enrich_variable_sets.sh create mode 100755 scripts/migrate.sh create mode 100755 scripts/tools.sh create mode 100755 sg-migrate.sh diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..51be1ac --- /dev/null +++ b/.dockerignore @@ -0,0 +1,9 @@ +.git +.sg +export +out +**/.terraform +**/.terraform.lock.hcl +*.tfstate +*.tfstate.* +*.tfvars diff --git a/.gitignore b/.gitignore index 1d638cc..953654c 100644 --- a/.gitignore +++ b/.gitignore @@ -163,4 +163,7 @@ out/* zip zip/* +# Migrator local cache + config (downloaded tool binaries, workflow-group map) +.sg/ + .DS_Store \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..a8668cf --- /dev/null +++ b/Dockerfile @@ -0,0 +1,53 @@ +# Migrator runtime: bundles all pinned tooling so the flow behaves identically +# on Linux/macOS/Windows hosts (anywhere Docker runs). The repo is bind-mounted +# at /app at runtime; this image only provides the tools on PATH. + +# yajsv has no linux/arm64 release asset, so build it from source for the +# image's target architecture. +FROM golang:1.22-bookworm AS yajsv +ARG YAJSV_VERSION=v1.4.1 +RUN go install "github.com/neilpa/yajsv@${YAJSV_VERSION}" + +FROM debian:bookworm-slim + +ARG TERRAFORM_VERSION=1.9.8 +ARG JQ_VERSION=1.8.1 +ARG HCL2JSON_VERSION=0.6.7 + +RUN apt-get update && apt-get install -y --no-install-recommends \ + bash curl ca-certificates git unzip tar coreutils \ + && rm -rf /var/lib/apt/lists/* + +# terraform (arch from dpkg: amd64/arm64 — works with or without buildx) +RUN arch="$(dpkg --print-architecture)" \ + && curl -fsSL "https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/terraform_${TERRAFORM_VERSION}_linux_${arch}.zip" -o /tmp/tf.zip \ + && unzip /tmp/tf.zip -d /usr/local/bin \ + && rm /tmp/tf.zip + +# jq +RUN arch="$(dpkg --print-architecture)" \ + && curl -fsSL "https://github.com/jqlang/jq/releases/download/jq-${JQ_VERSION}/jq-linux-${arch}" -o /usr/local/bin/jq \ + && chmod +x /usr/local/bin/jq + +# hcl2json +RUN arch="$(dpkg --print-architecture)" \ + && curl -fsSL "https://github.com/tmccombs/hcl2json/releases/download/v${HCL2JSON_VERSION}/hcl2json_linux_${arch}" -o /usr/local/bin/hcl2json \ + && chmod +x /usr/local/bin/hcl2json + +# yajsv (from the build stage above) +COPY --from=yajsv /go/bin/yajsv /usr/local/bin/yajsv + +# sg-cli (latest release, Go binary). Assets: sg-cli__.tar.gz. +RUN set -eu; \ + arch="$(dpkg --print-architecture)"; \ + case "$arch" in amd64) arch=x86_64 ;; arm64) arch=arm64 ;; esac; \ + curl -fsSL "https://github.com/StackGuardian/sg-cli/releases/latest/download/sg-cli_Linux_${arch}.tar.gz" -o /tmp/sg-cli.tar.gz; \ + mkdir -p /tmp/sgcli; \ + tar -xzf /tmp/sg-cli.tar.gz -C /tmp/sgcli; \ + realcli="$(find /tmp/sgcli -maxdepth 2 -type f -name sg-cli | head -1)"; \ + test -n "$realcli"; \ + install -m 0755 "$realcli" /usr/local/bin/sg-cli; \ + rm -rf /tmp/sg-cli.tar.gz /tmp/sgcli + +WORKDIR /app +ENTRYPOINT ["/bin/bash"] diff --git a/convert_hcl_to_json.sh b/scripts/convert_hcl_to_json.sh similarity index 66% rename from convert_hcl_to_json.sh rename to scripts/convert_hcl_to_json.sh index beea2f2..0446805 100755 --- a/convert_hcl_to_json.sh +++ b/scripts/convert_hcl_to_json.sh @@ -1,50 +1,12 @@ #!/bin/bash set -euo pipefail -log() { echo "[convert_hcl_to_json] $*" >&2; } +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=tools.sh +source "$SCRIPT_DIR/tools.sh" -WORKDIR=$(mktemp -d) -cleanup() { rm -rf "$WORKDIR"; } -trap cleanup EXIT - -# Normalize OS/arch to the names used by the jq and hcl2json release assets. -OS=$(uname -s) -case "$OS" in - Darwin) OS="macos" ;; - Linux) OS="linux" ;; - *) echo "Unsupported OS: $OS" >&2; exit 1 ;; -esac - -ARCH=$(uname -m) -case "$ARCH" in - x86_64 | amd64) ARCH="amd64" ;; - aarch64 | arm64) ARCH="arm64" ;; - *) echo "Unsupported architecture: $ARCH" >&2; exit 1 ;; -esac - -JQ_BIN="$WORKDIR/jq" -HCL2JSON_BIN="$WORKDIR/hcl2json" - -install_jq() { - local url="https://github.com/jqlang/jq/releases/download/jq-1.8.1/jq-${OS}-${ARCH}" - if ! curl -fsSL -o "$JQ_BIN" "$url"; then - echo "Failed to download jq from $url" >&2 - exit 1 - fi - chmod +x "$JQ_BIN" -} - -install_hcl2json() { - # hcl2json uses "darwin" rather than "macos" for the OS segment. - local hcl_os="$OS" - [[ "$hcl_os" == "macos" ]] && hcl_os="darwin" - local url="https://github.com/tmccombs/hcl2json/releases/download/v0.6.7/hcl2json_${hcl_os}_${ARCH}" - if ! curl -fsSL -o "$HCL2JSON_BIN" "$url"; then - echo "Failed to download hcl2json from $url" >&2 - exit 1 - fi - chmod +x "$HCL2JSON_BIN" -} +# Detail lines are shown only in verbose mode; warnings always show. +log() { [ "${SG_VERBOSE:-0}" = "1" ] || return 0; printf '%s[convert]%s %s\n' "$C_CYAN" "$C_RESET" "$*" >&2; } INPUT_FILE_JSON="${1:-}" if [ -z "$INPUT_FILE_JSON" ]; then @@ -56,16 +18,20 @@ if [ ! -f "$INPUT_FILE_JSON" ]; then exit 1 fi -log "Downloading jq and hcl2json..." -install_jq -install_hcl2json +WORKDIR=$(mktemp -d) +cleanup() { rm -rf "$WORKDIR"; } +trap cleanup EXIT + +# Resolve tooling from PATH (Docker image) or download+cache (native). +JQ_BIN=$(sg_resolve jq sg_ensure_jq) +HCL2JSON_BIN=$(sg_resolve hcl2json sg_ensure_hcl2json) # Read entire JSON array into a variable json_data=$(cat "$INPUT_FILE_JSON") # Use jq to get the length of array length=$($JQ_BIN length <<<"$json_data") -log "Processing $length workflow(s) from $INPUT_FILE_JSON" +log "Processing $length workflow(s) from $(sg_rel "$INPUT_FILE_JSON")" # Accumulate updated objects as newline-delimited JSON tmpfile="$WORKDIR/updated.ndjson" @@ -118,7 +84,7 @@ for ((i = 0; i < length; i++)); do log " workflow $((i + 1)): converted '$key' from HCL to JSON" new_val=$($JQ_BIN --arg k "$key" --argjson v "$parsed" '. + {($k): $v}' <<<"$new_val") else - log " workflow $((i + 1)): parsing failed, keeping original value for '$key'" + sg_warn "$(sg_rel "$INPUT_FILE_JSON") workflow $((i + 1)): could not parse '$key' as HCL; keeping original value" fi done < <($JQ_BIN -r 'keys[]' <<<"$val") @@ -133,4 +99,4 @@ done outfile="$WORKDIR/output.json" $JQ_BIN -s '.' "$tmpfile" >"$outfile" mv "$outfile" "$INPUT_FILE_JSON" -log "Done. Updated $INPUT_FILE_JSON in place." +log "Done. Updated $(sg_rel "$INPUT_FILE_JSON") in place." diff --git a/scripts/enrich_variable_sets.sh b/scripts/enrich_variable_sets.sh new file mode 100755 index 0000000..340548f --- /dev/null +++ b/scripts/enrich_variable_sets.sh @@ -0,0 +1,166 @@ +#!/bin/bash +# Enrich generated payloads with TFC/TFE Variable Set variables. +# +# The tfe provider can't enumerate variable sets, so we use the TFC API (same +# token as state export). For each workspace we compute the variable sets that +# apply (global / project-scoped / workspace-scoped), resolve set-vs-set +# precedence (priority + scope), and merge the result into the matching workflow +# payload. A workspace's own variable is only overridden by a *priority* set; +# otherwise set vars only fill keys the workspace doesn't define. Sensitive set +# vars can't be read from the API and are skipped + reported. +# +# Usage: enrich_variable_sets.sh [more.json ...] +set -uo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=tools.sh +source "$SCRIPT_DIR/tools.sh" + +ORG="${1:-}" +shift || true +if [ -z "$ORG" ] || [ "$#" -eq 0 ]; then + echo "Usage: $0 [more.json ...]" >&2 + exit 1 +fi + +HOST="${SG_TFC_HOSTNAME:-app.terraform.io}" +API="https://$HOST/api/v2" +command -v curl >/dev/null 2>&1 || { sg_err "curl is required for variable-set enrichment"; exit 1; } +JQ_BIN="$(sg_resolve jq sg_ensure_jq)" + +# TFC token (same sources as state export): credentials file or TFE_TOKEN. +creds="$HOME/.terraform.d/credentials.tfrc.json" +token="" +[ -f "$creds" ] && token="$("$JQ_BIN" -r --arg h "$HOST" '.credentials[$h].token // empty' "$creds" 2>/dev/null || true)" +[ -z "$token" ] && token="${TFE_TOKEN:-}" +if [ -z "$token" ]; then + sg_warn "no TFC token (terraform login / TFE_TOKEN); skipping variable-set enrichment" + exit 0 +fi + +WORK="$(mktemp -d)" +trap 'rm -rf "$WORK"' EXIT +AUTH=(-H "Authorization: Bearer $token") + +# fetch_all — GET a paginated JSON:API collection, print the merged .data array. +fetch_all() { + local path="$1" page=1 next + : >"$WORK/acc.ndjson" + while :; do + if ! curl -fsS "${AUTH[@]}" "$API/$path?page%5Bsize%5D=100&page%5Bnumber%5D=$page" >"$WORK/page.json"; then + sg_err "TFC API request failed: $path"; return 1 + fi + "$JQ_BIN" -c '.data[]?' "$WORK/page.json" >>"$WORK/acc.ndjson" + next="$("$JQ_BIN" -r '.meta.pagination."next-page" // empty' "$WORK/page.json" 2>/dev/null || true)" + [ -z "$next" ] && break + page="$next" + done + "$JQ_BIN" -s '.' "$WORK/acc.ndjson" +} + +sg_log "fetching workspaces and variable sets from $HOST (org: $ORG)..." + +# name -> {id, project} +fetch_all "organizations/$ORG/workspaces" \ + | "$JQ_BIN" '[.[] | {name: .attributes.name, id: .id, project: (.relationships.project.data.id // "")}]' \ + >"$WORK/workspaces.json" || exit 1 + +# Each set with its scope + variables. +: >"$WORK/sets.ndjson" +sets_raw="$(fetch_all "organizations/$ORG/varsets")" || exit 1 +echo "$sets_raw" | "$JQ_BIN" -c '.[]' | while IFS= read -r s; do + sid="$(echo "$s" | "$JQ_BIN" -r '.id')" + vars="$(curl -fsS "${AUTH[@]}" "$API/varsets/$sid/relationships/vars" 2>/dev/null \ + | "$JQ_BIN" -c '[.data[]? | {key: .attributes.key, value: (.attributes.value // ""), category: .attributes.category, sensitive: (.attributes.sensitive // false), hcl: (.attributes.hcl // false)}]' 2>/dev/null || echo '[]')" + echo "$s" | "$JQ_BIN" -c --argjson vars "$vars" '{ + name: .attributes.name, + global: (.attributes.global // false), + priority: (.attributes.priority // false), + wsids: [.relationships.workspaces.data[]?.id], + projids: [.relationships.projects.data[]?.id], + vars: $vars + }' >>"$WORK/sets.ndjson" +done +"$JQ_BIN" -s '.' "$WORK/sets.ndjson" >"$WORK/sets.json" + +set_count="$("$JQ_BIN" 'length' "$WORK/sets.json")" +if [ "$set_count" -eq 0 ]; then + sg_log "no variable sets found; nothing to enrich" + exit 0 +fi +sg_log "resolving $set_count variable set(s) across workspaces..." + +# Per workspace -> list of winning vars (set-vs-set precedence resolved; tagged +# with priority + sensitive + conflict). rank: non-priority global/proj/ws = 1/2/3, +# priority = 5/6/7; workspace's own vars sit at 4 and are applied during merge. +"$JQ_BIN" -n --slurpfile ws "$WORK/workspaces.json" --slurpfile sets "$WORK/sets.json" ' + ($ws[0]) as $workspaces | ($sets[0]) as $sets + | reduce $workspaces[] as $w ({}; + . + { ($w.name): ( + [ $sets[] + | . as $s + | (($s.global == true) + or (($s.wsids // []) | index($w.id) != null) + or (($w.project != "") and (($s.projids // []) | index($w.project) != null))) as $applies + | select($applies) + | (if (($s.wsids // []) | index($w.id) != null) then 3 + elif (($w.project != "") and (($s.projids // []) | index($w.project) != null)) then 2 + else 1 end) as $scope + | ($scope + (if $s.priority then 4 else 0 end)) as $rank + | ($s.vars[]? | {key, value, category, hcl, sensitive, rank: $rank, set: $s.name, priority: ($rank >= 5)}) + ] + | group_by(.key) + | map( (max_by(.rank)) as $win + | $win + { conflict: (([ .[] | select(.rank == $win.rank) ] | length) > 1) } ) + )} + ) +' >"$WORK/effective.json" + +# Merge the effective set vars into each payload, then report counts. +for f in "$@"; do + before_tf="$("$JQ_BIN" '[.[].VCSConfig.iacInputData.data | length] | add // 0' "$f")" + out="$WORK/merged.json" + "$JQ_BIN" --slurpfile eff "$WORK/effective.json" ' + ($eff[0]) as $E + | map( + ((.CLIConfiguration.TfStateFilePath // "") | sub(".*/"; "") | sub("\\.tfstate$"; "")) as $wsName + | ($E[$wsName] // []) as $all + | ($all | map(select(.sensitive != true and .category == "terraform"))) as $tf + | ($all | map(select(.sensitive != true and .category == "env"))) as $env + | .VCSConfig.iacInputData.data = ( + reduce $tf[] as $v ((.VCSConfig.iacInputData.data // {}); + ($v.value | (fromjson? // $v.value)) as $val + | if (has($v.key) | not) then . + {($v.key): $val} + elif $v.priority then . + {($v.key): $val} + else . end)) + | .EnvironmentVariables = ( + reduce $env[] as $v ((.EnvironmentVariables // []); + (map(.config.varName) | index($v.key)) as $idx + | if ($idx == null) then . + [{config: {textValue: $v.value, varName: $v.key}, kind: "PLAIN_TEXT"}] + elif $v.priority then (.[$idx].config.textValue = $v.value) + else . end)) + ) + ' "$f" >"$out" && mv "$out" "$f" + after_tf="$("$JQ_BIN" '[.[].VCSConfig.iacInputData.data | length] | add // 0' "$f")" + sg_log "$(basename "$f"): +$((after_tf - before_tf)) terraform var(s) from variable sets" +done + +# Report sensitive set vars (cannot be migrated) and key conflicts. +"$JQ_BIN" -r ' + to_entries[] | .key as $ws | .value[] + | select(.sensitive == true) | " - \($ws): \(.category):\(.key) (set \(.set))" +' "$WORK/effective.json" | sort -u >"$WORK/sensitive.txt" +if [ -s "$WORK/sensitive.txt" ]; then + sg_warn "sensitive variable-set vars skipped (recreate as SG secrets):" + cat "$WORK/sensitive.txt" >&2 +fi +"$JQ_BIN" -r ' + to_entries[] | .key as $ws | .value[] + | select(.conflict == true) | " - \($ws): \(.category):\(.key)" +' "$WORK/effective.json" | sort -u >"$WORK/conflicts.txt" +if [ -s "$WORK/conflicts.txt" ]; then + sg_warn "variable-set key conflicts (same key in multiple equal-precedence sets; picked one):" + cat "$WORK/conflicts.txt" >&2 +fi + +sg_success "variable-set enrichment complete" diff --git a/scripts/migrate.sh b/scripts/migrate.sh new file mode 100755 index 0000000..4adc693 --- /dev/null +++ b/scripts/migrate.sh @@ -0,0 +1,443 @@ +#!/bin/bash +# End-to-end StackGuardian migration orchestrator. +# +# Runs apply -> convert -> validate -> import, resolving tooling from PATH first +# (e.g. inside the Docker image) and falling back to cached downloads when run +# natively. Designed to run both on the host and inside the migrator container. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=tools.sh +source "$SCRIPT_DIR/tools.sh" + +# SCRIPT_DIR holds the sibling scripts; SG_REPO_ROOT (from tools.sh) is the repo +# root used for all repo-relative paths. +TRANSFORMER_DIR="$SG_REPO_ROOT/transformer/terraform-cloud" +TFVARS="$TRANSFORMER_DIR/terraform.tfvars" +EXPORT_DIR="${SG_EXPORT_DIR:-$SG_REPO_ROOT/export}" +MAPPING="${SG_WFGROUP_MAP:-$SG_REPO_ROOT/.sg/workflow-groups.json}" +ORG="${SG_ORG:-}" +SG_BASE_URL="${SG_BASE_URL:-https://api.app.stackguardian.io}" +ASSUME_YES=0 +PURGE=0 +CREATE_GROUPS=1 +ENRICH_VARSETS=1 +VCS_TRIGGERS=1 +VERBOSE="${SG_VERBOSE:-0}" +CONC="${SG_CONCURRENCY:-4}" +TF_PARALLELISM="${SG_TF_PARALLELISM:-20}" +RETRIES="${SG_RETRIES:-4}" +RETRY_BASE="${SG_RETRY_BASE:-2}" +PF=() + +usage() { + cat >&2 < enrich -> convert -> validate -> import (default) + clean Remove local working artifacts for a fresh start (export/, TF state, + tool cache). Add --all to also remove config (terraform.tfvars, mapping). + +Each TFC project maps to an SG workflow group named tfc-, created via the +API if missing. Override a project's target group in .sg/workflow-groups.json +(\`{"": ""}\`); override groups are not auto-created. + +Options: + --org NAME StackGuardian org for import (or set SG_ORG) + --export-dir DIR Payload/state output dir (default: ./export) + --mapping FILE Optional project-segment -> group override map (default: .sg/workflow-groups.json) + --concurrency N Max parallel jobs for convert/import (default: 4) + --no-create-groups Do not create missing workflow groups; require them to exist + --no-variable-sets Skip merging TFC Variable Set variables in the 'all' flow + --no-vcs-triggers Skip registering VCS triggers after import + --all With 'clean': also remove config (terraform.tfvars, mapping, .sg) + -v, --verbose Show full terraform/tool output (default: concise) + -y, --yes Skip the import confirmation prompt + -h, --help Show this help + +Environment: + SG_API_TOKEN StackGuardian API token (required for import) + SG_ORG StackGuardian org (alternative to --org) + SG_RETRIES Import retry attempts on failure (default: 4) + SG_TF_PARALLELISM terraform apply -parallelism (default: 20) +EOF +} + +die() { sg_err "$*"; exit 1; } + +throttle() { while [ "$(jobs -rp | wc -l | tr -d ' ')" -ge "$1" ]; do sleep 0.2; done; } + +# run_parallel — runs fn over items, up to max at a time. +# Each job's stdout+stderr is buffered to its own file (so concurrent tools see +# a non-TTY and don't scatter spinner output across the terminal), then flushed +# as a clean labeled block in submission order. Returns non-zero if any failed. +run_parallel() { + local fn="$1" max="$2"; shift 2 + local statusdir i=0 rc=0 item + statusdir="$(mktemp -d)" + for item in "$@"; do + throttle "$max" + ( "$fn" "$item" >"$statusdir/$i.out" 2>&1; echo "$?" >"$statusdir/$i.rc" ) & + i=$((i + 1)) + done + wait + i=0 + for item in "$@"; do + printf '%s── %s ──%s\n' "$C_CYAN" "$(basename "$item")" "$C_RESET" >&2 + [ -s "$statusdir/$i.out" ] && cat "$statusdir/$i.out" >&2 + [ "$(cat "$statusdir/$i.rc" 2>/dev/null)" = "0" ] || rc=1 + i=$((i + 1)) + done + rm -rf "$statusdir" + return "$rc" +} + +# Populate PF with the generated payload files. +payload_files() { + shopt -s nullglob + PF=("$EXPORT_DIR"/sg-payload.*.json) + shopt -u nullglob +} + +seg_of() { local b; b="$(basename "$1")"; b="${b#sg-payload.}"; echo "${b%.json}"; } + +cmd_init() { + sg_step "Phase: init" + mkdir -p "$SG_REPO_ROOT/.sg" "$SG_CACHE_BIN" + if [ ! -f "$TFVARS" ]; then + cp "$TRANSFORMER_DIR/terraform.tfvars.example" "$TFVARS" + sg_log "created $(sg_rel "$TFVARS") — edit it before 'apply'" + else + sg_log "$(sg_rel "$TFVARS") already exists" + fi + sg_log "workflow groups are created automatically as tfc-; no mapping needed" + sg_success "init complete" +} + +# --- StackGuardian API helpers (workflow groups) --------------------------- + +# group_for -> the SG workflow group for a project segment: an entry +# from the optional override map, else the default tfc-. +group_for() { + local seg="$1" override="" + if [ -f "$MAPPING" ]; then + override="$("$JQ_BIN" -r --arg k "$seg" '.[$k] // empty' "$MAPPING" 2>/dev/null || true)" + fi + [ -n "$override" ] && echo "$override" || echo "tfc-$seg" +} + +# wfgroup_http_code -> HTTP status of GET (200 exists, 404 missing). +wfgroup_http_code() { + curl -sS -o /dev/null -w '%{http_code}' \ + -H "Authorization: apikey $SG_API_TOKEN" \ + "$SG_BASE_URL/api/v1/orgs/$ORG/wfgrps/$1/" +} + +# wfgroup_create — create a workflow group (idempotent at call sites). +wfgroup_create() { + local body + body="$("$JQ_BIN" -nc --arg n "$1" '{ResourceName:$n, Description:"Created by stackguardian-migrator (Terraform Cloud import)"}')" + sg_retry "$RETRIES" "$RETRY_BASE" -- \ + curl -fsS -X POST \ + -H "Authorization: apikey $SG_API_TOKEN" -H "Content-Type: application/json" \ + -d "$body" "$SG_BASE_URL/api/v1/orgs/$ORG/wfgrps/" >/dev/null +} + +# wf_triggers_endpoint -> the VCS-triggers webhook URL for a workflow. +wf_triggers_endpoint() { + echo "$SG_BASE_URL/api/v1/orgs/$ORG/wfgrps/$1/wfs/$2/webhooks/vcs_triggers/" +} + +# do_set_triggers — for each workflow in the file with a non-null +# VCSTriggers block, register its VCS triggers via the dedicated webhooks +# endpoint (the bulk create API silently drops VCSTriggers; this second pass is +# what actually wires up the repo webhook). Sends {VCSConfig, VCSTriggers} taken +# straight from the (converted) payload. Per-workflow failures are surfaced but +# do not abort the rest of the file. +do_set_triggers() { + local f="$1" seg grp n i wf body rc=0 set=0 skip=0 + seg="$(seg_of "$f")" + grp="$(group_for "$seg")" + n="$("$JQ_BIN" 'length' "$f")" + for ((i = 0; i < n; i++)); do + if [ "$("$JQ_BIN" -r --argjson i "$i" '(.[$i].VCSTriggers // null) != null' "$f")" != "true" ]; then + skip=$((skip + 1)); continue + fi + wf="$("$JQ_BIN" -r --argjson i "$i" '.[$i].ResourceName' "$f")" + body="$("$JQ_BIN" -c --argjson i "$i" '{VCSConfig: .[$i].VCSConfig, VCSTriggers: .[$i].VCSTriggers}' "$f")" + if sg_retry "$RETRIES" "$RETRY_BASE" -- \ + curl -fsS -X POST \ + -H "Authorization: apikey $SG_API_TOKEN" -H "Content-Type: application/json" \ + -d "$body" "$(wf_triggers_endpoint "$grp" "$wf")" >/dev/null; then + set=$((set + 1)) + else + sg_warn " vcs triggers failed: $grp/$wf"; rc=1 + fi + done + sg_log "$(basename "$f"): set triggers on $set workflow(s) (skipped $skip without triggers)" + return "$rc" +} + +# set_triggers_pass — run do_set_triggers over all payload files (assumes +# JQ_BIN/SG_API_TOKEN/ORG are already set up by the caller). +set_triggers_pass() { + local total + total="$("$JQ_BIN" -s 'map(map(select((.VCSTriggers // null) != null)) | length) | add // 0' "${PF[@]}")" + if [ "$total" -eq 0 ]; then + sg_log "no workflows carry VCS triggers — nothing to register" + return 0 + fi + sg_log "registering VCS triggers for $total workflow(s), up to $CONC in parallel (retries: $RETRIES)" + if run_parallel do_set_triggers "$CONC" "${PF[@]}"; then + sg_success "vcs triggers registered" + else + sg_err "one or more VCS trigger registrations failed (re-run: $0 triggers)"; return 1 + fi +} + +cmd_triggers() { + sg_step "Phase: vcs triggers" + [ -n "${SG_API_TOKEN:-}" ] || die "SG_API_TOKEN is not set." + [ -n "$ORG" ] || die "StackGuardian org not set (use --org or SG_ORG)." + command -v curl >/dev/null 2>&1 || die "curl is required for VCS trigger registration." + export SG_API_TOKEN SG_BASE_URL + JQ_BIN="$(sg_resolve jq sg_ensure_jq)" + payload_files + [ "${#PF[@]}" -gt 0 ] || die "No payload files in $(sg_rel "$EXPORT_DIR")." + set_triggers_pass +} + +cmd_clean() { + sg_step "Phase: clean" + sg_log "removing local working artifacts..." + rm -rf "$EXPORT_DIR" + rm -rf "$TRANSFORMER_DIR/.terraform" "$TRANSFORMER_DIR/.terraform.lock.hcl" \ + "$TRANSFORMER_DIR/terraform.tfstate" "$TRANSFORMER_DIR/terraform.tfstate.backup" + rm -rf "$SG_CACHE_DIR" + if [ "$PURGE" -eq 1 ]; then + rm -f "$TFVARS" "$MAPPING" + rm -rf "$SG_REPO_ROOT/.sg" + sg_log "also removed config (terraform.tfvars, workflow-groups.json, .sg)" + fi + sg_success "clean complete" +} + +cmd_apply() { + sg_step "Phase: apply (terraform)" + command -v terraform >/dev/null 2>&1 || die "terraform not found on PATH" + [ -f "$TFVARS" ] || die "Missing $(sg_rel "$TFVARS"). Run: $0 init (then edit it)." + # State export (TFC API) calls curl + jq from terraform's local-exec; make sure + # both are on PATH for the apply (jq from cache if not already installed). + command -v curl >/dev/null 2>&1 || die "curl is required for state export" + local jqdir tflog rc=0 + jqdir="$(dirname "$(sg_resolve jq sg_ensure_jq)")" + + if [ "$VERBOSE" -eq 1 ]; then + ( cd "$TRANSFORMER_DIR" && export PATH="$jqdir:$PATH" TF_IN_AUTOMATION=1 \ + && terraform init -input=false \ + && terraform apply -auto-approve -compact-warnings -parallelism="$TF_PARALLELISM" -var-file=terraform.tfvars ) || rc=$? + else + # Quiet: capture terraform's verbose plan/output; surface only progress, the + # final summary, and (on failure) the captured log. + tflog="$(mktemp)" + sg_log "initializing terraform (providers)..." + ( cd "$TRANSFORMER_DIR" && export PATH="$jqdir:$PATH" TF_IN_AUTOMATION=1 && terraform init -input=false -no-color ) >"$tflog" 2>&1 || rc=$? + if [ "$rc" -eq 0 ]; then + sg_log "reading workspaces, generating payloads, exporting state..." + ( cd "$TRANSFORMER_DIR" && export PATH="$jqdir:$PATH" TF_IN_AUTOMATION=1 \ + && terraform apply -auto-approve -compact-warnings -no-color -parallelism="$TF_PARALLELISM" -var-file=terraform.tfvars ) >"$tflog" 2>&1 || rc=$? + fi + if [ "$rc" -ne 0 ]; then + sg_err "terraform failed (rc=$rc):"; cat "$tflog" >&2 + else + grep -E '^(Apply complete|No changes)' "$tflog" | sed 's/^/ /' >&2 || true + fi + rm -f "$tflog" + fi + + [ "$rc" -eq 0 ] || return "$rc" + sg_success "apply complete — payloads in $(sg_rel "$EXPORT_DIR")" +} + +cmd_enrich() { + sg_step "Phase: variable sets" + [ -f "$TFVARS" ] || die "Missing $(sg_rel "$TFVARS") (run: $0 init)." + payload_files + [ "${#PF[@]}" -gt 0 ] || die "No payload files in $(sg_rel "$EXPORT_DIR") (run 'apply' first)." + # Variable sets belong to the TFC org (tfOrg in terraform.tfvars), not SG_ORG. + local jqb h2j tforg tfhost + jqb="$(sg_resolve jq sg_ensure_jq)" + h2j="$(sg_resolve hcl2json sg_ensure_hcl2json)" + tforg="$("$h2j" "$TFVARS" | "$jqb" -r '.tfOrg // empty')" + [ -n "$tforg" ] || die "tfOrg not found in $(sg_rel "$TFVARS")" + tfhost="$("$h2j" "$TFVARS" | "$jqb" -r '.tfHostname // empty')" + SG_TFC_HOSTNAME="${tfhost:-app.terraform.io}" "$SCRIPT_DIR/enrich_variable_sets.sh" "$tforg" "${PF[@]}" +} + +do_convert() { "$SCRIPT_DIR/convert_hcl_to_json.sh" "$1"; } + +cmd_convert() { + sg_step "Phase: convert (HCL → JSON)" + payload_files + [ "${#PF[@]}" -gt 0 ] || die "No payload files in $(sg_rel "$EXPORT_DIR") (run 'apply' first)." + sg_log "converting ${#PF[@]} payload(s), up to $CONC in parallel" + if run_parallel do_convert "$CONC" "${PF[@]}"; then + sg_success "converted ${#PF[@]} payload(s)" + else + sg_err "conversion failed for one or more payloads"; return 1 + fi +} + +cmd_validate() { + sg_step "Phase: validate" + payload_files + [ "${#PF[@]}" -gt 0 ] || die "No payload files in $(sg_rel "$EXPORT_DIR")." + if "$SCRIPT_DIR/validate_payload.sh" "${PF[@]}"; then + sg_success "all ${#PF[@]} payload(s) valid" + else + sg_err "validation failed"; return 1 + fi +} + +do_import() { + local f="$1" seg grp + seg="$(seg_of "$f")" + grp="$(group_for "$seg")" + sg_log "importing $(basename "$f") -> $grp" + sg_retry "$RETRIES" "$RETRY_BASE" -- \ + "$SGCLI_BIN" workflow create --bulk --workflow-group "$grp" --org "$ORG" "$f" +} + +cmd_import() { + sg_step "Phase: import" + [ -n "${SG_API_TOKEN:-}" ] || die "SG_API_TOKEN is not set." + [ -n "$ORG" ] || die "StackGuardian org not set (use --org or SG_ORG)." + command -v curl >/dev/null 2>&1 || die "curl is required for workflow-group checks/creation." + # Both our API calls and sg-cli honor SG_BASE_URL (e.g. non-prod); export it + # so the sg-cli child process inherits the same target. + export SG_API_TOKEN SG_BASE_URL + JQ_BIN="$(sg_resolve jq sg_ensure_jq)" + + payload_files + [ "${#PF[@]}" -gt 0 ] || die "No payload files in $(sg_rel "$EXPORT_DIR")." + + # Build the plan: resolve each project's group, check existence, and decide + # which groups need creating. Override groups (from the map) must already exist. + local fail=0 to_create=" " f seg grp count override is_override code status + printf '%sImport plan%s (org: %s%s%s, %s)\n' "$C_BOLD" "$C_RESET" "$C_CYAN" "$ORG" "$C_RESET" "$SG_BASE_URL" >&2 + printf ' %s%-34s %-26s %-9s %s%s\n' "$C_BOLD" "FILE" "WORKFLOW GROUP" "WORKFLOWS" "STATUS" "$C_RESET" >&2 + for f in "${PF[@]}"; do + seg="$(seg_of "$f")" + count="$("$JQ_BIN" 'length' "$f")" + override="" + [ -f "$MAPPING" ] && override="$("$JQ_BIN" -r --arg k "$seg" '.[$k] // empty' "$MAPPING" 2>/dev/null || true)" + if [ -n "$override" ]; then grp="$override"; is_override=1; else grp="tfc-$seg"; is_override=0; fi + + code="$(wfgroup_http_code "$grp")" + case "$code" in + 200) status="${C_GREEN}exists${C_RESET}" ;; + 404) + if [ "$is_override" -eq 1 ]; then + status="${C_RED}missing!${C_RESET}"; fail=1 + elif [ "$CREATE_GROUPS" -eq 1 ]; then + status="${C_YELLOW}create${C_RESET}" + case "$to_create" in *" $grp "*) ;; *) to_create="$to_create$grp " ;; esac + else + status="${C_RED}missing!${C_RESET}"; fail=1 + fi ;; + 401 | 403) die "auth failed (HTTP $code) for org '$ORG' — check SG_API_TOKEN" ;; + 000) die "could not reach $SG_BASE_URL" ;; + *) die "unexpected HTTP $code checking group '$grp'" ;; + esac + printf ' %-34s %-26s %-9s %s\n' "$(basename "$f")" "$grp" "$count" "$status" >&2 + done + if [ "$fail" -ne 0 ]; then + die "some groups are missing (override groups are not auto-created; create them or remove the override)." + fi + + if [ "$ASSUME_YES" -ne 1 ]; then + printf '%sProceed?%s This imports to %s and creates any "create" groups. [y/N] ' "$C_BOLD$C_YELLOW" "$C_RESET" "$ORG" >&2 + read -r ans || ans="" + case "$ans" in y | Y | yes | YES) ;; *) die "Aborted." ;; esac + fi + + # Create the missing tfc-* groups before importing into them. + for grp in $to_create; do + sg_log "creating workflow group $grp" + wfgroup_create "$grp" || die "failed to create workflow group $grp" + done + + SGCLI_BIN="$(sg_resolve sg-cli sg_ensure_sgcli)" + sg_log "importing ${#PF[@]} payload(s), up to $CONC in parallel (retries: $RETRIES)" + if run_parallel do_import "$CONC" "${PF[@]}"; then + sg_success "import complete (${#PF[@]} payload(s))" + else + sg_err "one or more imports failed"; return 1 + fi + + # VCS triggers are not accepted by the bulk create API; register them in a + # second pass against the dedicated webhooks endpoint (skip with --no-vcs-triggers). + if [ "$VCS_TRIGGERS" -eq 1 ]; then + set_triggers_pass + fi +} + +CMD="" +while [ $# -gt 0 ]; do + case "$1" in + -y | --yes) ASSUME_YES=1 ;; + --org) ORG="$2"; shift ;; + --org=*) ORG="${1#*=}" ;; + --export-dir) EXPORT_DIR="$2"; shift ;; + --export-dir=*) EXPORT_DIR="${1#*=}" ;; + --mapping) MAPPING="$2"; shift ;; + --mapping=*) MAPPING="${1#*=}" ;; + --concurrency) CONC="$2"; shift ;; + --concurrency=*) CONC="${1#*=}" ;; + --no-create-groups) CREATE_GROUPS=0 ;; + --no-variable-sets) ENRICH_VARSETS=0 ;; + --no-vcs-triggers) VCS_TRIGGERS=0 ;; + -v | --verbose) VERBOSE=1 ;; + --all) PURGE=1 ;; + -h | --help) usage; exit 0 ;; + init | apply | enrich | convert | validate | import | triggers | all | clean) CMD="$1" ;; + *) echo "Unknown argument: $1" >&2; usage; exit 1 ;; + esac + shift +done +CMD="${CMD:-all}" +export SG_VERBOSE="$VERBOSE" + +case "$CMD" in + init) cmd_init ;; + clean) cmd_clean ;; + apply) cmd_apply ;; + enrich) cmd_enrich ;; + convert) cmd_convert ;; + validate) cmd_validate ;; + import) cmd_import ;; + triggers) cmd_triggers ;; + all) + if [ ! -f "$TFVARS" ]; then + cmd_init + die "Edit $(sg_rel "$TFVARS"), then re-run '$0 all'." + fi + # Fail fast on import prerequisites before the (long) apply. + [ -n "${SG_API_TOKEN:-}" ] || die "SG_API_TOKEN is not set (needed for import)." + [ -n "$ORG" ] || die "StackGuardian org not set (use --org or SG_ORG)." + cmd_apply + if [ "$ENRICH_VARSETS" -eq 1 ]; then cmd_enrich; fi + cmd_convert + cmd_validate + cmd_import + ;; +esac diff --git a/scripts/tools.sh b/scripts/tools.sh new file mode 100755 index 0000000..f3b568f --- /dev/null +++ b/scripts/tools.sh @@ -0,0 +1,162 @@ +#!/bin/bash +# Shared tool bootstrap + cache for the StackGuardian migrator. +# +# Source this file; it exposes the repo root and sg_* helpers that download and +# cache the required CLIs under .sg/cached/ (override with SG_CACHE_DIR). Each +# sg_ensure_* function prints the absolute path to a ready-to-run binary on +# stdout; all human-facing logs go to stderr so the path can be captured with +# command substitution: JQ_BIN=$(sg_ensure_jq). + +# This file lives in scripts/; the repo root is its parent directory. +SG_REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +SG_CACHE_DIR="${SG_CACHE_DIR:-$SG_REPO_ROOT/.sg/cached}" +SG_CACHE_BIN="$SG_CACHE_DIR/bin" + +# Pinned versions. +SG_JQ_VERSION="jq-1.8.1" +SG_HCL2JSON_VERSION="v0.6.7" +SG_YAJSV_VERSION="v1.4.1" + +# Colored logging — disabled when stderr is not a TTY, NO_COLOR is set, or +# TERM=dumb, so piped/CI output stays clean. +if [ -t 2 ] && [ -z "${NO_COLOR:-}" ] && [ "${TERM:-}" != "dumb" ]; then + C_RESET=$'\033[0m'; C_BOLD=$'\033[1m'; C_DIM=$'\033[2m' + C_RED=$'\033[31m'; C_GREEN=$'\033[32m'; C_YELLOW=$'\033[33m'; C_CYAN=$'\033[36m' +else + C_RESET=""; C_BOLD=""; C_DIM=""; C_RED=""; C_GREEN=""; C_YELLOW=""; C_CYAN="" +fi + +# sg_rel — render a path relative to the repo root for readable logs +# (operations still use absolute paths; this is display-only). +sg_rel() { + case "$1" in + "$SG_REPO_ROOT"/*) printf '%s' "${1#"$SG_REPO_ROOT"/}" ;; + "$SG_REPO_ROOT") printf '.' ;; + *) printf '%s' "$1" ;; + esac +} + +sg_log() { printf '%s[sg-migrate]%s %s\n' "$C_CYAN" "$C_RESET" "$*" >&2; } +sg_warn() { printf '%s[sg-migrate] WARN%s %s\n' "$C_YELLOW" "$C_RESET" "$*" >&2; } +sg_err() { printf '%s[sg-migrate] ERROR%s %s\n' "$C_RED$C_BOLD" "$C_RESET" "$*" >&2; } +sg_success() { printf '%s[sg-migrate] ✓%s %s\n' "$C_GREEN$C_BOLD" "$C_RESET" "$*" >&2; } +sg_step() { printf '\n%s==> %s%s\n' "$C_CYAN$C_BOLD" "$*" "$C_RESET" >&2; } +sg_dim() { printf '%s %s%s\n' "$C_DIM" "$*" "$C_RESET" >&2; } + +sg_arch() { + case "$(uname -m)" in + x86_64 | amd64) echo "amd64" ;; + aarch64 | arm64) echo "arm64" ;; + *) echo "Unsupported architecture: $(uname -m)" >&2; return 1 ;; + esac +} + +# sg_retry -- +# Runs the command, retrying with exponential backoff (capped at 60s, with a +# little jitter) until it succeeds or attempts are exhausted. Returns the +# command's last exit code. Used to ride out transient API failures/rate limits. +sg_retry() { + local max="$1" base="$2"; shift 2 + [ "${1:-}" = "--" ] && shift + local attempt=1 delay="$base" rc=0 + while :; do + if "$@"; then return 0; fi + rc=$? + if [ "$attempt" -ge "$max" ]; then + sg_err "failed after ${attempt} attempt(s) (exit ${rc}): $*" + return "$rc" + fi + local jitter=$((RANDOM % (base + 1))) + sg_warn "attempt ${attempt}/${max} failed (exit ${rc}); retrying in $((delay + jitter))s..." + sleep "$((delay + jitter))" + attempt=$((attempt + 1)) + delay=$((delay * 2)) + [ "$delay" -gt 60 ] && delay=60 + done +} + +# sg_resolve : prefer a binary already on PATH (e.g. +# installed in the Docker image); otherwise download+cache via the ensure fn. +# Prints the path to use on stdout. +sg_resolve() { + local name="$1" ensure="$2" + if command -v "$name" >/dev/null 2>&1; then command -v "$name"; return 0; fi + if [ -n "$ensure" ]; then "$ensure"; return $?; fi + echo "Required tool '$name' not found on PATH" >&2 + return 1 +} + +# sg_download +sg_download() { + local url="$1" dest="$2" + command -v curl >/dev/null 2>&1 || { echo "curl is required but not found" >&2; return 1; } + mkdir -p "$(dirname "$dest")" + if ! curl -fsSL -o "$dest" "$url"; then + echo "Failed to download: $url" >&2 + return 1 + fi +} + +sg_ensure_jq() { + local bin="$SG_CACHE_BIN/jq" arch os + if [ ! -x "$bin" ]; then + arch=$(sg_arch) || return 1 + case "$(uname -s)" in Darwin) os="macos" ;; Linux) os="linux" ;; *) echo "Unsupported OS: $(uname -s)" >&2; return 1 ;; esac + sg_log "caching jq ${SG_JQ_VERSION}..." + sg_download "https://github.com/jqlang/jq/releases/download/${SG_JQ_VERSION}/jq-${os}-${arch}" "$bin" || return 1 + chmod +x "$bin" + fi + echo "$bin" +} + +sg_ensure_hcl2json() { + local bin="$SG_CACHE_BIN/hcl2json" arch os + if [ ! -x "$bin" ]; then + arch=$(sg_arch) || return 1 + case "$(uname -s)" in Darwin) os="darwin" ;; Linux) os="linux" ;; *) echo "Unsupported OS: $(uname -s)" >&2; return 1 ;; esac + sg_log "caching hcl2json ${SG_HCL2JSON_VERSION}..." + sg_download "https://github.com/tmccombs/hcl2json/releases/download/${SG_HCL2JSON_VERSION}/hcl2json_${os}_${arch}" "$bin" || return 1 + chmod +x "$bin" + fi + echo "$bin" +} + +sg_ensure_yajsv() { + local bin="$SG_CACHE_BIN/yajsv" arch os + if [ ! -x "$bin" ]; then + arch=$(sg_arch) || return 1 + case "$(uname -s)" in Darwin) os="darwin" ;; Linux) os="linux" ;; *) echo "Unsupported OS: $(uname -s)" >&2; return 1 ;; esac + if [ "$os" = "linux" ] && [ "$arch" = "arm64" ]; then + echo "No yajsv prebuilt binary for linux/arm64; install yajsv manually." >&2 + return 1 + fi + sg_log "caching yajsv ${SG_YAJSV_VERSION}..." + sg_download "https://github.com/neilpa/yajsv/releases/download/${SG_YAJSV_VERSION}/yajsv.${os}.${arch}" "$bin" || return 1 + chmod +x "$bin" + fi + echo "$bin" +} + +# Caches the sg-cli Go binary (latest release) for the host OS/arch and prints +# its path. Release assets are named sg-cli__.tar.gz (Darwin/Linux, +# arm64/x86_64). +sg_ensure_sgcli() { + local bin="$SG_CACHE_BIN/sg-cli" os arch tmp realcli + if [ ! -x "$bin" ]; then + case "$(uname -s)" in Darwin) os="Darwin" ;; Linux) os="Linux" ;; *) echo "Unsupported OS: $(uname -s)" >&2; return 1 ;; esac + case "$(uname -m)" in x86_64 | amd64) arch="x86_64" ;; aarch64 | arm64) arch="arm64" ;; *) echo "Unsupported architecture: $(uname -m)" >&2; return 1 ;; esac + sg_log "caching sg-cli (latest release, ${os}/${arch})..." + tmp=$(mktemp -d) + if ! curl -fsSL "https://github.com/StackGuardian/sg-cli/releases/latest/download/sg-cli_${os}_${arch}.tar.gz" -o "$tmp/sg-cli.tar.gz"; then + echo "Failed to download sg-cli" >&2; rm -rf "$tmp"; return 1 + fi + tar -xzf "$tmp/sg-cli.tar.gz" -C "$tmp" + realcli=$(find "$tmp" -maxdepth 2 -type f -name sg-cli | head -1) + [ -n "$realcli" ] || { echo "sg-cli binary not found in release archive" >&2; rm -rf "$tmp"; return 1; } + mkdir -p "$SG_CACHE_BIN" + cp "$realcli" "$bin" + chmod +x "$bin" + rm -rf "$tmp" + fi + echo "$bin" +} diff --git a/sg-migrate.sh b/sg-migrate.sh new file mode 100755 index 0000000..d39287d --- /dev/null +++ b/sg-migrate.sh @@ -0,0 +1,69 @@ +#!/bin/bash +# Host entrypoint for the StackGuardian migrator. +# +# Runs scripts/migrate.sh inside the Docker image (so all tooling is isolated and +# identical across OSes). Mounts the repo at /app, mounts the Terraform Cloud +# credentials file read-only, and forwards SG_API_TOKEN/SG_ORG. +# +# Runs natively (no Docker) when: --native/--local is passed, SG_NATIVE=1 is set, +# the command is 'clean' (a local filesystem op), or docker is unavailable. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=scripts/tools.sh +source "$SCRIPT_DIR/scripts/tools.sh" +IMAGE="${SG_IMAGE:-stackguardian/migrator:local}" +CREDS="${TF_CREDENTIALS_FILE:-$HOME/.terraform.d/credentials.tfrc.json}" + +# Parse host-only flags (--native/--local, --build); everything else passes through. +NATIVE="${SG_NATIVE:-0}" +BUILD=0 +ARGS=() +for a in "$@"; do + case "$a" in + --native | --local) NATIVE=1 ;; + --build) BUILD=1 ;; + *) ARGS+=("$a") ;; + esac +done + +# 'clean' only touches the local filesystem — no container needed. +for a in ${ARGS[@]+"${ARGS[@]}"}; do + [ "$a" = "clean" ] && NATIVE=1 +done + +if [ "$NATIVE" = "1" ] || ! command -v docker >/dev/null 2>&1; then + [ "$NATIVE" = "1" ] || sg_warn "docker not found; running natively" + exec "$SCRIPT_DIR/scripts/migrate.sh" ${ARGS[@]+"${ARGS[@]}"} +fi + +if [ "$BUILD" = "1" ] || ! docker image inspect "$IMAGE" >/dev/null 2>&1; then + sg_log "building image $IMAGE ..." + docker build -t "$IMAGE" "$SCRIPT_DIR" +fi + +DOCKER_ARGS=(--rm -i + -v "$SCRIPT_DIR:/app" -w /app + -e SG_API_TOKEN -e SG_ORG -e SG_BASE_URL -e SG_CONCURRENCY -e SG_RETRIES -e SG_TF_PARALLELISM + -e TFE_TOKEN) + +# Interactive TTY only when attached to one (so the confirmation prompt works, +# but CI/non-tty invocations still run — use -y there). +if [ -t 0 ] && [ -t 1 ]; then DOCKER_ARGS+=(-t); fi + +# TFC auth: prefer a long-lived TFE_TOKEN (forwarded via -e above); otherwise +# mount the `terraform login` credentials file read-only. +if [ -n "${TFE_TOKEN:-}" ]; then + : +elif [ -f "$CREDS" ]; then + DOCKER_ARGS+=(-v "$CREDS:/root/.terraform.d/credentials.tfrc.json:ro") +else + sg_warn "no TFC auth found — set TFE_TOKEN (long-lived API token) or run 'terraform login'" +fi + +# Forward any TF_TOKEN_* env vars (alternative TFC/TFE auth) if present. +while IFS='=' read -r name _; do + case "$name" in TF_TOKEN_*) DOCKER_ARGS+=(-e "$name") ;; esac +done < <(env) + +exec docker run "${DOCKER_ARGS[@]}" "$IMAGE" /app/scripts/migrate.sh ${ARGS[@]+"${ARGS[@]}"} From 2e0242d0a4f1a37962cb7923574e565595687763 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adis=20Halilovi=C4=87?= Date: Wed, 24 Jun 2026 19:06:18 +0200 Subject: [PATCH 04/71] docs: update README for the dockerized pipeline --- README.md | 84 ++++++++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 67 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index a10bfd3..0b884fe 100644 --- a/README.md +++ b/README.md @@ -12,16 +12,44 @@ Migrate workloads from other platforms to [StackGuardian Platform](https://app.s - Review the bulk workflow creation payload. - Run sg-cli with the bulk workflow creation payload. +## Quick start (orchestrated) + +`./sg-migrate.sh` runs the whole flow — `terraform apply` → HCL→JSON conversion → schema validation → bulk import — with all tooling (Terraform, `jq`, `hcl2json`, `yajsv`, `sg-cli`) isolated in a Docker image, so it behaves identically on Linux, macOS, and Windows. Docker is required for this path; without it the script automatically falls back to running natively (downloading pinned tools into `.sg/cached/`). + +```shell +export TFE_TOKEN= # long-lived API token (User/Team/Org token from the TFC UI) +export SG_API_TOKEN= +export SG_ORG= + +./sg-migrate.sh init # scaffolds terraform.tfvars +# edit transformer/terraform-cloud/terraform.tfvars (org, integrations, workspaceOverrides) + +./sg-migrate.sh all # apply -> enrich -> convert -> validate -> import (prompts before importing) +``` + +That's it — no workflow-group mapping to fill in. Each TFC project is imported into an SG workflow group named `tfc-`, **created automatically via the API** if it doesn't exist. The import prompt shows each group as `exists` or `create` before anything is written. + +- Single phase: `./sg-migrate.sh apply|enrich|convert|validate|import`. +- TFC **Variable Set** variables are merged into the payloads automatically (the `enrich` phase, via the TFC API); skip it with `--no-variable-sets`. +- `./sg-migrate.sh clean` removes local working artifacts (`export/`, Terraform state, tool cache) for a fresh start; add `--all` to also remove config. `clean` always runs locally. +- **Override** a project's target group (to reuse an existing group) in `.sg/workflow-groups.json`: `{"": ""}`. Override groups must already exist (they're not auto-created). +- Output is concise by default (terraform's plan/init noise is hidden; shown on error). Add `-v`/`--verbose` for full output. +- Flags: `-y` skip the import prompt (CI), `--concurrency N` parallel jobs, `--org NAME`, `--no-create-groups` require groups to pre-exist, `--build` rebuild the image, `--native`/`--local` force a local run even when Docker is available. +- Tuning via env: `SG_RETRIES`, `SG_TF_PARALLELISM`, `SG_NATIVE=1`. +- TFC auth: set `TFE_TOKEN` (recommended — a long-lived token avoids re-running `terraform login`); otherwise the `terraform login` credentials file is mounted read-only into the container. SG/TFC tokens are passed as env vars. + +The manual, step-by-step flow below remains supported for fine-grained control and is what each phase runs under the hood (the helper scripts live in `scripts/`). + ## Prerequisites - An organization on [StackGuardian Platform](https://app.stackguardian.io) - Optionally, pre-configure VCS, cloud integrations or private runners to use when importing into StackGuardian Platform. - Terraform -- [sg-cli](https://github.com/StackGuardian/sg-cli/tree/main/shell) +- [sg-cli](https://github.com/StackGuardian/sg-cli) -### Perform terraform login +### Authenticate to Terraform Cloud/Enterprise -Perform `terraform login` to ensure that your local Terraform can interact with your Terraform Cloud/Enterprise account. +Set `TFE_TOKEN` to a long-lived API token (create one under **User Settings → Tokens**, or use a Team/Organization token) — this is the recommended path and avoids session expiry. Alternatively run `terraform login`, which writes `~/.terraform.d/credentials.tfrc.json`. The `tfe` provider, the API state export, and variable-set enrichment all use whichever is present. ### Export the resource definitions and Terraform state @@ -35,9 +63,14 @@ terraform init terraform apply -auto-approve -var-file=terraform.tfvars ``` -A new `export` folder should have been created. The `sg-payload.json` file contains the definition for each workflow that will be created for each Terraform Workspace, and the `states` folder contains the files for the Terraform state for each of your workspaces, if the state export was enabled. +A new `export` folder should have been created, containing: -After completing the export, edit the `sg-payload.json` file to tune each workflow configuration with the following: +- One payload file **per TFC project**, named `sg-payload..json`. Each contains the workflow definitions for the workspaces in that project, and is imported into its own StackGuardian workflow group. +- `migration-summary.md` (and `migration-summary.json`) — a report of what was migrated and what needs manual attention: skipped sensitive variables, Terraform-version fallbacks, renamed workspaces, and workspaces whose state could not be exported. **Read this before importing.** +- The `states` folder with the Terraform state for each workspace, if state export was enabled. +- `state-export-failures.log`, if any workspace's state could not be pulled. + +After completing the export, tune each `sg-payload..json` file with the fields below. For values that differ per workspace (cloud integration, VCS auth, approvers, Terraform version), prefer setting `workspaceOverrides` in `terraform.tfvars` and re-running `terraform apply` instead of editing the JSON by hand — see `terraform.tfvars.example`. ### Use the example_payload.jsonc file as a reference and edit the schema of the `sg-payload.json` @@ -82,19 +115,29 @@ After completing the export, edit the `sg-payload.json` file to tune each workfl ### Convert HCL variables to JSON -HCL variables from Terraform Cloud appear as strings in `sg-payload.json` and need to be converted to JSON before importing. +HCL variables from Terraform Cloud appear as strings in the payload files and need to be converted to JSON before importing. + +Run the script from the repo root, once per payload file. It rewrites the file in place — converting the HCL-string variable values under `VCSConfig.iacInputData.data` into JSON — so none of the following steps need any change. The script downloads `jq` and `hcl2json` at runtime. + +```shell +for f in export/sg-payload.*.json; do ./scripts/convert_hcl_to_json.sh "$f"; done +``` + +### Validate the payloads (recommended) -Run the script from the repo root, passing the exported payload. It rewrites the file in place — converting the HCL-string variable values under `VCSConfig.iacInputData.data` into JSON — so none of the following steps need any change. The script downloads `jq` and `hcl2json` at runtime. +Validate each payload against the StackGuardian workflow schema before importing, to catch malformed payloads before a bulk API run. The schema (`schema/sg-payload.schema.json`) is derived from the SG OpenAPI spec; the script downloads `yajsv` at runtime. ```shell -./convert_hcl_to_json.sh export/sg-payload.json +./scripts/validate_payload.sh export/sg-payload.*.json ``` ### Bulk import workflows to StackGuardian Platform -- Fetch [sg-cli](https://github.com/StackGuardian/sg-cli.git) and set it up locally (documentation present in repo) -- Run the following commands and pass the `sg-payload.json` as payload (represented below) -- `--workflow-group` is required even though `wfgrpName` is set in the payload. Pass the workflow group ID (e.g. `prj-ThpsFFz59kqFaVr4`). +> The orchestrated flow above creates the `tfc-` groups for you. If you run sg-cli by hand, the target workflow group must already exist — create it in the SG UI/API first, or just use `./sg-migrate.sh import`. + +- Fetch the [sg-cli](https://github.com/StackGuardian/sg-cli) Go binary for your platform (assets: `sg-cli__.tar.gz`). +- Import **one payload file at a time**, each into its own workflow group. There is one file per TFC project (`sg-payload..json`), and the payload's group name is `tfc-`. +- `--workflow-group` takes the group name (e.g. `tfc-networking`); the group must exist. - Get your SG API Key here: - Login to Stackguardian. - Go to profile at the bottom left. Click on the email or the username. @@ -104,13 +147,20 @@ Run the script from the repo root, passing the exported payload. It rewrites the cd ../../export export SG_API_TOKEN= -wget -q "$(wget -qO- "https://api.github.com/repos/stackguardian/sg-cli/releases/latest" | jq -r '.tarball_url')" -O sg-cli.tar.gz && tar -xf sg-cli.tar.gz && rm -f sg-cli.tar.gz && /bin/cp -rf StackGuardian-sg-cli*/shell/sg-cli . && rm -rfd StackGuardian-sg-cli* +OS=$(uname -s); ARCH=$(uname -m); case "$ARCH" in x86_64|amd64) ARCH=x86_64;; arm64|aarch64) ARCH=arm64;; esac +curl -fsSL "https://github.com/StackGuardian/sg-cli/releases/latest/download/sg-cli_${OS}_${ARCH}.tar.gz" | tar -xz sg-cli -./sg-cli workflow create --bulk --workflow-group "" --org "" -- sg-payload.json +# Run once per project file (group tfc- must already exist): +./sg-cli workflow create --bulk --workflow-group "tfc-" --org "" sg-payload..json ``` -if you want to update a workflow with different details, please re-run the sg-cli command with the modified sg-payload.json and your workflow will be updated with the new details, as long as the ResourceName (Workflow name) remains the same. +To update workflows with different details, re-run the sg-cli command with the modified payload file; workflows are updated as long as the `ResourceName` (workflow name) stays the same. Add `--dry-run` to preview a payload without applying. -```shell -./sg-cli workflow create --bulk --workflow-group "" --org "" -- sg-payload.json -``` +## Notes and limitations + +- **Workflow groups.** Each TFC project imports into an SG workflow group `tfc-`, created via the API if missing (disable with `--no-create-groups`). Override the target group per project in `.sg/workflow-groups.json`; override groups must already exist. +- **Variable Sets are migrated** (the `enrich` phase) — global, project-, and workspace-scoped sets are resolved per workspace with TFC precedence (priority sets override workspace vars; otherwise workspace vars win). **Sensitive** set variables can't be read from the API, so they're skipped and reported — recreate them as StackGuardian secrets. +- **Sensitive variables are skipped.** TFC never returns sensitive values via the API, so they are omitted from the payload and listed in `migration-summary.md`. Recreate them as StackGuardian secrets. +- **Terraform version fallback.** Workspaces set to `latest` or a version constraint (or running an engine SG can't map) use `SGDefaultTerraformVersion`. Override per workspace via `workspaceOverrides`. +- **State export is idempotent.** Re-running `terraform apply` only pulls state for workspaces not yet exported. Set `forceStateRefresh = true` to re-pull everything. Workspaces not using `remote` execution may export incomplete state — see the summary. +- **Workflow naming.** `ResourceName` currently mirrors the TFC workspace name. Confirm it satisfies StackGuardian's naming rules before import. From 60d4813eb9113b1641b34d0b5568baf18e7b68e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adis=20Halilovi=C4=87?= Date: Mon, 7 Sep 2026 07:57:06 +0200 Subject: [PATCH 05/71] Reformat scripts and example payload --- scripts/convert_hcl_to_json.sh | 5 ++- scripts/enrich_variable_sets.sh | 18 ++++++---- sg-migrate.sh | 6 ++-- transformer/terraform-cloud/data.tf | 2 +- .../terraform-cloud/example_payload.jsonc | 36 +++++++++++++++---- 5 files changed, 48 insertions(+), 19 deletions(-) diff --git a/scripts/convert_hcl_to_json.sh b/scripts/convert_hcl_to_json.sh index 0446805..cb778f5 100755 --- a/scripts/convert_hcl_to_json.sh +++ b/scripts/convert_hcl_to_json.sh @@ -6,7 +6,10 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" source "$SCRIPT_DIR/tools.sh" # Detail lines are shown only in verbose mode; warnings always show. -log() { [ "${SG_VERBOSE:-0}" = "1" ] || return 0; printf '%s[convert]%s %s\n' "$C_CYAN" "$C_RESET" "$*" >&2; } +log() { + [ "${SG_VERBOSE:-0}" = "1" ] || return 0 + printf '%s[convert]%s %s\n' "$C_CYAN" "$C_RESET" "$*" >&2 +} INPUT_FILE_JSON="${1:-}" if [ -z "$INPUT_FILE_JSON" ]; then diff --git a/scripts/enrich_variable_sets.sh b/scripts/enrich_variable_sets.sh index 340548f..d7bbc6b 100755 --- a/scripts/enrich_variable_sets.sh +++ b/scripts/enrich_variable_sets.sh @@ -25,7 +25,10 @@ fi HOST="${SG_TFC_HOSTNAME:-app.terraform.io}" API="https://$HOST/api/v2" -command -v curl >/dev/null 2>&1 || { sg_err "curl is required for variable-set enrichment"; exit 1; } +command -v curl >/dev/null 2>&1 || { + sg_err "curl is required for variable-set enrichment" + exit 1 +} JQ_BIN="$(sg_resolve jq sg_ensure_jq)" # TFC token (same sources as state export): credentials file or TFE_TOKEN. @@ -48,7 +51,8 @@ fetch_all() { : >"$WORK/acc.ndjson" while :; do if ! curl -fsS "${AUTH[@]}" "$API/$path?page%5Bsize%5D=100&page%5Bnumber%5D=$page" >"$WORK/page.json"; then - sg_err "TFC API request failed: $path"; return 1 + sg_err "TFC API request failed: $path" + return 1 fi "$JQ_BIN" -c '.data[]?' "$WORK/page.json" >>"$WORK/acc.ndjson" next="$("$JQ_BIN" -r '.meta.pagination."next-page" // empty' "$WORK/page.json" 2>/dev/null || true)" @@ -61,17 +65,17 @@ fetch_all() { sg_log "fetching workspaces and variable sets from $HOST (org: $ORG)..." # name -> {id, project} -fetch_all "organizations/$ORG/workspaces" \ - | "$JQ_BIN" '[.[] | {name: .attributes.name, id: .id, project: (.relationships.project.data.id // "")}]' \ - >"$WORK/workspaces.json" || exit 1 +fetch_all "organizations/$ORG/workspaces" | + "$JQ_BIN" '[.[] | {name: .attributes.name, id: .id, project: (.relationships.project.data.id // "")}]' \ + >"$WORK/workspaces.json" || exit 1 # Each set with its scope + variables. : >"$WORK/sets.ndjson" sets_raw="$(fetch_all "organizations/$ORG/varsets")" || exit 1 echo "$sets_raw" | "$JQ_BIN" -c '.[]' | while IFS= read -r s; do sid="$(echo "$s" | "$JQ_BIN" -r '.id')" - vars="$(curl -fsS "${AUTH[@]}" "$API/varsets/$sid/relationships/vars" 2>/dev/null \ - | "$JQ_BIN" -c '[.data[]? | {key: .attributes.key, value: (.attributes.value // ""), category: .attributes.category, sensitive: (.attributes.sensitive // false), hcl: (.attributes.hcl // false)}]' 2>/dev/null || echo '[]')" + vars="$(curl -fsS "${AUTH[@]}" "$API/varsets/$sid/relationships/vars" 2>/dev/null | + "$JQ_BIN" -c '[.data[]? | {key: .attributes.key, value: (.attributes.value // ""), category: .attributes.category, sensitive: (.attributes.sensitive // false), hcl: (.attributes.hcl // false)}]' 2>/dev/null || echo '[]')" echo "$s" | "$JQ_BIN" -c --argjson vars "$vars" '{ name: .attributes.name, global: (.attributes.global // false), diff --git a/sg-migrate.sh b/sg-migrate.sh index d39287d..c295768 100755 --- a/sg-migrate.sh +++ b/sg-migrate.sh @@ -21,9 +21,9 @@ BUILD=0 ARGS=() for a in "$@"; do case "$a" in - --native | --local) NATIVE=1 ;; - --build) BUILD=1 ;; - *) ARGS+=("$a") ;; + --native | --local) NATIVE=1 ;; + --build) BUILD=1 ;; + *) ARGS+=("$a") ;; esac done diff --git a/transformer/terraform-cloud/data.tf b/transformer/terraform-cloud/data.tf index be241df..5925934 100644 --- a/transformer/terraform-cloud/data.tf +++ b/transformer/terraform-cloud/data.tf @@ -20,4 +20,4 @@ data "tfe_variables" "data" { data "tfe_projects" "data" { organization = var.tfOrg -} \ No newline at end of file +} diff --git a/transformer/terraform-cloud/example_payload.jsonc b/transformer/terraform-cloud/example_payload.jsonc index cdc11b0..60a67d9 100644 --- a/transformer/terraform-cloud/example_payload.jsonc +++ b/transformer/terraform-cloud/example_payload.jsonc @@ -26,10 +26,15 @@ "ERRORED": [] // list of emails to notify } }, - "wfChaining": { "COMPLETED": [], "ERRORED": [] } + "wfChaining": { + "COMPLETED": [], + "ERRORED": [] + } }, "ResourceName": "", // workspace name - "RunnerConstraints": { "type": "" }, // type should be "shared" or "private" i.e. "RunnerConstraints": { "type": "private", "names":["runner-group-name"]} + "RunnerConstraints": { + "type": "" + }, // type should be "shared" or "private" i.e. "RunnerConstraints": { "type": "private", "names":["runner-group-name"]} "Tags": [], // workflow tags "TerraformConfig": { "managedTerraformState": true, // managed state from StackGuardian @@ -37,7 +42,10 @@ }, "UserSchedules": [], "VCSConfig": { - "iacInputData": { "data": {}, "schemaType": "RAW_JSON" }, // data key consists of key value pairs { "env1" : "secret} as terraform environment variables + "iacInputData": { + "data": {}, + "schemaType": "RAW_JSON" + }, // data key consists of key value pairs { "env1" : "secret} as terraform environment variables "iacVCSConfig": { "customSource": { "config": { @@ -67,10 +75,24 @@ "gh_check": true, "gl_pipeline": true, "post_comments": true, - "push": { "createWfRun": { "enabled": true } }, // run on push to tracked branch - "pull_request_opened": { "createWfRun": { "enabled": true } }, // TFC speculative plan - "pull_request_modified": { "createWfRun": { "enabled": true } }, + "push": { + "createWfRun": { + "enabled": true + } + }, // run on push to tracked branch + "pull_request_opened": { + "createWfRun": { + "enabled": true + } + }, // TFC speculative plan + "pull_request_modified": { + "createWfRun": { + "enabled": true + } + }, "file_triggers_enabled": true, // from TFC file_triggers_enabled - "file_trigger_patterns": ["path/to/workdir/*"] // TFC trigger_patterns / trigger_prefixes / working_directory + "file_trigger_patterns": [ + "path/to/workdir/*" + ] // TFC trigger_patterns / trigger_prefixes / working_directory } } From b42113d9c9d34a6a016924449fd7847107a77818 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adis=20Halilovi=C4=87?= Date: Mon, 7 Sep 2026 07:58:34 +0200 Subject: [PATCH 06/71] fix: fall back to default terraform version above SG ceiling on import sg-cli exits 0 even when individual workflows fail, so the importer now parses its output. Workflows rejected because their pinned Terraform version is above SG's managed ceiling (1.5.7, last MPL/FOSS release) are re-imported with SGDefaultTerraformVersion, the payload is patched in place, and each case is logged to export/terraform-version-fallbacks.log with a printed notice. Any other per-workflow failure now fails the run. Also: sg_retry captured the wrong exit status (always 0), definitive HTTP 4xx responses are no longer retried, the trigger pass skips workflows that were never created, and failure output no longer echoes the API token. --- README.md | 3 +- scripts/migrate.sh | 337 ++++++++++++++---- scripts/tools.sh | 114 ++++-- .../terraform-cloud/terraform.tfvars.example | 20 +- transformer/terraform-cloud/variables.tf | 4 +- 5 files changed, 360 insertions(+), 118 deletions(-) diff --git a/README.md b/README.md index 0b884fe..63819d2 100644 --- a/README.md +++ b/README.md @@ -67,6 +67,7 @@ A new `export` folder should have been created, containing: - One payload file **per TFC project**, named `sg-payload..json`. Each contains the workflow definitions for the workspaces in that project, and is imported into its own StackGuardian workflow group. - `migration-summary.md` (and `migration-summary.json`) — a report of what was migrated and what needs manual attention: skipped sensitive variables, Terraform-version fallbacks, renamed workspaces, and workspaces whose state could not be exported. **Read this before importing.** +- `terraform-version-fallbacks.log` (written by the `import` phase) — workflows that were created with `SGDefaultTerraformVersion` because their pinned version is above StackGuardian's managed ceiling; see _Notes and limitations_. - The `states` folder with the Terraform state for each workspace, if state export was enabled. - `state-export-failures.log`, if any workspace's state could not be pulled. @@ -161,6 +162,6 @@ To update workflows with different details, re-run the sg-cli command with the m - **Workflow groups.** Each TFC project imports into an SG workflow group `tfc-`, created via the API if missing (disable with `--no-create-groups`). Override the target group per project in `.sg/workflow-groups.json`; override groups must already exist. - **Variable Sets are migrated** (the `enrich` phase) — global, project-, and workspace-scoped sets are resolved per workspace with TFC precedence (priority sets override workspace vars; otherwise workspace vars win). **Sensitive** set variables can't be read from the API, so they're skipped and reported — recreate them as StackGuardian secrets. - **Sensitive variables are skipped.** TFC never returns sensitive values via the API, so they are omitted from the payload and listed in `migration-summary.md`. Recreate them as StackGuardian secrets. -- **Terraform version fallback.** Workspaces set to `latest` or a version constraint (or running an engine SG can't map) use `SGDefaultTerraformVersion`. Override per workspace via `workspaceOverrides`. +- **Terraform version fallback (FOSS ceiling).** Workspaces set to `latest` or a version constraint use `SGDefaultTerraformVersion` at export time. Pinned versions are carried over as-is and tried first at import, so a custom runtime image or private runner that ships that binary keeps working. StackGuardian's _managed_ runtimes only go up to **1.5.7**, the last MPL-licensed (FOSS) Terraform release; newer versions are BSL-licensed and are not bundled. When the API rejects a workflow for that reason, the importer **automatically re-imports it with `SGDefaultTerraformVersion`**, patches the payload file to match, prints a notice, and records each case in `export/terraform-version-fallbacks.log`. Those workflows run a different Terraform than they did in TFC, so check compatibility before the first run. To keep a newer version, set `workspaceOverrides[].terraformVersion` to a binary path mounted from a private runner (or use a custom runtime container template) and re-import. - **State export is idempotent.** Re-running `terraform apply` only pulls state for workspaces not yet exported. Set `forceStateRefresh = true` to re-pull everything. Workspaces not using `remote` execution may export incomplete state — see the summary. - **Workflow naming.** `ResourceName` currently mirrors the TFC workspace name. Confirm it satisfies StackGuardian's naming rules before import. diff --git a/scripts/migrate.sh b/scripts/migrate.sh index 4adc693..57bd964 100755 --- a/scripts/migrate.sh +++ b/scripts/migrate.sh @@ -72,7 +72,10 @@ Environment: EOF } -die() { sg_err "$*"; exit 1; } +die() { + sg_err "$*" + exit 1 +} throttle() { while [ "$(jobs -rp | wc -l | tr -d ' ')" -ge "$1" ]; do sleep 0.2; done; } @@ -81,12 +84,16 @@ throttle() { while [ "$(jobs -rp | wc -l | tr -d ' ')" -ge "$1" ]; do sleep 0.2; # a non-TTY and don't scatter spinner output across the terminal), then flushed # as a clean labeled block in submission order. Returns non-zero if any failed. run_parallel() { - local fn="$1" max="$2"; shift 2 + local fn="$1" max="$2" + shift 2 local statusdir i=0 rc=0 item statusdir="$(mktemp -d)" for item in "$@"; do throttle "$max" - ( "$fn" "$item" >"$statusdir/$i.out" 2>&1; echo "$?" >"$statusdir/$i.rc" ) & + ( + "$fn" "$item" >"$statusdir/$i.out" 2>&1 + echo "$?" >"$statusdir/$i.rc" + ) & i=$((i + 1)) done wait @@ -108,7 +115,12 @@ payload_files() { shopt -u nullglob } -seg_of() { local b; b="$(basename "$1")"; b="${b#sg-payload.}"; echo "${b%.json}"; } +seg_of() { + local b + b="$(basename "$1")" + b="${b#sg-payload.}" + echo "${b%.json}" +} cmd_init() { sg_step "Phase: init" @@ -123,6 +135,7 @@ cmd_init() { sg_success "init complete" } + # --- StackGuardian API helpers (workflow groups) --------------------------- # group_for -> the SG workflow group for a project segment: an entry @@ -170,23 +183,60 @@ do_set_triggers() { n="$("$JQ_BIN" 'length' "$f")" for ((i = 0; i < n; i++)); do if [ "$("$JQ_BIN" -r --argjson i "$i" '(.[$i].VCSTriggers // null) != null' "$f")" != "true" ]; then - skip=$((skip + 1)); continue + skip=$((skip + 1)) + continue fi wf="$("$JQ_BIN" -r --argjson i "$i" '.[$i].ResourceName' "$f")" + # A workflow that failed to import has nothing to attach triggers to. + if [ "$(sg_http_code GET "$SG_BASE_URL/api/v1/orgs/$ORG/wfgrps/$grp/wfs/$wf/")" != "200" ]; then + sg_warn " $grp/$wf does not exist in SG (import failed?) — skipping triggers" + rc=1 + continue + fi body="$("$JQ_BIN" -c --argjson i "$i" '{VCSConfig: .[$i].VCSConfig, VCSTriggers: .[$i].VCSTriggers}' "$f")" - if sg_retry "$RETRIES" "$RETRY_BASE" -- \ - curl -fsS -X POST \ - -H "Authorization: apikey $SG_API_TOKEN" -H "Content-Type: application/json" \ - -d "$body" "$(wf_triggers_endpoint "$grp" "$wf")" >/dev/null; then + if SG_NO_RETRY_RC=22 sg_retry "$RETRIES" "$RETRY_BASE" -- \ + sg_api_post "$(wf_triggers_endpoint "$grp" "$wf")" "$body"; then set=$((set + 1)) else - sg_warn " vcs triggers failed: $grp/$wf"; rc=1 + sg_warn " vcs triggers failed: $grp/$wf" + rc=1 fi done sg_log "$(basename "$f"): set triggers on $set workflow(s) (skipped $skip without triggers)" return "$rc" } +# sg_http_code — prints the HTTP status (000 on network error). +sg_http_code() { + curl -sS -o /dev/null -w '%{http_code}' -X "$1" -H "Authorization: apikey $SG_API_TOKEN" "$2" 2>/dev/null || echo 000 +} + +# sg_api_post — POST to the SG API. Exit 0 on 2xx, 22 on a +# definitive 4xx (not retryable; response echoed), 1 on 5xx/network (retryable). +sg_api_post() { + local url="$1" body="$2" tmp code + tmp="$(mktemp)" + code="$(curl -sS -o "$tmp" -w '%{http_code}' -X POST \ + -H "Authorization: apikey $SG_API_TOKEN" -H "Content-Type: application/json" \ + -d "$body" "$url" 2>/dev/null || echo 000)" + case "$code" in + 2*) + rm -f "$tmp" + return 0 + ;; + 4*) + sg_err " HTTP $code from ${url#"$SG_BASE_URL"}: $(head -c 400 "$tmp")" + rm -f "$tmp" + return 22 + ;; + *) + sg_warn " HTTP $code from ${url#"$SG_BASE_URL"}" + rm -f "$tmp" + return 1 + ;; + esac +} + # set_triggers_pass — run do_set_triggers over all payload files (assumes # JQ_BIN/SG_API_TOKEN/ORG are already set up by the caller). set_triggers_pass() { @@ -200,7 +250,8 @@ set_triggers_pass() { if run_parallel do_set_triggers "$CONC" "${PF[@]}"; then sg_success "vcs triggers registered" else - sg_err "one or more VCS trigger registrations failed (re-run: $0 triggers)"; return 1 + sg_err "one or more VCS trigger registrations failed (re-run: $0 triggers)" + return 1 fi } @@ -242,22 +293,23 @@ cmd_apply() { jqdir="$(dirname "$(sg_resolve jq sg_ensure_jq)")" if [ "$VERBOSE" -eq 1 ]; then - ( cd "$TRANSFORMER_DIR" && export PATH="$jqdir:$PATH" TF_IN_AUTOMATION=1 \ - && terraform init -input=false \ - && terraform apply -auto-approve -compact-warnings -parallelism="$TF_PARALLELISM" -var-file=terraform.tfvars ) || rc=$? + (cd "$TRANSFORMER_DIR" && export PATH="$jqdir:$PATH" TF_IN_AUTOMATION=1 && + terraform init -input=false && + terraform apply -auto-approve -compact-warnings -parallelism="$TF_PARALLELISM" -var-file=terraform.tfvars) || rc=$? else # Quiet: capture terraform's verbose plan/output; surface only progress, the # final summary, and (on failure) the captured log. tflog="$(mktemp)" sg_log "initializing terraform (providers)..." - ( cd "$TRANSFORMER_DIR" && export PATH="$jqdir:$PATH" TF_IN_AUTOMATION=1 && terraform init -input=false -no-color ) >"$tflog" 2>&1 || rc=$? + (cd "$TRANSFORMER_DIR" && export PATH="$jqdir:$PATH" TF_IN_AUTOMATION=1 && terraform init -input=false -no-color) >"$tflog" 2>&1 || rc=$? if [ "$rc" -eq 0 ]; then sg_log "reading workspaces, generating payloads, exporting state..." - ( cd "$TRANSFORMER_DIR" && export PATH="$jqdir:$PATH" TF_IN_AUTOMATION=1 \ - && terraform apply -auto-approve -compact-warnings -no-color -parallelism="$TF_PARALLELISM" -var-file=terraform.tfvars ) >"$tflog" 2>&1 || rc=$? + (cd "$TRANSFORMER_DIR" && export PATH="$jqdir:$PATH" TF_IN_AUTOMATION=1 && + terraform apply -auto-approve -compact-warnings -no-color -parallelism="$TF_PARALLELISM" -var-file=terraform.tfvars) >"$tflog" 2>&1 || rc=$? fi if [ "$rc" -ne 0 ]; then - sg_err "terraform failed (rc=$rc):"; cat "$tflog" >&2 + sg_err "terraform failed (rc=$rc):" + cat "$tflog" >&2 else grep -E '^(Apply complete|No changes)' "$tflog" | sed 's/^/ /' >&2 || true fi @@ -293,7 +345,8 @@ cmd_convert() { if run_parallel do_convert "$CONC" "${PF[@]}"; then sg_success "converted ${#PF[@]} payload(s)" else - sg_err "conversion failed for one or more payloads"; return 1 + sg_err "conversion failed for one or more payloads" + return 1 fi } @@ -304,17 +357,102 @@ cmd_validate() { if "$SCRIPT_DIR/validate_payload.sh" "${PF[@]}"; then sg_success "all ${#PF[@]} payload(s) valid" else - sg_err "validation failed"; return 1 + sg_err "validation failed" + return 1 fi } +# sgcli_bulk — run the bulk create, teeing output to . +# sg-cli exits 0 even when individual workflows fail, so callers must inspect +# the output ("Failed to create : ..." lines). +sgcli_bulk() { + "$SGCLI_BIN" workflow create --bulk --workflow-group "$1" --org "$ORG" "$2" 2>&1 | tee "$3" + return "${PIPESTATUS[0]}" +} + +# Regex for the API's rejection of a Terraform version above SG's managed +# ceiling. SG bundles managed runtimes only up to the last MPL-licensed (FOSS) +# Terraform release; newer versions are BSL and are not shipped. +TF_CEILING_RE='Failed to create ([^:]+): 400: .*above the highest managed version \(([0-9.]+)\)' + +# names_json — JSON array of the given names (for jq --argjson). +names_json() { printf '%s\n' "$@" | "$JQ_BIN" -R . | "$JQ_BIN" -s .; } + +# do_import — bulk-import one file. Workflows rejected because their +# Terraform version is above the SG ceiling are re-imported with +# SG_DEFAULT_TF_VERSION (the payload file is patched in place so re-runs and +# the trigger pass see what was actually imported); each fallback is appended to +# terraform-version-fallbacks.log. Any other per-workflow failure fails the file. do_import() { - local f="$1" seg grp + local f="$1" seg grp out rc=0 ceiling="" failed=() fb=() name line tmp names seg="$(seg_of "$f")" grp="$(group_for "$seg")" sg_log "importing $(basename "$f") -> $grp" - sg_retry "$RETRIES" "$RETRY_BASE" -- \ - "$SGCLI_BIN" workflow create --bulk --workflow-group "$grp" --org "$ORG" "$f" + out="$(mktemp)" + sg_retry "$RETRIES" "$RETRY_BASE" -- sgcli_bulk "$grp" "$f" "$out" || rc=1 + + while IFS= read -r line; do + if [[ "$line" =~ $TF_CEILING_RE ]]; then + fb+=("${BASH_REMATCH[1]}") + ceiling="${BASH_REMATCH[2]}" + elif [[ "$line" =~ Failed\ to\ create\ ([^:]+): ]]; then + failed+=("${BASH_REMATCH[1]}") + fi + done <"$out" + rm -f "$out" + + if [ "${#fb[@]}" -gt 0 ]; then + sg_warn "${#fb[@]} workflow(s) pinned above SG's managed Terraform ceiling ($ceiling); re-importing with $SG_DEFAULT_TF_VERSION" + names="$(names_json "${fb[@]}")" + tmp="$(mktemp "$EXPORT_DIR/.fallback.$seg.XXXXXX")" + # Patch the affected workflows in the payload and re-import only those. + "$JQ_BIN" --arg v "$SG_DEFAULT_TF_VERSION" --argjson names "$names" \ + 'map(if (.ResourceName as $n | $names | index($n)) != null then .TerraformConfig.terraformVersion = $v else . end)' "$f" >"$tmp.full" && + "$JQ_BIN" --argjson names "$names" \ + 'map(select(.ResourceName as $n | $names | index($n) != null))' "$tmp.full" >"$tmp" || + { + rm -f "$tmp" "$tmp.full" + die "could not patch $(basename "$f") for the Terraform version fallback" + } + out="$(mktemp)" + sg_retry "$RETRIES" "$RETRY_BASE" -- sgcli_bulk "$grp" "$tmp" "$out" || rc=1 + for name in "${fb[@]}"; do + if grep -q "Failed to create $name:" "$out"; then + failed+=("$name") + else + line="$("$JQ_BIN" -r --arg n "$name" '.[] | select(.ResourceName == $n) | .TerraformConfig.terraformVersion' "$f")" + printf '%s/%s: %s -> %s (above SG managed ceiling %s)\n' "$grp" "$name" "$line" "$SG_DEFAULT_TF_VERSION" "$ceiling" >>"$EXPORT_DIR/terraform-version-fallbacks.log" + fi + done + rm -f "$out" + mv -f "$tmp.full" "$f" + rm -f "$tmp" + fi + + if [ "${#failed[@]}" -gt 0 ]; then + sg_err "$(basename "$f"): ${#failed[@]} workflow(s) failed to import: ${failed[*]}" + return 1 + fi + return "$rc" +} + +# Print the customer-facing notice when any workflow fell back to the default +# Terraform version during this import. +tf_fallback_notice() { + local log="$EXPORT_DIR/terraform-version-fallbacks.log" + [ -s "$log" ] || return 0 + sg_warn "Terraform version fallback applied to $(wc -l <"$log" | tr -d ' ') workflow(s):" + sed 's/^/ /' "$log" >&2 + cat >&2 <].terraformVersion to a binary path mounted from a + private runner, or use a custom runtime container template + (wfStepTemplateRevisionId), and re-import. +NOTICE } cmd_import() { @@ -340,23 +478,32 @@ cmd_import() { count="$("$JQ_BIN" 'length' "$f")" override="" [ -f "$MAPPING" ] && override="$("$JQ_BIN" -r --arg k "$seg" '.[$k] // empty' "$MAPPING" 2>/dev/null || true)" - if [ -n "$override" ]; then grp="$override"; is_override=1; else grp="tfc-$seg"; is_override=0; fi + if [ -n "$override" ]; then + grp="$override" + is_override=1 + else + grp="tfc-$seg" + is_override=0 + fi code="$(wfgroup_http_code "$grp")" case "$code" in - 200) status="${C_GREEN}exists${C_RESET}" ;; - 404) - if [ "$is_override" -eq 1 ]; then - status="${C_RED}missing!${C_RESET}"; fail=1 - elif [ "$CREATE_GROUPS" -eq 1 ]; then - status="${C_YELLOW}create${C_RESET}" - case "$to_create" in *" $grp "*) ;; *) to_create="$to_create$grp " ;; esac - else - status="${C_RED}missing!${C_RESET}"; fail=1 - fi ;; - 401 | 403) die "auth failed (HTTP $code) for org '$ORG' — check SG_API_TOKEN" ;; - 000) die "could not reach $SG_BASE_URL" ;; - *) die "unexpected HTTP $code checking group '$grp'" ;; + 200) status="${C_GREEN}exists${C_RESET}" ;; + 404) + if [ "$is_override" -eq 1 ]; then + status="${C_RED}missing!${C_RESET}" + fail=1 + elif [ "$CREATE_GROUPS" -eq 1 ]; then + status="${C_YELLOW}create${C_RESET}" + case "$to_create" in *" $grp "*) ;; *) to_create="$to_create$grp " ;; esac + else + status="${C_RED}missing!${C_RESET}" + fail=1 + fi + ;; + 401 | 403) die "auth failed (HTTP $code) for org '$ORG' — check SG_API_TOKEN" ;; + 000) die "could not reach $SG_BASE_URL" ;; + *) die "unexpected HTTP $code checking group '$grp'" ;; esac printf ' %-34s %-26s %-9s %s\n' "$(basename "$f")" "$grp" "$count" "$status" >&2 done @@ -377,40 +524,72 @@ cmd_import() { done SGCLI_BIN="$(sg_resolve sg-cli sg_ensure_sgcli)" + # Fallback Terraform version for workflows the API rejects as above the + # managed ceiling: SGDefaultTerraformVersion from terraform.tfvars, else 1.5.7. + SG_DEFAULT_TF_VERSION="${SG_DEFAULT_TF_VERSION:-}" + if [ -z "$SG_DEFAULT_TF_VERSION" ] && [ -f "$TFVARS" ]; then + SG_DEFAULT_TF_VERSION="$("$(sg_resolve hcl2json sg_ensure_hcl2json)" "$TFVARS" | "$JQ_BIN" -r '.SGDefaultTerraformVersion // empty')" + fi + SG_DEFAULT_TF_VERSION="${SG_DEFAULT_TF_VERSION:-TERRAFORM-1.5.7}" + rm -f "$EXPORT_DIR/terraform-version-fallbacks.log" + sg_log "importing ${#PF[@]} payload(s), up to $CONC in parallel (retries: $RETRIES)" - if run_parallel do_import "$CONC" "${PF[@]}"; then + local import_rc=0 + run_parallel do_import "$CONC" "${PF[@]}" || import_rc=1 + tf_fallback_notice + if [ "$import_rc" -eq 0 ]; then sg_success "import complete (${#PF[@]} payload(s))" else - sg_err "one or more imports failed"; return 1 + sg_err "one or more workflows failed to import (see above); VCS triggers are still registered for the ones that succeeded" fi # VCS triggers are not accepted by the bulk create API; register them in a # second pass against the dedicated webhooks endpoint (skip with --no-vcs-triggers). if [ "$VCS_TRIGGERS" -eq 1 ]; then - set_triggers_pass + set_triggers_pass || import_rc=1 fi + return "$import_rc" } CMD="" while [ $# -gt 0 ]; do case "$1" in - -y | --yes) ASSUME_YES=1 ;; - --org) ORG="$2"; shift ;; - --org=*) ORG="${1#*=}" ;; - --export-dir) EXPORT_DIR="$2"; shift ;; - --export-dir=*) EXPORT_DIR="${1#*=}" ;; - --mapping) MAPPING="$2"; shift ;; - --mapping=*) MAPPING="${1#*=}" ;; - --concurrency) CONC="$2"; shift ;; - --concurrency=*) CONC="${1#*=}" ;; - --no-create-groups) CREATE_GROUPS=0 ;; - --no-variable-sets) ENRICH_VARSETS=0 ;; - --no-vcs-triggers) VCS_TRIGGERS=0 ;; - -v | --verbose) VERBOSE=1 ;; - --all) PURGE=1 ;; - -h | --help) usage; exit 0 ;; - init | apply | enrich | convert | validate | import | triggers | all | clean) CMD="$1" ;; - *) echo "Unknown argument: $1" >&2; usage; exit 1 ;; + -y | --yes) ASSUME_YES=1 ;; + --org) + ORG="$2" + shift + ;; + --org=*) ORG="${1#*=}" ;; + --export-dir) + EXPORT_DIR="$2" + shift + ;; + --export-dir=*) EXPORT_DIR="${1#*=}" ;; + --mapping) + MAPPING="$2" + shift + ;; + --mapping=*) MAPPING="${1#*=}" ;; + --concurrency) + CONC="$2" + shift + ;; + --concurrency=*) CONC="${1#*=}" ;; + --no-create-groups) CREATE_GROUPS=0 ;; + --no-variable-sets) ENRICH_VARSETS=0 ;; + --no-vcs-triggers) VCS_TRIGGERS=0 ;; + -v | --verbose) VERBOSE=1 ;; + --all) PURGE=1 ;; + -h | --help) + usage + exit 0 + ;; + init | apply | enrich | convert | validate | import | triggers | all | clean) CMD="$1" ;; + *) + echo "Unknown argument: $1" >&2 + usage + exit 1 + ;; esac shift done @@ -418,26 +597,26 @@ CMD="${CMD:-all}" export SG_VERBOSE="$VERBOSE" case "$CMD" in - init) cmd_init ;; - clean) cmd_clean ;; - apply) cmd_apply ;; - enrich) cmd_enrich ;; - convert) cmd_convert ;; - validate) cmd_validate ;; - import) cmd_import ;; - triggers) cmd_triggers ;; - all) - if [ ! -f "$TFVARS" ]; then - cmd_init - die "Edit $(sg_rel "$TFVARS"), then re-run '$0 all'." - fi - # Fail fast on import prerequisites before the (long) apply. - [ -n "${SG_API_TOKEN:-}" ] || die "SG_API_TOKEN is not set (needed for import)." - [ -n "$ORG" ] || die "StackGuardian org not set (use --org or SG_ORG)." - cmd_apply - if [ "$ENRICH_VARSETS" -eq 1 ]; then cmd_enrich; fi - cmd_convert - cmd_validate - cmd_import - ;; +init) cmd_init ;; +clean) cmd_clean ;; +apply) cmd_apply ;; +enrich) cmd_enrich ;; +convert) cmd_convert ;; +validate) cmd_validate ;; +import) cmd_import ;; +triggers) cmd_triggers ;; +all) + if [ ! -f "$TFVARS" ]; then + cmd_init + die "Edit $(sg_rel "$TFVARS"), then re-run '$0 all'." + fi + # Fail fast on import prerequisites before the (long) apply. + [ -n "${SG_API_TOKEN:-}" ] || die "SG_API_TOKEN is not set (needed for import)." + [ -n "$ORG" ] || die "StackGuardian org not set (use --org or SG_ORG)." + cmd_apply + if [ "$ENRICH_VARSETS" -eq 1 ]; then cmd_enrich; fi + cmd_convert + cmd_validate + cmd_import + ;; esac diff --git a/scripts/tools.sh b/scripts/tools.sh index f3b568f..901a9ff 100755 --- a/scripts/tools.sh +++ b/scripts/tools.sh @@ -20,34 +20,48 @@ SG_YAJSV_VERSION="v1.4.1" # Colored logging — disabled when stderr is not a TTY, NO_COLOR is set, or # TERM=dumb, so piped/CI output stays clean. if [ -t 2 ] && [ -z "${NO_COLOR:-}" ] && [ "${TERM:-}" != "dumb" ]; then - C_RESET=$'\033[0m'; C_BOLD=$'\033[1m'; C_DIM=$'\033[2m' - C_RED=$'\033[31m'; C_GREEN=$'\033[32m'; C_YELLOW=$'\033[33m'; C_CYAN=$'\033[36m' + C_RESET=$'\033[0m' + C_BOLD=$'\033[1m' + C_DIM=$'\033[2m' + C_RED=$'\033[31m' + C_GREEN=$'\033[32m' + C_YELLOW=$'\033[33m' + C_CYAN=$'\033[36m' else - C_RESET=""; C_BOLD=""; C_DIM=""; C_RED=""; C_GREEN=""; C_YELLOW=""; C_CYAN="" + C_RESET="" + C_BOLD="" + C_DIM="" + C_RED="" + C_GREEN="" + C_YELLOW="" + C_CYAN="" fi # sg_rel — render a path relative to the repo root for readable logs # (operations still use absolute paths; this is display-only). sg_rel() { case "$1" in - "$SG_REPO_ROOT"/*) printf '%s' "${1#"$SG_REPO_ROOT"/}" ;; - "$SG_REPO_ROOT") printf '.' ;; - *) printf '%s' "$1" ;; + "$SG_REPO_ROOT"/*) printf '%s' "${1#"$SG_REPO_ROOT"/}" ;; + "$SG_REPO_ROOT") printf '.' ;; + *) printf '%s' "$1" ;; esac } -sg_log() { printf '%s[sg-migrate]%s %s\n' "$C_CYAN" "$C_RESET" "$*" >&2; } -sg_warn() { printf '%s[sg-migrate] WARN%s %s\n' "$C_YELLOW" "$C_RESET" "$*" >&2; } -sg_err() { printf '%s[sg-migrate] ERROR%s %s\n' "$C_RED$C_BOLD" "$C_RESET" "$*" >&2; } -sg_success() { printf '%s[sg-migrate] ✓%s %s\n' "$C_GREEN$C_BOLD" "$C_RESET" "$*" >&2; } -sg_step() { printf '\n%s==> %s%s\n' "$C_CYAN$C_BOLD" "$*" "$C_RESET" >&2; } -sg_dim() { printf '%s %s%s\n' "$C_DIM" "$*" "$C_RESET" >&2; } +sg_log() { printf '%s[sg-migrate]%s %s\n' "$C_CYAN" "$C_RESET" "$*" >&2; } +sg_warn() { printf '%s[sg-migrate] WARN%s %s\n' "$C_YELLOW" "$C_RESET" "$*" >&2; } +sg_err() { printf '%s[sg-migrate] ERROR%s %s\n' "$C_RED$C_BOLD" "$C_RESET" "$*" >&2; } +sg_success() { printf '%s[sg-migrate] ✓%s %s\n' "$C_GREEN$C_BOLD" "$C_RESET" "$*" >&2; } +sg_step() { printf '\n%s==> %s%s\n' "$C_CYAN$C_BOLD" "$*" "$C_RESET" >&2; } +sg_dim() { printf '%s %s%s\n' "$C_DIM" "$*" "$C_RESET" >&2; } sg_arch() { case "$(uname -m)" in - x86_64 | amd64) echo "amd64" ;; - aarch64 | arm64) echo "arm64" ;; - *) echo "Unsupported architecture: $(uname -m)" >&2; return 1 ;; + x86_64 | amd64) echo "amd64" ;; + aarch64 | arm64) echo "arm64" ;; + *) + echo "Unsupported architecture: $(uname -m)" >&2 + return 1 + ;; esac } @@ -55,15 +69,24 @@ sg_arch() { # Runs the command, retrying with exponential backoff (capped at 60s, with a # little jitter) until it succeeds or attempts are exhausted. Returns the # command's last exit code. Used to ride out transient API failures/rate limits. +# Exit codes listed in SG_NO_RETRY_RC (space-separated) are returned immediately +# (e.g. a definitive HTTP 4xx). Only the command name is echoed on failure so +# that tokens passed as arguments never land in the log. sg_retry() { - local max="$1" base="$2"; shift 2 + local max="$1" base="$2" + shift 2 [ "${1:-}" = "--" ] && shift local attempt=1 delay="$base" rc=0 while :; do - if "$@"; then return 0; fi + "$@" && return 0 rc=$? + case " ${SG_NO_RETRY_RC:-} " in *" $rc "*) + sg_err "$(basename "$1") failed (exit ${rc}, not retryable)" + return "$rc" + ;; + esac if [ "$attempt" -ge "$max" ]; then - sg_err "failed after ${attempt} attempt(s) (exit ${rc}): $*" + sg_err "$(basename "$1") failed after ${attempt} attempt(s) (exit ${rc})" return "$rc" fi local jitter=$((RANDOM % (base + 1))) @@ -80,8 +103,14 @@ sg_retry() { # Prints the path to use on stdout. sg_resolve() { local name="$1" ensure="$2" - if command -v "$name" >/dev/null 2>&1; then command -v "$name"; return 0; fi - if [ -n "$ensure" ]; then "$ensure"; return $?; fi + if command -v "$name" >/dev/null 2>&1; then + command -v "$name" + return 0 + fi + if [ -n "$ensure" ]; then + "$ensure" + return $? + fi echo "Required tool '$name' not found on PATH" >&2 return 1 } @@ -89,7 +118,10 @@ sg_resolve() { # sg_download sg_download() { local url="$1" dest="$2" - command -v curl >/dev/null 2>&1 || { echo "curl is required but not found" >&2; return 1; } + command -v curl >/dev/null 2>&1 || { + echo "curl is required but not found" >&2 + return 1 + } mkdir -p "$(dirname "$dest")" if ! curl -fsSL -o "$dest" "$url"; then echo "Failed to download: $url" >&2 @@ -101,7 +133,11 @@ sg_ensure_jq() { local bin="$SG_CACHE_BIN/jq" arch os if [ ! -x "$bin" ]; then arch=$(sg_arch) || return 1 - case "$(uname -s)" in Darwin) os="macos" ;; Linux) os="linux" ;; *) echo "Unsupported OS: $(uname -s)" >&2; return 1 ;; esac + case "$(uname -s)" in Darwin) os="macos" ;; Linux) os="linux" ;; *) + echo "Unsupported OS: $(uname -s)" >&2 + return 1 + ;; + esac sg_log "caching jq ${SG_JQ_VERSION}..." sg_download "https://github.com/jqlang/jq/releases/download/${SG_JQ_VERSION}/jq-${os}-${arch}" "$bin" || return 1 chmod +x "$bin" @@ -113,7 +149,11 @@ sg_ensure_hcl2json() { local bin="$SG_CACHE_BIN/hcl2json" arch os if [ ! -x "$bin" ]; then arch=$(sg_arch) || return 1 - case "$(uname -s)" in Darwin) os="darwin" ;; Linux) os="linux" ;; *) echo "Unsupported OS: $(uname -s)" >&2; return 1 ;; esac + case "$(uname -s)" in Darwin) os="darwin" ;; Linux) os="linux" ;; *) + echo "Unsupported OS: $(uname -s)" >&2 + return 1 + ;; + esac sg_log "caching hcl2json ${SG_HCL2JSON_VERSION}..." sg_download "https://github.com/tmccombs/hcl2json/releases/download/${SG_HCL2JSON_VERSION}/hcl2json_${os}_${arch}" "$bin" || return 1 chmod +x "$bin" @@ -125,7 +165,11 @@ sg_ensure_yajsv() { local bin="$SG_CACHE_BIN/yajsv" arch os if [ ! -x "$bin" ]; then arch=$(sg_arch) || return 1 - case "$(uname -s)" in Darwin) os="darwin" ;; Linux) os="linux" ;; *) echo "Unsupported OS: $(uname -s)" >&2; return 1 ;; esac + case "$(uname -s)" in Darwin) os="darwin" ;; Linux) os="linux" ;; *) + echo "Unsupported OS: $(uname -s)" >&2 + return 1 + ;; + esac if [ "$os" = "linux" ] && [ "$arch" = "arm64" ]; then echo "No yajsv prebuilt binary for linux/arm64; install yajsv manually." >&2 return 1 @@ -143,16 +187,30 @@ sg_ensure_yajsv() { sg_ensure_sgcli() { local bin="$SG_CACHE_BIN/sg-cli" os arch tmp realcli if [ ! -x "$bin" ]; then - case "$(uname -s)" in Darwin) os="Darwin" ;; Linux) os="Linux" ;; *) echo "Unsupported OS: $(uname -s)" >&2; return 1 ;; esac - case "$(uname -m)" in x86_64 | amd64) arch="x86_64" ;; aarch64 | arm64) arch="arm64" ;; *) echo "Unsupported architecture: $(uname -m)" >&2; return 1 ;; esac + case "$(uname -s)" in Darwin) os="Darwin" ;; Linux) os="Linux" ;; *) + echo "Unsupported OS: $(uname -s)" >&2 + return 1 + ;; + esac + case "$(uname -m)" in x86_64 | amd64) arch="x86_64" ;; aarch64 | arm64) arch="arm64" ;; *) + echo "Unsupported architecture: $(uname -m)" >&2 + return 1 + ;; + esac sg_log "caching sg-cli (latest release, ${os}/${arch})..." tmp=$(mktemp -d) if ! curl -fsSL "https://github.com/StackGuardian/sg-cli/releases/latest/download/sg-cli_${os}_${arch}.tar.gz" -o "$tmp/sg-cli.tar.gz"; then - echo "Failed to download sg-cli" >&2; rm -rf "$tmp"; return 1 + echo "Failed to download sg-cli" >&2 + rm -rf "$tmp" + return 1 fi tar -xzf "$tmp/sg-cli.tar.gz" -C "$tmp" realcli=$(find "$tmp" -maxdepth 2 -type f -name sg-cli | head -1) - [ -n "$realcli" ] || { echo "sg-cli binary not found in release archive" >&2; rm -rf "$tmp"; return 1; } + [ -n "$realcli" ] || { + echo "sg-cli binary not found in release archive" >&2 + rm -rf "$tmp" + return 1 + } mkdir -p "$SG_CACHE_BIN" cp "$realcli" "$bin" chmod +x "$bin" diff --git a/transformer/terraform-cloud/terraform.tfvars.example b/transformer/terraform-cloud/terraform.tfvars.example index 9454029..6b0024a 100644 --- a/transformer/terraform-cloud/terraform.tfvars.example +++ b/transformer/terraform-cloud/terraform.tfvars.example @@ -27,20 +27,24 @@ SGDefaultVCSAuthIntegrationID = "/integrations/github_com" # Integration to use to authenticate against your cloud provider SGDefaultDeploymentPlatformConfig = [ - { - "kind" : "AWS_RBAC", - "config" : { - "integrationId" : "/integrations/aws-dev-account", - "profileName" : "default" - } + { + "kind" : "AWS_RBAC", + "config" : { + "integrationId" : "/integrations/aws-dev-account", + "profileName" : "default" } - ] + } +] # Choose from: GITHUB_COM, BITBUCKET_ORG, GITLAB_COM, AZURE_DEVOPS, GIT_OTHER SGDefaultSourceConfigDestKind = "GITHUB_COM" # SG Terraform version used when a workspace's terraform_version is not a pinned -# semver (e.g. "latest" or a constraint), or runs an engine SG cannot map. +# semver (e.g. "latest" or a constraint). Pinned versions are carried over as-is; +# if the SG API rejects one as above the managed ceiling (1.5.7 - the last MPL/ +# FOSS Terraform release; newer versions are BSL and not bundled), the importer +# re-creates that workflow with this version and logs it in +# export/terraform-version-fallbacks.log. SGDefaultTerraformVersion = "TERRAFORM-1.5.7" # Pre-configure VCS triggers on each VCS-backed workflow, remapped from the diff --git a/transformer/terraform-cloud/variables.tf b/transformer/terraform-cloud/variables.tf index 52c6df6..3c3b960 100644 --- a/transformer/terraform-cloud/variables.tf +++ b/transformer/terraform-cloud/variables.tf @@ -79,7 +79,7 @@ variable "SGDefaultSourceConfigDestKind" { variable "SGDefaultTerraformVersion" { default = "TERRAFORM-1.5.7" - description = "SG Terraform version used when a workspace's terraform_version is not a pinned semver (e.g. 'latest' or a version constraint), or the workspace runs an engine SG cannot map. Use the SG-formatted value, e.g. TERRAFORM-1.5.7." + description = "SG Terraform version used when a workspace's terraform_version is not a pinned semver (e.g. 'latest' or a version constraint). Also used by the importer when the SG API rejects a pinned version as above the managed ceiling (1.5.7, the last MPL/FOSS release; newer versions are BSL and not bundled). Use the SG-formatted value, e.g. TERRAFORM-1.5.7." type = string } @@ -109,4 +109,4 @@ variable "workspaceOverrides" { extraEnvironmentVariables = optional(list(any), []) VCSTriggers = optional(any) })) -} \ No newline at end of file +} From 5de752347f3ca27d24bd99aeee1e9591b9b9fb1f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adis=20Halilovi=C4=87?= Date: Mon, 7 Sep 2026 07:59:13 +0200 Subject: [PATCH 07/71] feat: shell completion subcommand 'sg-migrate.sh completion bash|zsh' prints a completion script for the current shell session; 'init' prints the source line for the user's shell. Runs natively like 'clean'. --- README.md | 1 + scripts/migrate.sh | 98 ++++++++++++++++++++++++++++++++++++++++++++++ sg-migrate.sh | 4 +- 3 files changed, 101 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 63819d2..4c0d291 100644 --- a/README.md +++ b/README.md @@ -36,6 +36,7 @@ That's it — no workflow-group mapping to fill in. Each TFC project is imported - Output is concise by default (terraform's plan/init noise is hidden; shown on error). Add `-v`/`--verbose` for full output. - Flags: `-y` skip the import prompt (CI), `--concurrency N` parallel jobs, `--org NAME`, `--no-create-groups` require groups to pre-exist, `--build` rebuild the image, `--native`/`--local` force a local run even when Docker is available. - Tuning via env: `SG_RETRIES`, `SG_TF_PARALLELISM`, `SG_NATIVE=1`. +- Tab completion for the current shell session: `source <(./sg-migrate.sh completion zsh)` (or `bash`). `init` prints this line for your shell. - TFC auth: set `TFE_TOKEN` (recommended — a long-lived token avoids re-running `terraform login`); otherwise the `terraform login` credentials file is mounted read-only into the container. SG/TFC tokens are passed as env vars. The manual, step-by-step flow below remains supported for fine-grained control and is what each phase runs under the hood (the helper scripts live in `scripts/`). diff --git a/scripts/migrate.sh b/scripts/migrate.sh index 57bd964..c765e63 100755 --- a/scripts/migrate.sh +++ b/scripts/migrate.sh @@ -46,6 +46,8 @@ Commands: all apply -> enrich -> convert -> validate -> import (default) clean Remove local working artifacts for a fresh start (export/, TF state, tool cache). Add --all to also remove config (terraform.tfvars, mapping). + completion Print a shell completion script for the current session: + \`source <($0 completion zsh)\` (or bash) Each TFC project maps to an SG workflow group named tfc-, created via the API if missing. Override a project's target group in .sg/workflow-groups.json @@ -132,9 +134,18 @@ cmd_init() { sg_log "$(sg_rel "$TFVARS") already exists" fi sg_log "workflow groups are created automatically as tfc-; no mapping needed" + completion_hint sg_success "init complete" } +# completion_hint — tell the user how to enable tab completion for their shell. +# A child process cannot register completions in the parent shell, so this only +# prints the one-liner (nothing is written to the user's rc files). +completion_hint() { + local sh + case "$(basename "${SHELL:-}")" in bash) sh=bash ;; *) sh=zsh ;; esac + sg_log "tab completion for this shell session: source <(./sg-migrate.sh completion $sh)" +} # --- StackGuardian API helpers (workflow groups) --------------------------- @@ -551,6 +562,89 @@ cmd_import() { return "$import_rc" } +# Single source of truth for shell completion (keep in sync with the parser below +# and the host-only flags in sg-migrate.sh). +SG_COMMANDS="init apply enrich convert validate import triggers all clean completion" +SG_OPTIONS="--org --export-dir --mapping --concurrency --no-create-groups --no-variable-sets --no-vcs-triggers --all -v --verbose -y --yes -h --help --native --local --build" + +# cmd_completion — print a completion script for sg-migrate.sh / +# migrate.sh to stdout. Both shells fall back to the basename when the command +# is invoked by path, so ./sg-migrate.sh completes too. +cmd_completion() { + local shell="${1:-}" + case "$shell" in + bash) + cat < enrich -> convert -> validate -> import (default)' + 'clean:Remove local working artifacts' + 'completion:Print a shell completion script' + ) + _arguments -s \\ + '--org[StackGuardian org for import]:org' \\ + '--export-dir[Payload/state output dir]:dir:_files -/' \\ + '--mapping[Project-segment -> group override map]:file:_files' \\ + '--concurrency[Max parallel jobs for convert/import]:n' \\ + '--no-create-groups[Require workflow groups to pre-exist]' \\ + '--no-variable-sets[Skip merging TFC Variable Sets]' \\ + '--no-vcs-triggers[Skip registering VCS triggers after import]' \\ + '--all[With clean: also remove config]' \\ + '(-v --verbose)'{-v,--verbose}'[Show full terraform/tool output]' \\ + '(-y --yes)'{-y,--yes}'[Skip the import confirmation prompt]' \\ + '(-h --help)'{-h,--help}'[Show help]' \\ + '(--native --local)'{--native,--local}'[Run natively instead of in Docker]' \\ + '--build[Rebuild the Docker image first]' \\ + '1:command:->cmd' \\ + '2:shell:->shell' + case "\$state" in + cmd) _describe -t commands 'command' cmds ;; + shell) [[ "\${words[CURRENT-1]}" == completion ]] && _values 'shell' bash zsh ;; + esac +} +compdef _sg_migrate sg-migrate.sh migrate.sh +ZSH + ;; + *) die "usage: $0 completion " ;; + esac +} + CMD="" while [ $# -gt 0 ]; do case "$1" in @@ -585,6 +679,10 @@ while [ $# -gt 0 ]; do exit 0 ;; init | apply | enrich | convert | validate | import | triggers | all | clean) CMD="$1" ;; + completion) + cmd_completion "${2:-}" + exit 0 + ;; *) echo "Unknown argument: $1" >&2 usage diff --git a/sg-migrate.sh b/sg-migrate.sh index c295768..e1138d6 100755 --- a/sg-migrate.sh +++ b/sg-migrate.sh @@ -27,9 +27,9 @@ for a in "$@"; do esac done -# 'clean' only touches the local filesystem — no container needed. +# 'clean' and 'completion' only touch the local shell/filesystem — no container. for a in ${ARGS[@]+"${ARGS[@]}"}; do - [ "$a" = "clean" ] && NATIVE=1 + case "$a" in clean | completion) NATIVE=1 ;; esac done if [ "$NATIVE" = "1" ] || ! command -v docker >/dev/null 2>&1; then From 57e02e0ab3a4c33b2a296fb39941da6891eb1577 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adis=20Halilovi=C4=87?= Date: Mon, 7 Sep 2026 07:59:13 +0200 Subject: [PATCH 08/71] docs: add CLAUDE.md --- CLAUDE.md | 87 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 87 insertions(+) create mode 100644 CLAUDE.md diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..ef472f3 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,87 @@ +# CLAUDE.md + +This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. + +## What this is + +A migration tool that extracts workloads from other IaC platforms and transforms them into StackGuardian Workflow definitions (`sg-payload.json`), ready for bulk import via [sg-cli](https://github.com/StackGuardian/sg-cli). Currently the only implemented source is **Terraform Cloud / Enterprise (TFC/TFE)**. + +There is no application code to build — the "engine" is Terraform itself. The migrator is a Terraform root module that uses the `tfe` provider to read workspaces and the `local`/`null` providers to write the output payload and pull state files. + +## Architecture + +The migration is a user-driven pipeline, not a single program: + +1. **Extract + transform** — `transformer/terraform-cloud/` is a Terraform root module. `terraform apply` reads TFC/TFE workspaces and emits one `/sg-payload..json` per TFC project, a `migration-summary.md`/`.json` report, and per-workspace `.tfstate` files when `exportStateFiles` is true. +2. **Enrich (variable sets)** — `scripts/enrich_variable_sets.sh` merges TFC Variable Set variables into the payloads via the TFC API (the provider can't enumerate sets). Resolves global/project/workspace scope + TFC precedence per workspace; non-sensitive only. +3. **Tuning** — adjust per-workspace differences (integration IDs, VCS auth, runners, approvers, version) via the `workspaceOverrides` variable and re-apply; or hand-edit the payload files. `example_payload.jsonc` is the annotated field reference. +4. **HCL→JSON conversion** — `scripts/convert_hcl_to_json.sh` rewrites HCL-string variable values in each payload to real JSON. +5. **Validation** — `scripts/validate_payload.sh` checks each payload against `schema/sg-payload.schema.json` (downloads `yajsv`). +6. **Import** — `sg-cli workflow create --bulk`, run once per project file, each into its own workflow group. + +### Orchestration & tooling + +The five phases are wrapped by an orchestrator so users don't run them by hand: + +- `sg-migrate.sh` (host entrypoint, repo root) — runs `scripts/migrate.sh` **inside the Docker image** (`Dockerfile`), bind-mounting the repo at `/app` and forwarding `SG_API_TOKEN`/`SG_ORG`/`SG_BASE_URL`/`TFE_TOKEN`/`TF_TOKEN_*`. For TFC auth it prefers a long-lived `TFE_TOKEN` (env); otherwise it mounts `~/.terraform.d/credentials.tfrc.json` read-only. Runs natively instead when `--native`/`--local` is passed, `SG_NATIVE=1` is set, the command is `clean`, or Docker is absent. +- `scripts/migrate.sh` — the actual orchestrator. Subcommands `init|apply|enrich|convert|validate|import|triggers|all|clean|completion` (default `all`; `enrich` runs in `all` unless `--no-variable-sets`). Resolves each tool via `sg_resolve` (PATH first — the image installs them — else `sg_ensure_*` cache). Runs `convert` and `import` in parallel (`--concurrency`, default 4), raises `terraform apply -parallelism`, and retries `sg-cli` imports with backoff (`sg_retry`, `SG_RETRIES`). Import resolves each project's workflow group, checks existence via the SG API, shows a plan (`exists`/`create`), prompts (skip with `-y`), **creates the missing `tfc-` groups via the API**, then imports. sg-cli exits 0 even when individual workflows fail, so `do_import` parses its output: a workflow rejected as above SG's managed Terraform ceiling (1.5.7, last MPL/FOSS release) is re-imported with `SGDefaultTerraformVersion` (read from `terraform.tfvars`), the payload patched in place, and the case logged to `export/terraform-version-fallbacks.log` plus a printed notice; any other per-workflow failure fails the run. The trigger pass skips workflows that do not exist in SG, and `sg_api_post` returns 22 on 4xx so `sg_retry` (via `SG_NO_RETRY_RC`) does not retry definitive errors. `clean` removes local artifacts (`export/`, TF state, tool cache); `clean --all` also removes config. `completion bash|zsh` prints a completion script (`cmd_completion`; commands/options come from `SG_COMMANDS`/`SG_OPTIONS`, keep them in sync with the parser and the host flags); like `clean` it always runs natively. Output is concise by default — `apply` captures terraform's init/plan output and shows only progress + the final summary (or the full log on failure, `TF_IN_AUTOMATION=1`/`-no-color`); `-v`/`--verbose` (`SG_VERBOSE`) streams everything and un-gates the convert detail lines. Colored logging via `sg_step`/`sg_log`/`sg_success`/`sg_warn`/`sg_err`; paths shown relative via `sg_rel` (all auto-off when not a TTY / `NO_COLOR`). +- `scripts/tools.sh` — sourced by the other scripts (sets `SG_REPO_ROOT` to its parent dir). Provides `sg_resolve` (PATH-or-cache), `sg_ensure_{jq,hcl2json,yajsv,sgcli}` (pinned downloads into `.sg/cached/`; `sg-cli` is the Go binary), `sg_retry`, the color vars + log helpers. The Docker image bundles these tools at build time (`yajsv` built from source so arm64 works), so in-container runs never download. +- **Variable sets** — `scripts/enrich_variable_sets.sh ` (run by `cmd_enrich`, which reads `tfOrg`/`tfHostname` from `terraform.tfvars` via `hcl2json` — variable sets live in the **TFC** org, not `SG_ORG`). Lists all sets + vars via the TFC API (Bearer token from the `terraform login` creds file or `TFE_TOKEN`), computes per-workspace effective vars (scope: global/project/workspace; precedence: priority sets > workspace vars > non-priority sets), and merges them into each workflow (matched by workspace name parsed from `TfStateFilePath`). Sensitive set vars and key conflicts are reported. Skipped by `--no-variable-sets`. +- **Workflow groups** — each TFC project maps to an SG workflow group `tfc-`, created via `POST /api/v1/orgs/{org}/wfgrps/` (auth `Authorization: apikey `) if missing. (Both the API calls and sg-cli honor the undocumented `SG_BASE_URL` for non-prod targets.) Groups are addressed by name, so no generated ID is tracked. `.sg/workflow-groups.json` (gitignored, optional) overrides the target group per segment (`{"": ""}`); override groups are not auto-created. `--no-create-groups` requires all groups to pre-exist. + +### The transformer (`transformer/terraform-cloud/`) + +The whole transformation lives in `locals.tf` — there are no `outputs.tf`/`main.tf` business logic files; `main.tf` only pins provider versions. + +- `data.tf` — four data sources: `tfe_workspace_ids` (selects workspaces by name/tags), `tfe_workspace` (per-workspace details), `tfe_variables` (per-workspace variables), `tfe_projects` (project id→name, used to name per-project payload files). +- `locals.tf` — builds `local.workflowPayload` (workspace name → SG workflow object), groups it into `local.payloadByProject`, and assembles `local.summary`. This is the core mapping from TFC concepts to StackGuardian's payload schema. Key mappings: + - TFC `terraform` (non-sensitive) variables → `VCSConfig.iacInputData.data` (kept as strings here; `try(jsondecode(...), v.value)` only decodes values that are already valid JSON). + - TFC `env` (non-sensitive) variables → `EnvironmentVariables` as `PLAIN_TEXT`. + - `auto_apply` → inverted into `approvalPreApply` / gated `Approvers`. + - `terraform_version` → `TERRAFORM-` only for a pinned semver; otherwise `SGDefaultTerraformVersion`. + - `project_id` → `CLIConfiguration.WorkflowGroup.name` = `tfc-` (matches the per-project filename and the group the importer creates/targets). + - Sensitive variables (terraform + env) are skipped and recorded in the summary. + - Per-workspace `var.workspaceOverrides[]` fields take precedence over the `SGDefault*` values (resolved via `try(var.workspaceOverrides[name]., null) != null ? ... : `). + - `local.resourceNames` sanitizes workspace names to a valid SG `ResourceName` (≤100 chars, `^[-a-zA-Z0-9_]+$`, collision-disambiguated). For normal TFC names this is a no-op; any actual rename is reported in the summary. This is the single place to adjust naming rules. +- `resources.tf` — writes one `sg-payload..json` per project directly via `for_each` (no `mv`), plus `migration-summary.{md,json}`. When `exportStateFiles=true`, `null_resource.exportState` pulls each workspace's state **directly from the TFC/TFE API** (`GET /api/v2/workspaces/{id}/current-state-version` → `hosted-state-download-url`) via a `local-exec` `curl`/`jq` script — no `terraform init` or providers per workspace (avoids the plugin-cache concurrency bug and per-workspace provider downloads). The token is read at runtime from `~/.terraform.d/credentials.tfrc.json` (the `terraform login` file) or `TFE_TOKEN`, so it never enters TF state. Idempotent (keyed by workspace name/id; `forceStateRefresh` re-pulls), with per-workspace failures (no token / no state / download error) recorded in `state-export-failures.log` instead of aborting. `cmd_apply` ensures `jq`/`curl` are on PATH for the apply. +- `summary.tmpl` — renders `local.summary` to `migration-summary.md`. +- `variables.tf` / `terraform.tfvars.example` — inputs. `SGDefault*` variables supply global defaults baked into every workflow (deployment platform, VCS auth, repo prefix, source kind, approvers, Terraform version); `workspaceOverrides` overrides them per workspace; `forceStateRefresh` controls state re-pull. Requires Terraform `>= 1.3` (for `optional()` object attributes). + +### `scripts/convert_hcl_to_json.sh` + +Run from repo root, once per payload file (`for f in export/sg-payload.*.json; do ./scripts/convert_hcl_to_json.sh "$f"; done`). Rewrites the file **in place** (atomically — writes to a temp file, then `mv`s over the original). Downloads pinned `jq` and `hcl2json` binaries to a temp dir at runtime. For each workflow it walks `.VCSConfig.iacInputData.data` and, for any string value that looks like an HCL object/list (`{`/`[`), wraps it as `temp = ` and pipes through `hcl2json` to get real JSON. Plain scalar strings (ids, names) and already-converted objects pass through untouched. + +### `scripts/validate_payload.sh` + `schema/` + +`schema/sg-payload.schema.json` is a self-contained draft-07 schema for the generated payload array, with constraints (ResourceName length, `kind`/`sourceConfigDestKind` enums) derived from the StackGuardian OpenAPI spec (request body `#/components/schemas/Workflow`). The OpenAPI spec is **not committed** — download it from the [API Explorer](https://docs.stackguardian.io/api-reference/) if the schema needs regenerating. The schema is intentionally lenient (unknown fields allowed) — it checks the fields the transformer emits, not every optional API field. `scripts/validate_payload.sh` downloads `yajsv` and validates one or more payload files against it. + +## Common commands + +```shell +# Run the transformer +cd transformer/terraform-cloud +cp terraform.tfvars.example terraform.tfvars # then edit +terraform init +terraform apply -auto-approve -var-file=terraform.tfvars + +# Convert HCL-string vars to JSON, then validate (from repo root, per file) +for f in export/sg-payload.*.json; do ./scripts/convert_hcl_to_json.sh "$f"; done +./scripts/validate_payload.sh export/sg-payload.*.json + +# Bulk import (from the export dir, sg-cli fetched per README) — once per project file +export SG_API_TOKEN= +./sg-cli workflow create --bulk --workflow-group "" --org "" -- sg-payload..json +``` + +`terraform login` must be run first so the `tfe` provider can authenticate to TFC/TFE. + +## Conventions + +- Terraform variable/local naming is **camelCase** (e.g. `workflowNames`, `exportStateFiles`, `SGDefaultVCSAuthIntegrationID`) — not the usual snake_case. Match it. +- Output payload keys are PascalCase to match StackGuardian's API schema (`ResourceName`, `DeploymentPlatformConfig`, `VCSConfig`, ...). Keep `example_payload.jsonc` in sync when you change the generated shape. +- Run `terraform fmt` on `.tf` changes (see commit history). +- `export/`, `*.tfvars`, `*.tfstate`, and `.terraform/` are gitignored — they hold generated output and secrets. + +## Adding a new source platform + +The `transformer/` directory is the extension point: each platform is its own self-contained Terraform root module (mirror `terraform-cloud/`). The contract a transformer must satisfy is producing `sg-payload.*.json` arrays matching `example_payload.jsonc`; everything downstream (`scripts/convert_hcl_to_json.sh`, sg-cli import) is platform-agnostic. From 5ba9edfca5de7752959d51aa876f04994c52de5e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adis=20Halilovi=C4=87?= Date: Mon, 7 Sep 2026 08:18:32 +0200 Subject: [PATCH 09/71] feat: show help menu when no command is given Running sg-migrate.sh without a command used to start the whole pipeline; it now prints the help menu, locally, without touching Docker. Usage and hints show the name the user invoked (./sg-migrate.sh) instead of the in-container script path. --- CLAUDE.md | 4 ++-- README.md | 2 +- scripts/migrate.sh | 21 +++++++++++++-------- sg-migrate.sh | 15 ++++++++++++--- 4 files changed, 28 insertions(+), 14 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index ef472f3..e05407d 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -23,8 +23,8 @@ The migration is a user-driven pipeline, not a single program: The five phases are wrapped by an orchestrator so users don't run them by hand: -- `sg-migrate.sh` (host entrypoint, repo root) — runs `scripts/migrate.sh` **inside the Docker image** (`Dockerfile`), bind-mounting the repo at `/app` and forwarding `SG_API_TOKEN`/`SG_ORG`/`SG_BASE_URL`/`TFE_TOKEN`/`TF_TOKEN_*`. For TFC auth it prefers a long-lived `TFE_TOKEN` (env); otherwise it mounts `~/.terraform.d/credentials.tfrc.json` read-only. Runs natively instead when `--native`/`--local` is passed, `SG_NATIVE=1` is set, the command is `clean`, or Docker is absent. -- `scripts/migrate.sh` — the actual orchestrator. Subcommands `init|apply|enrich|convert|validate|import|triggers|all|clean|completion` (default `all`; `enrich` runs in `all` unless `--no-variable-sets`). Resolves each tool via `sg_resolve` (PATH first — the image installs them — else `sg_ensure_*` cache). Runs `convert` and `import` in parallel (`--concurrency`, default 4), raises `terraform apply -parallelism`, and retries `sg-cli` imports with backoff (`sg_retry`, `SG_RETRIES`). Import resolves each project's workflow group, checks existence via the SG API, shows a plan (`exists`/`create`), prompts (skip with `-y`), **creates the missing `tfc-` groups via the API**, then imports. sg-cli exits 0 even when individual workflows fail, so `do_import` parses its output: a workflow rejected as above SG's managed Terraform ceiling (1.5.7, last MPL/FOSS release) is re-imported with `SGDefaultTerraformVersion` (read from `terraform.tfvars`), the payload patched in place, and the case logged to `export/terraform-version-fallbacks.log` plus a printed notice; any other per-workflow failure fails the run. The trigger pass skips workflows that do not exist in SG, and `sg_api_post` returns 22 on 4xx so `sg_retry` (via `SG_NO_RETRY_RC`) does not retry definitive errors. `clean` removes local artifacts (`export/`, TF state, tool cache); `clean --all` also removes config. `completion bash|zsh` prints a completion script (`cmd_completion`; commands/options come from `SG_COMMANDS`/`SG_OPTIONS`, keep them in sync with the parser and the host flags); like `clean` it always runs natively. Output is concise by default — `apply` captures terraform's init/plan output and shows only progress + the final summary (or the full log on failure, `TF_IN_AUTOMATION=1`/`-no-color`); `-v`/`--verbose` (`SG_VERBOSE`) streams everything and un-gates the convert detail lines. Colored logging via `sg_step`/`sg_log`/`sg_success`/`sg_warn`/`sg_err`; paths shown relative via `sg_rel` (all auto-off when not a TTY / `NO_COLOR`). +- `sg-migrate.sh` (host entrypoint, repo root) — runs `scripts/migrate.sh` **inside the Docker image** (`Dockerfile`), bind-mounting the repo at `/app` and forwarding `SG_API_TOKEN`/`SG_ORG`/`SG_BASE_URL`/`TFE_TOKEN`/`TF_TOKEN_*`. For TFC auth it prefers a long-lived `TFE_TOKEN` (env); otherwise it mounts `~/.terraform.d/credentials.tfrc.json` read-only. Runs natively instead when `--native`/`--local` is passed, `SG_NATIVE=1` is set, the command is `clean`/`completion`/help (or no command is given), or Docker is absent. Exports `SG_PROG` so `migrate.sh` shows `./sg-migrate.sh` in its usage/hints. +- `scripts/migrate.sh` — the actual orchestrator. Subcommands `init|apply|enrich|convert|validate|import|triggers|all|clean|completion` (no command prints the help menu; `enrich` runs in `all` unless `--no-variable-sets`). Resolves each tool via `sg_resolve` (PATH first — the image installs them — else `sg_ensure_*` cache). Runs `convert` and `import` in parallel (`--concurrency`, default 4), raises `terraform apply -parallelism`, and retries `sg-cli` imports with backoff (`sg_retry`, `SG_RETRIES`). Import resolves each project's workflow group, checks existence via the SG API, shows a plan (`exists`/`create`), prompts (skip with `-y`), **creates the missing `tfc-` groups via the API**, then imports. sg-cli exits 0 even when individual workflows fail, so `do_import` parses its output: a workflow rejected as above SG's managed Terraform ceiling (1.5.7, last MPL/FOSS release) is re-imported with `SGDefaultTerraformVersion` (read from `terraform.tfvars`), the payload patched in place, and the case logged to `export/terraform-version-fallbacks.log` plus a printed notice; any other per-workflow failure fails the run. The trigger pass skips workflows that do not exist in SG, and `sg_api_post` returns 22 on 4xx so `sg_retry` (via `SG_NO_RETRY_RC`) does not retry definitive errors. `clean` removes local artifacts (`export/`, TF state, tool cache); `clean --all` also removes config. `completion bash|zsh` prints a completion script (`cmd_completion`; commands/options come from `SG_COMMANDS`/`SG_OPTIONS`, keep them in sync with the parser and the host flags); like `clean` it always runs natively. Output is concise by default — `apply` captures terraform's init/plan output and shows only progress + the final summary (or the full log on failure, `TF_IN_AUTOMATION=1`/`-no-color`); `-v`/`--verbose` (`SG_VERBOSE`) streams everything and un-gates the convert detail lines. Colored logging via `sg_step`/`sg_log`/`sg_success`/`sg_warn`/`sg_err`; paths shown relative via `sg_rel` (all auto-off when not a TTY / `NO_COLOR`). - `scripts/tools.sh` — sourced by the other scripts (sets `SG_REPO_ROOT` to its parent dir). Provides `sg_resolve` (PATH-or-cache), `sg_ensure_{jq,hcl2json,yajsv,sgcli}` (pinned downloads into `.sg/cached/`; `sg-cli` is the Go binary), `sg_retry`, the color vars + log helpers. The Docker image bundles these tools at build time (`yajsv` built from source so arm64 works), so in-container runs never download. - **Variable sets** — `scripts/enrich_variable_sets.sh ` (run by `cmd_enrich`, which reads `tfOrg`/`tfHostname` from `terraform.tfvars` via `hcl2json` — variable sets live in the **TFC** org, not `SG_ORG`). Lists all sets + vars via the TFC API (Bearer token from the `terraform login` creds file or `TFE_TOKEN`), computes per-workspace effective vars (scope: global/project/workspace; precedence: priority sets > workspace vars > non-priority sets), and merges them into each workflow (matched by workspace name parsed from `TfStateFilePath`). Sensitive set vars and key conflicts are reported. Skipped by `--no-variable-sets`. - **Workflow groups** — each TFC project maps to an SG workflow group `tfc-`, created via `POST /api/v1/orgs/{org}/wfgrps/` (auth `Authorization: apikey `) if missing. (Both the API calls and sg-cli honor the undocumented `SG_BASE_URL` for non-prod targets.) Groups are addressed by name, so no generated ID is tracked. `.sg/workflow-groups.json` (gitignored, optional) overrides the target group per segment (`{"": ""}`); override groups are not auto-created. `--no-create-groups` requires all groups to pre-exist. diff --git a/README.md b/README.md index 4c0d291..d965523 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,7 @@ export SG_ORG= That's it — no workflow-group mapping to fill in. Each TFC project is imported into an SG workflow group named `tfc-`, **created automatically via the API** if it doesn't exist. The import prompt shows each group as `exists` or `create` before anything is written. -- Single phase: `./sg-migrate.sh apply|enrich|convert|validate|import`. +- Single phase: `./sg-migrate.sh apply|enrich|convert|validate|import|triggers`. Running `./sg-migrate.sh` with no command prints the help menu. - TFC **Variable Set** variables are merged into the payloads automatically (the `enrich` phase, via the TFC API); skip it with `--no-variable-sets`. - `./sg-migrate.sh clean` removes local working artifacts (`export/`, Terraform state, tool cache) for a fresh start; add `--all` to also remove config. `clean` always runs locally. - **Override** a project's target group (to reuse an existing group) in `.sg/workflow-groups.json`: `{"": ""}`. Override groups must already exist (they're not auto-created). diff --git a/scripts/migrate.sh b/scripts/migrate.sh index c765e63..7d4e063 100755 --- a/scripts/migrate.sh +++ b/scripts/migrate.sh @@ -14,6 +14,7 @@ source "$SCRIPT_DIR/tools.sh" # root used for all repo-relative paths. TRANSFORMER_DIR="$SG_REPO_ROOT/transformer/terraform-cloud" TFVARS="$TRANSFORMER_DIR/terraform.tfvars" +PROG="${SG_PROG:-$0}" EXPORT_DIR="${SG_EXPORT_DIR:-$SG_REPO_ROOT/export}" MAPPING="${SG_WFGROUP_MAP:-$SG_REPO_ROOT/.sg/workflow-groups.json}" ORG="${SG_ORG:-}" @@ -32,7 +33,7 @@ PF=() usage() { cat >&2 < Commands: init Create terraform.tfvars from the template @@ -43,7 +44,7 @@ Commands: import Import each payload to StackGuardian (parallel, with confirmation), then register VCS triggers (unless --no-vcs-triggers) triggers Register VCS triggers for already-imported workflows (second pass) - all apply -> enrich -> convert -> validate -> import (default) + all apply -> enrich -> convert -> validate -> import clean Remove local working artifacts for a fresh start (export/, TF state, tool cache). Add --all to also remove config (terraform.tfvars, mapping). completion Print a shell completion script for the current session: @@ -261,7 +262,7 @@ set_triggers_pass() { if run_parallel do_set_triggers "$CONC" "${PF[@]}"; then sg_success "vcs triggers registered" else - sg_err "one or more VCS trigger registrations failed (re-run: $0 triggers)" + sg_err "one or more VCS trigger registrations failed (re-run: $PROG triggers)" return 1 fi } @@ -333,7 +334,7 @@ cmd_apply() { cmd_enrich() { sg_step "Phase: variable sets" - [ -f "$TFVARS" ] || die "Missing $(sg_rel "$TFVARS") (run: $0 init)." + [ -f "$TFVARS" ] || die "Missing $(sg_rel "$TFVARS") (run: $PROG init)." payload_files [ "${#PF[@]}" -gt 0 ] || die "No payload files in $(sg_rel "$EXPORT_DIR") (run 'apply' first)." # Variable sets belong to the TFC org (tfOrg in terraform.tfvars), not SG_ORG. @@ -613,7 +614,7 @@ _sg_migrate() { 'validate:Validate payloads against the SG schema' 'import:Import payloads to StackGuardian, then register VCS triggers' 'triggers:Register VCS triggers for already-imported workflows' - 'all:apply -> enrich -> convert -> validate -> import (default)' + 'all:apply -> enrich -> convert -> validate -> import' 'clean:Remove local working artifacts' 'completion:Print a shell completion script' ) @@ -641,7 +642,7 @@ _sg_migrate() { compdef _sg_migrate sg-migrate.sh migrate.sh ZSH ;; - *) die "usage: $0 completion " ;; + *) die "usage: $PROG completion " ;; esac } @@ -691,7 +692,11 @@ while [ $# -gt 0 ]; do esac shift done -CMD="${CMD:-all}" +# No command: show the help menu instead of running the whole pipeline. +if [ -z "$CMD" ]; then + usage + exit 0 +fi export SG_VERBOSE="$VERBOSE" case "$CMD" in @@ -706,7 +711,7 @@ triggers) cmd_triggers ;; all) if [ ! -f "$TFVARS" ]; then cmd_init - die "Edit $(sg_rel "$TFVARS"), then re-run '$0 all'." + die "Edit $(sg_rel "$TFVARS"), then re-run '$PROG all'." fi # Fail fast on import prerequisites before the (long) apply. [ -n "${SG_API_TOKEN:-}" ] || die "SG_API_TOKEN is not set (needed for import)." diff --git a/sg-migrate.sh b/sg-migrate.sh index e1138d6..14cd745 100755 --- a/sg-migrate.sh +++ b/sg-migrate.sh @@ -27,10 +27,19 @@ for a in "$@"; do esac done -# 'clean' and 'completion' only touch the local shell/filesystem — no container. +# Help, 'clean' and 'completion' only touch the local shell/filesystem — no +# container. With no command at all, migrate.sh prints the help menu. +HAS_CMD=0 for a in ${ARGS[@]+"${ARGS[@]}"}; do - case "$a" in clean | completion) NATIVE=1 ;; esac + case "$a" in + clean | completion | -h | --help) NATIVE=1; HAS_CMD=1 ;; + init | apply | enrich | convert | validate | import | triggers | all) HAS_CMD=1 ;; + esac done +[ "$HAS_CMD" -eq 1 ] || NATIVE=1 +# Let migrate.sh print the name the user actually invoked in its help/hints. +export SG_PROG="${0##*/}" +case "$0" in */*) SG_PROG="./${0##*/}" ;; esac if [ "$NATIVE" = "1" ] || ! command -v docker >/dev/null 2>&1; then [ "$NATIVE" = "1" ] || sg_warn "docker not found; running natively" @@ -45,7 +54,7 @@ fi DOCKER_ARGS=(--rm -i -v "$SCRIPT_DIR:/app" -w /app -e SG_API_TOKEN -e SG_ORG -e SG_BASE_URL -e SG_CONCURRENCY -e SG_RETRIES -e SG_TF_PARALLELISM - -e TFE_TOKEN) + -e TFE_TOKEN -e SG_PROG) # Interactive TTY only when attached to one (so the confirmation prompt works, # but CI/non-tty invocations still run — use -y there). From 7a518234cbcdf1272d8deb8a8ca75123b63e9f55 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adis=20Halilovi=C4=87?= Date: Mon, 7 Sep 2026 08:23:42 +0200 Subject: [PATCH 10/71] fix: fail fast without TFC credentials and configure tfe provider host Without TFE_TOKEN or a terraform login credentials file, apply used to warn and then fail deep inside terraform with "Invalid provider configuration". Both the host entrypoint and migrate.sh now stop with a clear message before anything runs. Add an explicit provider "tfe" block so tfHostname also applies to the provider, not just the state export. --- scripts/migrate.sh | 18 ++++++++++++++++-- sg-migrate.sh | 6 +++++- transformer/terraform-cloud/main.tf | 7 +++++++ 3 files changed, 28 insertions(+), 3 deletions(-) diff --git a/scripts/migrate.sh b/scripts/migrate.sh index 7d4e063..b186a61 100755 --- a/scripts/migrate.sh +++ b/scripts/migrate.sh @@ -294,10 +294,23 @@ cmd_clean() { sg_success "clean complete" } +# require_tfc_auth — fail fast with a clear message when no TFC/TFE credential +# is available for the tfe provider. Accepted: TFE_TOKEN, any TF_TOKEN_* var +# (terraform's per-host token env), or the `terraform login` credentials file. +# Without this, terraform fails later with a confusing "Invalid provider +# configuration" error. +require_tfc_auth() { + [ -n "${TFE_TOKEN:-}" ] && return 0 + if env | grep -q '^TF_TOKEN_'; then return 0; fi + [ -f "$HOME/.terraform.d/credentials.tfrc.json" ] && return 0 + die "no Terraform Cloud/Enterprise credentials found. Set TFE_TOKEN= (recommended) or run 'terraform login' before '$PROG apply'." +} + cmd_apply() { sg_step "Phase: apply (terraform)" command -v terraform >/dev/null 2>&1 || die "terraform not found on PATH" - [ -f "$TFVARS" ] || die "Missing $(sg_rel "$TFVARS"). Run: $0 init (then edit it)." + [ -f "$TFVARS" ] || die "Missing $(sg_rel "$TFVARS"). Run: $PROG init (then edit it)." + require_tfc_auth # State export (TFC API) calls curl + jq from terraform's local-exec; make sure # both are on PATH for the apply (jq from cache if not already installed). command -v curl >/dev/null 2>&1 || die "curl is required for state export" @@ -713,7 +726,8 @@ all) cmd_init die "Edit $(sg_rel "$TFVARS"), then re-run '$PROG all'." fi - # Fail fast on import prerequisites before the (long) apply. + # Fail fast on prerequisites before the (long) apply. + require_tfc_auth [ -n "${SG_API_TOKEN:-}" ] || die "SG_API_TOKEN is not set (needed for import)." [ -n "$ORG" ] || die "StackGuardian org not set (use --org or SG_ORG)." cmd_apply diff --git a/sg-migrate.sh b/sg-migrate.sh index 14cd745..b43c84e 100755 --- a/sg-migrate.sh +++ b/sg-migrate.sh @@ -67,7 +67,11 @@ if [ -n "${TFE_TOKEN:-}" ]; then elif [ -f "$CREDS" ]; then DOCKER_ARGS+=(-v "$CREDS:/root/.terraform.d/credentials.tfrc.json:ro") else - sg_warn "no TFC auth found — set TFE_TOKEN (long-lived API token) or run 'terraform login'" + # Only apply/all need TFC auth; migrate.sh fails fast there with a clear message. + case " ${ARGS[*]-} " in *" apply "* | *" all "*) + sg_err "no Terraform Cloud/Enterprise credentials found. Set TFE_TOKEN= (recommended) or run 'terraform login' first." + exit 1 ;; + esac fi # Forward any TF_TOKEN_* env vars (alternative TFC/TFE auth) if present. diff --git a/transformer/terraform-cloud/main.tf b/transformer/terraform-cloud/main.tf index 9a19179..7e8dd58 100644 --- a/transformer/terraform-cloud/main.tf +++ b/transformer/terraform-cloud/main.tf @@ -16,3 +16,10 @@ terraform { } } } + +# Token comes from TFE_TOKEN / TF_TOKEN_ or the `terraform login` +# credentials file; only the hostname is configured here so TFE (self-hosted) +# installs work by setting tfHostname. +provider "tfe" { + hostname = var.tfHostname +} From 80dbd14ade87040679e87088a23849cf4f2e430c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adis=20Halilovi=C4=87?= Date: Mon, 7 Sep 2026 08:43:51 +0200 Subject: [PATCH 11/71] fix: verify TFC credentials before apply An expired 'terraform login' session (or a bad TFE_TOKEN) used to fail deep inside terraform. Resolve the token the tfe provider will use, in its own precedence, and check it against /api/v2/account/details first; stop with a clear message on 401/403 or when the host is unreachable. --- CLAUDE.md | 2 +- scripts/migrate.sh | 48 +++++++++++++++++++++++++++++++++++++--------- 2 files changed, 40 insertions(+), 10 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index e05407d..d972fd8 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -24,7 +24,7 @@ The migration is a user-driven pipeline, not a single program: The five phases are wrapped by an orchestrator so users don't run them by hand: - `sg-migrate.sh` (host entrypoint, repo root) — runs `scripts/migrate.sh` **inside the Docker image** (`Dockerfile`), bind-mounting the repo at `/app` and forwarding `SG_API_TOKEN`/`SG_ORG`/`SG_BASE_URL`/`TFE_TOKEN`/`TF_TOKEN_*`. For TFC auth it prefers a long-lived `TFE_TOKEN` (env); otherwise it mounts `~/.terraform.d/credentials.tfrc.json` read-only. Runs natively instead when `--native`/`--local` is passed, `SG_NATIVE=1` is set, the command is `clean`/`completion`/help (or no command is given), or Docker is absent. Exports `SG_PROG` so `migrate.sh` shows `./sg-migrate.sh` in its usage/hints. -- `scripts/migrate.sh` — the actual orchestrator. Subcommands `init|apply|enrich|convert|validate|import|triggers|all|clean|completion` (no command prints the help menu; `enrich` runs in `all` unless `--no-variable-sets`). Resolves each tool via `sg_resolve` (PATH first — the image installs them — else `sg_ensure_*` cache). Runs `convert` and `import` in parallel (`--concurrency`, default 4), raises `terraform apply -parallelism`, and retries `sg-cli` imports with backoff (`sg_retry`, `SG_RETRIES`). Import resolves each project's workflow group, checks existence via the SG API, shows a plan (`exists`/`create`), prompts (skip with `-y`), **creates the missing `tfc-` groups via the API**, then imports. sg-cli exits 0 even when individual workflows fail, so `do_import` parses its output: a workflow rejected as above SG's managed Terraform ceiling (1.5.7, last MPL/FOSS release) is re-imported with `SGDefaultTerraformVersion` (read from `terraform.tfvars`), the payload patched in place, and the case logged to `export/terraform-version-fallbacks.log` plus a printed notice; any other per-workflow failure fails the run. The trigger pass skips workflows that do not exist in SG, and `sg_api_post` returns 22 on 4xx so `sg_retry` (via `SG_NO_RETRY_RC`) does not retry definitive errors. `clean` removes local artifacts (`export/`, TF state, tool cache); `clean --all` also removes config. `completion bash|zsh` prints a completion script (`cmd_completion`; commands/options come from `SG_COMMANDS`/`SG_OPTIONS`, keep them in sync with the parser and the host flags); like `clean` it always runs natively. Output is concise by default — `apply` captures terraform's init/plan output and shows only progress + the final summary (or the full log on failure, `TF_IN_AUTOMATION=1`/`-no-color`); `-v`/`--verbose` (`SG_VERBOSE`) streams everything and un-gates the convert detail lines. Colored logging via `sg_step`/`sg_log`/`sg_success`/`sg_warn`/`sg_err`; paths shown relative via `sg_rel` (all auto-off when not a TTY / `NO_COLOR`). +- `scripts/migrate.sh` — the actual orchestrator. Subcommands `init|apply|enrich|convert|validate|import|triggers|all|clean|completion` (no command prints the help menu; `enrich` runs in `all` unless `--no-variable-sets`). Resolves each tool via `sg_resolve` (PATH first — the image installs them — else `sg_ensure_*` cache). `apply`/`all` first run `require_tfc_auth`, which resolves the token the tfe provider will use (`TFE_TOKEN`, `TF_TOKEN_`, or the `terraform login` file for `tfHostname`) and verifies it with `GET /api/v2/account/details`, failing fast on a missing or rejected (expired) credential. Runs `convert` and `import` in parallel (`--concurrency`, default 4), raises `terraform apply -parallelism`, and retries `sg-cli` imports with backoff (`sg_retry`, `SG_RETRIES`). Import resolves each project's workflow group, checks existence via the SG API, shows a plan (`exists`/`create`), prompts (skip with `-y`), **creates the missing `tfc-` groups via the API**, then imports. sg-cli exits 0 even when individual workflows fail, so `do_import` parses its output: a workflow rejected as above SG's managed Terraform ceiling (1.5.7, last MPL/FOSS release) is re-imported with `SGDefaultTerraformVersion` (read from `terraform.tfvars`), the payload patched in place, and the case logged to `export/terraform-version-fallbacks.log` plus a printed notice; any other per-workflow failure fails the run. The trigger pass skips workflows that do not exist in SG, and `sg_api_post` returns 22 on 4xx so `sg_retry` (via `SG_NO_RETRY_RC`) does not retry definitive errors. `clean` removes local artifacts (`export/`, TF state, tool cache); `clean --all` also removes config. `completion bash|zsh` prints a completion script (`cmd_completion`; commands/options come from `SG_COMMANDS`/`SG_OPTIONS`, keep them in sync with the parser and the host flags); like `clean` it always runs natively. Output is concise by default — `apply` captures terraform's init/plan output and shows only progress + the final summary (or the full log on failure, `TF_IN_AUTOMATION=1`/`-no-color`); `-v`/`--verbose` (`SG_VERBOSE`) streams everything and un-gates the convert detail lines. Colored logging via `sg_step`/`sg_log`/`sg_success`/`sg_warn`/`sg_err`; paths shown relative via `sg_rel` (all auto-off when not a TTY / `NO_COLOR`). - `scripts/tools.sh` — sourced by the other scripts (sets `SG_REPO_ROOT` to its parent dir). Provides `sg_resolve` (PATH-or-cache), `sg_ensure_{jq,hcl2json,yajsv,sgcli}` (pinned downloads into `.sg/cached/`; `sg-cli` is the Go binary), `sg_retry`, the color vars + log helpers. The Docker image bundles these tools at build time (`yajsv` built from source so arm64 works), so in-container runs never download. - **Variable sets** — `scripts/enrich_variable_sets.sh ` (run by `cmd_enrich`, which reads `tfOrg`/`tfHostname` from `terraform.tfvars` via `hcl2json` — variable sets live in the **TFC** org, not `SG_ORG`). Lists all sets + vars via the TFC API (Bearer token from the `terraform login` creds file or `TFE_TOKEN`), computes per-workspace effective vars (scope: global/project/workspace; precedence: priority sets > workspace vars > non-priority sets), and merges them into each workflow (matched by workspace name parsed from `TfStateFilePath`). Sensitive set vars and key conflicts are reported. Skipped by `--no-variable-sets`. - **Workflow groups** — each TFC project maps to an SG workflow group `tfc-`, created via `POST /api/v1/orgs/{org}/wfgrps/` (auth `Authorization: apikey `) if missing. (Both the API calls and sg-cli honor the undocumented `SG_BASE_URL` for non-prod targets.) Groups are addressed by name, so no generated ID is tracked. `.sg/workflow-groups.json` (gitignored, optional) overrides the target group per segment (`{"": ""}`); override groups are not auto-created. `--no-create-groups` requires all groups to pre-exist. diff --git a/scripts/migrate.sh b/scripts/migrate.sh index b186a61..4d1f3a3 100755 --- a/scripts/migrate.sh +++ b/scripts/migrate.sh @@ -294,16 +294,46 @@ cmd_clean() { sg_success "clean complete" } -# require_tfc_auth — fail fast with a clear message when no TFC/TFE credential -# is available for the tfe provider. Accepted: TFE_TOKEN, any TF_TOKEN_* var -# (terraform's per-host token env), or the `terraform login` credentials file. -# Without this, terraform fails later with a confusing "Invalid provider -# configuration" error. +# tfc_hostname — TFC/TFE host from terraform.tfvars (tfHostname), default app.terraform.io. +tfc_hostname() { + local h="" + if [ -f "$TFVARS" ]; then + h="$("$(sg_resolve hcl2json sg_ensure_hcl2json)" "$TFVARS" 2>/dev/null | "$(sg_resolve jq sg_ensure_jq)" -r '.tfHostname // empty' 2>/dev/null || true)" + fi + echo "${h:-app.terraform.io}" +} + +# tfc_token — resolve the TFC/TFE token the tfe provider will use, in +# terraform's own precedence: TFE_TOKEN, TF_TOKEN_ ('.'->'_', '-'->'__'), +# then the `terraform login` credentials file. Prints nothing if none is set. +tfc_token() { + local host="$1" var creds + if [ -n "${TFE_TOKEN:-}" ]; then echo "$TFE_TOKEN"; return; fi + var="TF_TOKEN_$(echo "$host" | sed 's/-/__/g; s/\./_/g')" + if [ -n "${!var:-}" ]; then echo "${!var}"; return; fi + creds="$HOME/.terraform.d/credentials.tfrc.json" + [ -f "$creds" ] && "$(sg_resolve jq sg_ensure_jq)" -r --arg h "$host" '.credentials[$h].token // empty' "$creds" 2>/dev/null || true +} + +# require_tfc_auth — fail fast, before the (long) apply, when the tfe provider +# would not be able to authenticate: no credential at all, or a credential +# that TFC/TFE rejects (e.g. an expired `terraform login` session). Verified +# with GET /api/v2/account/details. Without this, terraform fails later with a +# confusing "Invalid provider configuration" error. require_tfc_auth() { - [ -n "${TFE_TOKEN:-}" ] && return 0 - if env | grep -q '^TF_TOKEN_'; then return 0; fi - [ -f "$HOME/.terraform.d/credentials.tfrc.json" ] && return 0 - die "no Terraform Cloud/Enterprise credentials found. Set TFE_TOKEN= (recommended) or run 'terraform login' before '$PROG apply'." + local host token code src + host="$(tfc_hostname)" + token="$(tfc_token "$host")" + if [ -n "${TFE_TOKEN:-}" ]; then src="TFE_TOKEN"; elif env | grep -q '^TF_TOKEN_'; then src="the TF_TOKEN_* variable"; else src="the 'terraform login' session (likely expired)"; fi + [ -n "$token" ] || die "no Terraform Cloud/Enterprise credentials found for $host. Set TFE_TOKEN= (recommended) or run 'terraform login' before '$PROG apply'." + command -v curl >/dev/null 2>&1 || return 0 + code="$(curl -sS -o /dev/null -w '%{http_code}' -H "Authorization: Bearer $token" "https://$host/api/v2/account/details" 2>/dev/null || echo 000)" + case "$code" in + 200) sg_log "TFC/TFE credentials verified ($host)" ;; + 401 | 403) die "Terraform Cloud/Enterprise rejected $src for $host (HTTP $code). Set TFE_TOKEN= or re-run 'terraform login'." ;; + 000) die "could not reach https://$host to verify TFC/TFE credentials (network/proxy?)" ;; + *) sg_warn "unexpected HTTP $code verifying TFC/TFE credentials at $host; continuing" ;; + esac } cmd_apply() { From 573a66f9ab6f48733d5cb21408c1f12dca6eb413 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adis=20Halilovi=C4=87?= Date: Mon, 7 Sep 2026 08:50:49 +0200 Subject: [PATCH 12/71] feat: global runner constraints via SGDefaultRunnerConstraints Every workflow was hardcoded to shared runners unless overridden per workspace. Add SGDefaultRunnerConstraints (default shared) so all workflows can be put behind a private runner group in one place; workspaceOverrides[name].RunnerConstraints still wins per workspace. Validated: type is shared|private, and private requires names. --- CLAUDE.md | 2 +- README.md | 2 +- transformer/terraform-cloud/locals.tf | 2 +- .../terraform-cloud/terraform.tfvars.example | 5 +++++ transformer/terraform-cloud/variables.tf | 17 +++++++++++++++++ 5 files changed, 25 insertions(+), 3 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index d972fd8..ad9f762 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -45,7 +45,7 @@ The whole transformation lives in `locals.tf` — there are no `outputs.tf`/`mai - `local.resourceNames` sanitizes workspace names to a valid SG `ResourceName` (≤100 chars, `^[-a-zA-Z0-9_]+$`, collision-disambiguated). For normal TFC names this is a no-op; any actual rename is reported in the summary. This is the single place to adjust naming rules. - `resources.tf` — writes one `sg-payload..json` per project directly via `for_each` (no `mv`), plus `migration-summary.{md,json}`. When `exportStateFiles=true`, `null_resource.exportState` pulls each workspace's state **directly from the TFC/TFE API** (`GET /api/v2/workspaces/{id}/current-state-version` → `hosted-state-download-url`) via a `local-exec` `curl`/`jq` script — no `terraform init` or providers per workspace (avoids the plugin-cache concurrency bug and per-workspace provider downloads). The token is read at runtime from `~/.terraform.d/credentials.tfrc.json` (the `terraform login` file) or `TFE_TOKEN`, so it never enters TF state. Idempotent (keyed by workspace name/id; `forceStateRefresh` re-pulls), with per-workspace failures (no token / no state / download error) recorded in `state-export-failures.log` instead of aborting. `cmd_apply` ensures `jq`/`curl` are on PATH for the apply. - `summary.tmpl` — renders `local.summary` to `migration-summary.md`. -- `variables.tf` / `terraform.tfvars.example` — inputs. `SGDefault*` variables supply global defaults baked into every workflow (deployment platform, VCS auth, repo prefix, source kind, approvers, Terraform version); `workspaceOverrides` overrides them per workspace; `forceStateRefresh` controls state re-pull. Requires Terraform `>= 1.3` (for `optional()` object attributes). +- `variables.tf` / `terraform.tfvars.example` — inputs. `SGDefault*` variables supply global defaults baked into every workflow (deployment platform, VCS auth, repo prefix, source kind, approvers, Terraform version, runner constraints — `SGDefaultRunnerConstraints`, validated to `shared` or `private`+`names`); `workspaceOverrides` overrides them per workspace; `forceStateRefresh` controls state re-pull. Requires Terraform `>= 1.3` (for `optional()` object attributes). ### `scripts/convert_hcl_to_json.sh` diff --git a/README.md b/README.md index d965523..b47ab17 100644 --- a/README.md +++ b/README.md @@ -44,7 +44,7 @@ The manual, step-by-step flow below remains supported for fine-grained control a ## Prerequisites - An organization on [StackGuardian Platform](https://app.stackguardian.io) -- Optionally, pre-configure VCS, cloud integrations or private runners to use when importing into StackGuardian Platform. +- Optionally, pre-configure VCS, cloud integrations or private runners to use when importing into StackGuardian Platform. To run every workflow on a private runner group, set `SGDefaultRunnerConstraints = { type = "private", names = [""] }` in `terraform.tfvars` (per-workspace exceptions via `workspaceOverrides[].RunnerConstraints`). - Terraform - [sg-cli](https://github.com/StackGuardian/sg-cli) diff --git a/transformer/terraform-cloud/locals.tf b/transformer/terraform-cloud/locals.tf index 48a6ebc..d2ba8f0 100644 --- a/transformer/terraform-cloud/locals.tf +++ b/transformer/terraform-cloud/locals.tf @@ -77,7 +77,7 @@ locals { ) DeploymentPlatformConfig = try(var.workspaceOverrides[wsName].DeploymentPlatformConfig, null) != null ? var.workspaceOverrides[wsName].DeploymentPlatformConfig : var.SGDefaultDeploymentPlatformConfig - RunnerConstraints = try(var.workspaceOverrides[wsName].RunnerConstraints, null) != null ? var.workspaceOverrides[wsName].RunnerConstraints : { "type" : "shared" } + RunnerConstraints = try(var.workspaceOverrides[wsName].RunnerConstraints, null) != null ? var.workspaceOverrides[wsName].RunnerConstraints : { for k, v in var.SGDefaultRunnerConstraints : k => v if v != null } VCSConfig = { "iacVCSConfig" : { diff --git a/transformer/terraform-cloud/terraform.tfvars.example b/transformer/terraform-cloud/terraform.tfvars.example index 6b0024a..5b7143f 100644 --- a/transformer/terraform-cloud/terraform.tfvars.example +++ b/transformer/terraform-cloud/terraform.tfvars.example @@ -36,6 +36,11 @@ SGDefaultDeploymentPlatformConfig = [ } ] +# Runners for every workflow: SG-hosted shared runners (default), or put all +# workflows behind a private runner group: +# SGDefaultRunnerConstraints = { type = "private", names = ["sg-runner"] } +SGDefaultRunnerConstraints = { type = "shared" } + # Choose from: GITHUB_COM, BITBUCKET_ORG, GITLAB_COM, AZURE_DEVOPS, GIT_OTHER SGDefaultSourceConfigDestKind = "GITHUB_COM" diff --git a/transformer/terraform-cloud/variables.tf b/transformer/terraform-cloud/variables.tf index 3c3b960..26d8299 100644 --- a/transformer/terraform-cloud/variables.tf +++ b/transformer/terraform-cloud/variables.tf @@ -71,6 +71,23 @@ variable "SGDefaultDeploymentPlatformConfig" { type = list(any) } +variable "SGDefaultRunnerConstraints" { + default = { type = "shared" } + description = "Runner constraints applied to every workflow. Use { type = \"shared\" } for SG-hosted runners, or { type = \"private\", names = [\"\"] } to put every workflow behind a private runner group. Override per workspace via workspaceOverrides[name].RunnerConstraints." + type = object({ + type = string + names = optional(list(string)) + }) + validation { + condition = contains(["shared", "private"], var.SGDefaultRunnerConstraints.type) + error_message = "SGDefaultRunnerConstraints.type must be \"shared\" or \"private\"." + } + validation { + condition = var.SGDefaultRunnerConstraints.type != "private" || length(coalesce(var.SGDefaultRunnerConstraints.names, [])) > 0 + error_message = "SGDefaultRunnerConstraints.names must list at least one runner group when type is \"private\"." + } +} + variable "SGDefaultSourceConfigDestKind" { default = "GIT_OTHER" description = "Choose from: GITHUB_COM, BITBUCKET_ORG, GITLAB_COM, AZURE_DEVOPS, GIT_OTHER" From 056f633c365e1ec1ddc96567889752101fc05f37 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adis=20Halilovi=C4=87?= Date: Mon, 7 Sep 2026 09:14:40 +0200 Subject: [PATCH 13/71] feat: shared libs for prompts, tfvars, TFC and SG API access Move the TFC/SG API helpers out of migrate.sh into scripts/lib/ and add the building blocks the upcoming init wizard, preflight and checklist need: interactive prompt helpers (tty or scripted answers), cached tfvars reads, paginated TFC listing (orgs/projects/workspaces) and SG lookups for integrations, runner groups, workflows and secrets. The enrich script now shares the same token resolution as the provider. Also stop turning curl connection failures into "000000" status codes. --- .gitignore | 2 + scripts/enrich_variable_sets.sh | 38 +++------ scripts/lib/prompt.sh | 121 ++++++++++++++++++++++++++++ scripts/lib/sg_api.sh | 136 ++++++++++++++++++++++++++++++++ scripts/lib/tfc_api.sh | 105 ++++++++++++++++++++++++ scripts/lib/tfvars.sh | 36 +++++++++ scripts/migrate.sh | 122 ++++------------------------ 7 files changed, 426 insertions(+), 134 deletions(-) create mode 100644 scripts/lib/prompt.sh create mode 100644 scripts/lib/sg_api.sh create mode 100644 scripts/lib/tfc_api.sh create mode 100644 scripts/lib/tfvars.sh diff --git a/.gitignore b/.gitignore index 953654c..4e5a8c2 100644 --- a/.gitignore +++ b/.gitignore @@ -16,6 +16,8 @@ eggs/ .eggs/ lib/ lib64/ +# The migrator's sourced shell libraries are not Python build output. +!scripts/lib/ parts/ sdist/ var/ diff --git a/scripts/enrich_variable_sets.sh b/scripts/enrich_variable_sets.sh index d7bbc6b..167b04a 100755 --- a/scripts/enrich_variable_sets.sh +++ b/scripts/enrich_variable_sets.sh @@ -15,6 +15,11 @@ set -uo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # shellcheck source=tools.sh source "$SCRIPT_DIR/tools.sh" +# shellcheck source=lib/tfvars.sh +source "$SCRIPT_DIR/lib/tfvars.sh" +# shellcheck source=lib/tfc_api.sh +source "$SCRIPT_DIR/lib/tfc_api.sh" +TFVARS="${TFVARS:-$SG_REPO_ROOT/transformer/terraform-cloud/terraform.tfvars}" ORG="${1:-}" shift || true @@ -23,44 +28,25 @@ if [ -z "$ORG" ] || [ "$#" -eq 0 ]; then exit 1 fi -HOST="${SG_TFC_HOSTNAME:-app.terraform.io}" -API="https://$HOST/api/v2" +TFC_HOST="${SG_TFC_HOSTNAME:-$(tfc_hostname)}" +HOST="$TFC_HOST" command -v curl >/dev/null 2>&1 || { sg_err "curl is required for variable-set enrichment" exit 1 } JQ_BIN="$(sg_resolve jq sg_ensure_jq)" -# TFC token (same sources as state export): credentials file or TFE_TOKEN. -creds="$HOME/.terraform.d/credentials.tfrc.json" -token="" -[ -f "$creds" ] && token="$("$JQ_BIN" -r --arg h "$HOST" '.credentials[$h].token // empty' "$creds" 2>/dev/null || true)" -[ -z "$token" ] && token="${TFE_TOKEN:-}" -if [ -z "$token" ]; then +# TFC token: same resolution as the tfe provider / state export (lib/tfc_api.sh). +if [ -z "$(tfc_token "$HOST")" ]; then sg_warn "no TFC token (terraform login / TFE_TOKEN); skipping variable-set enrichment" exit 0 fi WORK="$(mktemp -d)" trap 'rm -rf "$WORK"' EXIT -AUTH=(-H "Authorization: Bearer $token") -# fetch_all — GET a paginated JSON:API collection, print the merged .data array. -fetch_all() { - local path="$1" page=1 next - : >"$WORK/acc.ndjson" - while :; do - if ! curl -fsS "${AUTH[@]}" "$API/$path?page%5Bsize%5D=100&page%5Bnumber%5D=$page" >"$WORK/page.json"; then - sg_err "TFC API request failed: $path" - return 1 - fi - "$JQ_BIN" -c '.data[]?' "$WORK/page.json" >>"$WORK/acc.ndjson" - next="$("$JQ_BIN" -r '.meta.pagination."next-page" // empty' "$WORK/page.json" 2>/dev/null || true)" - [ -z "$next" ] && break - page="$next" - done - "$JQ_BIN" -s '.' "$WORK/acc.ndjson" -} +# fetch_all — paginated GET, merged .data array (lib/tfc_api.sh). +fetch_all() { tfc_get_all "$1"; } sg_log "fetching workspaces and variable sets from $HOST (org: $ORG)..." @@ -74,7 +60,7 @@ fetch_all "organizations/$ORG/workspaces" | sets_raw="$(fetch_all "organizations/$ORG/varsets")" || exit 1 echo "$sets_raw" | "$JQ_BIN" -c '.[]' | while IFS= read -r s; do sid="$(echo "$s" | "$JQ_BIN" -r '.id')" - vars="$(curl -fsS "${AUTH[@]}" "$API/varsets/$sid/relationships/vars" 2>/dev/null | + vars="$(tfc_http "varsets/$sid/relationships/vars" 2>/dev/null | "$JQ_BIN" -c '[.data[]? | {key: .attributes.key, value: (.attributes.value // ""), category: .attributes.category, sensitive: (.attributes.sensitive // false), hcl: (.attributes.hcl // false)}]' 2>/dev/null || echo '[]')" echo "$s" | "$JQ_BIN" -c --argjson vars "$vars" '{ name: .attributes.name, diff --git a/scripts/lib/prompt.sh b/scripts/lib/prompt.sh new file mode 100644 index 0000000..3ca5540 --- /dev/null +++ b/scripts/lib/prompt.sh @@ -0,0 +1,121 @@ +#!/bin/bash +# Interactive prompt helpers for the migrator (sourced; needs tools.sh). +# +# Every prompt is written to stderr and read from the terminal; the answer is +# printed on stdout so callers capture it: v="$(sg_ask "Org" "demo")". +# Non-interactive runs (SG_NONINTERACTIVE=1, or no usable TTY) get the default +# answer, and fail when a prompt has none. Tests can script answers by pointing +# SG_ANSWERS_FILE at a file with one answer per line, consumed in order. + +# sg_interactive — true when we can talk to a terminal (and were not told not to). +sg_interactive() { + [ "${SG_NONINTERACTIVE:-0}" != "1" ] || return 1 + [ -n "${SG_ANSWERS_FILE:-}" ] && return 0 + (: /dev/null +} + +# The scripted-answers file is opened once, here, on fd 9: prompts run inside +# $(...) subshells, which inherit the descriptor and advance the shared offset, +# so answers are consumed in order across calls. +if [ -n "${SG_ANSWERS_FILE:-}" ]; then + exec 9<"$SG_ANSWERS_FILE" +fi + +# _sg_read — print the prompt, read one line (tty or answers +# file). Returns 1 on EOF so callers never loop on an exhausted input. +_sg_read() { + local ans + printf '%s' "$1" >&2 + if [ -n "${SG_ANSWERS_FILE:-}" ]; then + IFS= read -r ans <&9 || { printf '\n' >&2; return 1; } + printf '%s\n' "$ans" >&2 + else + IFS= read -r ans &2; return 1; } + fi + printf '%s' "$ans" +} + +# sg_ask [default] — free-text answer (default when empty). +sg_ask() { + local q="$1" def="${2-}" ans + if ! sg_interactive; then + [ -n "$def" ] || { sg_err "'$q' has no default and the run is non-interactive"; return 1; } + printf '%s' "$def" + return 0 + fi + if [ -n "$def" ]; then + ans="$(_sg_read "$(printf '%s? %s%s %s[%s]%s: ' "$C_BOLD" "$q" "$C_RESET" "$C_DIM" "$def" "$C_RESET")")" || { sg_err "input closed while asking '$q'"; return 1; } + else + ans="$(_sg_read "$(printf '%s? %s%s: ' "$C_BOLD" "$q" "$C_RESET")")" || { sg_err "input closed while asking '$q'"; return 1; } + fi + printf '%s' "${ans:-$def}" +} + +# sg_ask_required — like sg_ask, but re-prompts until non-empty. +sg_ask_required() { + local ans + while :; do + ans="$(sg_ask "$1")" || return 1 + [ -n "$ans" ] && { printf '%s' "$ans"; return 0; } + sg_warn "a value is required" + done +} + +# sg_confirm [Y|N] — exit 0 for yes, 1 for no. Default Y unless "N". +sg_confirm() { + local q="$1" def="${2:-Y}" ans hint + if ! sg_interactive; then + [ "$def" = "Y" ] && return 0 || return 1 + fi + [ "$def" = "Y" ] && hint="Y/n" || hint="y/N" + ans="$(_sg_read "$(printf '%s? %s%s %s[%s]%s ' "$C_BOLD" "$q" "$C_RESET" "$C_DIM" "$hint" "$C_RESET")")" || { sg_err "input closed while asking '$q'"; return 1; } + ans="${ans:-$def}" + case "$ans" in y | Y | yes | YES | Yes) return 0 ;; *) return 1 ;; esac +} + +# sg_select ... — numbered menu; prints the chosen item's value. +# Items are "value" or "value|description". The first item is the default. +# SG_SELECT_OTHER=1 adds an "Other (type a value)" entry for free text. +sg_select() { + local q="$1" + shift + local -a vals descs + local i n item ans + n=0 + for item in "$@"; do + vals[n]="${item%%|*}" + case "$item" in *"|"*) descs[n]="${item#*|}" ;; *) descs[n]="" ;; esac + n=$((n + 1)) + done + [ "$n" -gt 0 ] || { sg_err "sg_select: no items for '$q'"; return 1; } + if ! sg_interactive; then + printf '%s' "${vals[0]}" + return 0 + fi + printf '%s? %s%s\n' "$C_BOLD" "$q" "$C_RESET" >&2 + for ((i = 0; i < n; i++)); do + if [ -n "${descs[i]}" ]; then + printf ' %s%2d)%s %s %s%s%s\n' "$C_CYAN" "$((i + 1))" "$C_RESET" "${vals[i]}" "$C_DIM" "${descs[i]}" "$C_RESET" >&2 + else + printf ' %s%2d)%s %s\n' "$C_CYAN" "$((i + 1))" "$C_RESET" "${vals[i]}" >&2 + fi + done + [ "${SG_SELECT_OTHER:-0}" = "1" ] && printf ' %s%2d)%s Other (type a value)\n' "$C_CYAN" "$((n + 1))" "$C_RESET" >&2 + while :; do + ans="$(_sg_read "$(printf ' %schoice%s %s[1]%s: ' "$C_BOLD" "$C_RESET" "$C_DIM" "$C_RESET")")" || { sg_err "input closed while asking '$q'"; return 1; } + ans="${ans:-1}" + if [[ "$ans" =~ ^[0-9]+$ ]] && [ "$ans" -ge 1 ] && [ "$ans" -le "$n" ]; then + printf '%s' "${vals[ans - 1]}" + return 0 + fi + if [ "${SG_SELECT_OTHER:-0}" = "1" ] && [[ "$ans" =~ ^[0-9]+$ ]] && [ "$ans" -eq "$((n + 1))" ]; then + sg_ask_required "value" + return $? + fi + # Typing an item's value verbatim also works. + for ((i = 0; i < n; i++)); do + [ "$ans" = "${vals[i]}" ] && { printf '%s' "$ans"; return 0; } + done + sg_warn "enter a number between 1 and $n" + done +} diff --git a/scripts/lib/sg_api.sh b/scripts/lib/sg_api.sh new file mode 100644 index 0000000..9937a4b --- /dev/null +++ b/scripts/lib/sg_api.sh @@ -0,0 +1,136 @@ +#!/bin/bash +# StackGuardian API access (sourced; needs tools.sh). Expects SG_API_TOKEN, +# SG_BASE_URL and ORG to be set by the caller. Bodies go to stdout, logs to +# stderr. Return-code contract shared by every call: 0 on 2xx, 22 on a +# definitive 4xx (not retryable — pair with SG_NO_RETRY_RC=22), 1 on 5xx or a +# network error (retryable). SG_HTTP_CODE holds the last status. + +sg_org_url() { printf '%s/api/v1/orgs/%s' "$SG_BASE_URL" "$ORG"; } + +# sg_http_code — prints the HTTP status only (000 on network error). +sg_http_code() { + local code + code="$(curl -sS -o /dev/null -w '%{http_code}' -X "$1" -H "Authorization: apikey $SG_API_TOKEN" "$2" 2>/dev/null)" || true + sg_norm_code "$code" +} + +# sg_norm_code — curl prints "000" and exits non-zero on connection +# errors, so never append a fallback to its output; normalize instead. +sg_norm_code() { case "$1" in [0-9][0-9][0-9]) printf '%s' "$1" ;; *) printf '000' ;; esac; } + +# _sg_api [json-body] — shared request; body on stdout. +_sg_api() { + local method="$1" url="$2" body="${3-}" tmp code + tmp="$(mktemp)" + if [ -n "$body" ]; then + code="$(curl -sS -o "$tmp" -w '%{http_code}' -X "$method" \ + -H "Authorization: apikey $SG_API_TOKEN" -H "Content-Type: application/json" \ + -d "$body" "$url" 2>/dev/null)" || true + else + code="$(curl -sS -o "$tmp" -w '%{http_code}' -X "$method" \ + -H "Authorization: apikey $SG_API_TOKEN" "$url" 2>/dev/null)" || true + fi + code="$(sg_norm_code "$code")" + # shellcheck disable=SC2034 # read by callers + SG_HTTP_CODE="$code" + case "$code" in + 2*) + cat "$tmp" + rm -f "$tmp" + return 0 + ;; + 4*) + sg_err " HTTP $code from ${url#"$SG_BASE_URL"}: $(head -c 400 "$tmp")" + if declare -F explain_api_error >/dev/null; then explain_api_error "$(head -c 2000 "$tmp")"; fi + rm -f "$tmp" + return 22 + ;; + *) + sg_warn " HTTP $code from ${url#"$SG_BASE_URL"}" + rm -f "$tmp" + return 1 + ;; + esac +} + +sg_api_get() { _sg_api GET "$1"; } +sg_api_post() { _sg_api POST "$1" "$2" >/dev/null; } +sg_api_patch() { _sg_api PATCH "$1" "$2" >/dev/null; } + +# --- workflow groups ------------------------------------------------------- + +# wfgroup_http_code -> HTTP status of GET (200 exists, 404 missing). +wfgroup_http_code() { sg_http_code GET "$(sg_org_url)/wfgrps/$1/"; } + +# wfgroup_create — create a workflow group (idempotent at call sites). +wfgroup_create() { + local body + body="$("$(sg_resolve jq sg_ensure_jq)" -nc --arg n "$1" '{ResourceName:$n, Description:"Created by stackguardian-migrator (Terraform Cloud import)"}')" + SG_NO_RETRY_RC=22 sg_retry "$RETRIES" "$RETRY_BASE" -- sg_api_post "$(sg_org_url)/wfgrps/" "$body" +} + +# --- workflows ------------------------------------------------------------- + +wf_url() { printf '%s/wfgrps/%s/wfs/%s/' "$(sg_org_url)" "$1" "$2"; } +wf_triggers_endpoint() { printf '%swebhooks/vcs_triggers/' "$(wf_url "$1" "$2")"; } + +# sg_workflow_exists — exit 0 when the workflow exists. +sg_workflow_exists() { [ "$(sg_http_code GET "$(wf_url "$1" "$2")")" = "200" ]; } + +# sg_list_workflows — ["wf-name", ...] in the group ([] on 404). +sg_list_workflows() { + local body + if body="$(sg_api_get "$(sg_org_url)/wfgrps/$1/wfs/listall/" 2>/dev/null)"; then + printf '%s' "$body" | "$(sg_resolve jq sg_ensure_jq)" -c '[(.msg // .data // [])[] | .ResourceName] | map(select(. != null))' + else + echo '[]' + fi +} + +# sg_patch_workflow — PATCH a workflow. +sg_patch_workflow() { sg_api_patch "$(wf_url "$1" "$2")" "$3"; } + +# --- integrations (connectors) -------------------------------------------- + +# sg_list_integrations — [{name, type}, ...] for the org (fails on error). +sg_list_integrations() { + local body + body="$(sg_api_get "$(sg_org_url)/integrations/listall/")" || return $? + printf '%s' "$body" | "$(sg_resolve jq sg_ensure_jq)" -c '[(.msg // .data // [])[] | {name: .ResourceName, type: .ResourceType}] | map(select(.name != null))' +} + +# sg_integration_exists — exit 0 when it exists. +sg_integration_exists() { + local n="${1#/integrations/}" + [ "$(sg_http_code GET "$(sg_org_url)/integrations/$n/")" = "200" ] +} + +# --- runner groups --------------------------------------------------------- + +# sg_runnergroup_exists — exit 0 when the runner group exists. +sg_runnergroup_exists() { [ "$(sg_http_code GET "$(sg_org_url)/runnergroups/$1/")" = "200" ]; } + +# sg_list_runnergroups — ["name", ...]; empty output (exit 1) when the API has +# no list endpoint, so callers fall back to free text. +sg_list_runnergroups() { + local body + body="$(sg_api_get "$(sg_org_url)/runnergroups/listall/" 2>/dev/null)" || return 1 + printf '%s' "$body" | "$(sg_resolve jq sg_ensure_jq)" -c '[(.msg // .data // [])[] | .ResourceName] | map(select(. != null))' +} + +# --- secrets --------------------------------------------------------------- + +# sg_secret_exists — exit 0 when a secret with that name exists. +sg_secret_exists() { + local body + body="$(sg_api_get "$(sg_org_url)/secrets/listall/" 2>/dev/null)" || return 1 + printf '%s' "$body" | "$(sg_resolve jq sg_ensure_jq)" -e --arg n "$1" '[(.msg // .data // [])[] | .ResourceName] | index($n) != null' >/dev/null +} + +# sg_create_secret [description] +sg_create_secret() { + local body + body="$("$(sg_resolve jq sg_ensure_jq)" -nc --arg n "$1" --arg v "$2" --arg d "${3:-Created by stackguardian-migrator (placeholder — set the real value)}" \ + '{ResourceName:$n, ResourceType:"SECRET", Description:$d, Value:$v}')" + SG_NO_RETRY_RC=22 sg_retry "$RETRIES" "$RETRY_BASE" -- sg_api_post "$(sg_org_url)/secrets/" "$body" +} diff --git a/scripts/lib/tfc_api.sh b/scripts/lib/tfc_api.sh new file mode 100644 index 0000000..5a9c74e --- /dev/null +++ b/scripts/lib/tfc_api.sh @@ -0,0 +1,105 @@ +#!/bin/bash +# Terraform Cloud/Enterprise API access (sourced; needs tools.sh + tfvars.sh). +# +# Token resolution mirrors the tfe provider: TFE_TOKEN, then TF_TOKEN_, +# then the `terraform login` credentials file. All helpers print JSON on stdout +# and log on stderr. + +# tfc_hostname — TFC/TFE host from terraform.tfvars (tfHostname), default app.terraform.io. +tfc_hostname() { + local h + h="$(tfvars_get '.tfHostname')" + printf '%s' "${h:-app.terraform.io}" +} + +# tfc_token — the token the tfe provider will use, or nothing. +tfc_token() { + local host="$1" var creds + if [ -n "${TFE_TOKEN:-}" ]; then printf '%s' "$TFE_TOKEN"; return; fi + var="TF_TOKEN_$(echo "$host" | sed 's/-/__/g; s/\./_/g')" + if [ -n "${!var:-}" ]; then printf '%s' "${!var}"; return; fi + creds="$HOME/.terraform.d/credentials.tfrc.json" + [ -f "$creds" ] && "$(sg_resolve jq sg_ensure_jq)" -r --arg h "$host" '.credentials[$h].token // empty' "$creds" 2>/dev/null || true +} + +# tfc_token_source — human description of where the token came from. +tfc_token_source() { + if [ -n "${TFE_TOKEN:-}" ]; then printf 'TFE_TOKEN' + elif env | grep -q '^TF_TOKEN_'; then printf 'the TF_TOKEN_* variable' + else printf "the 'terraform login' session"; fi +} + +# tfc_http [curl-args...] — GET https:///api/v2/, body on +# stdout. Exit 0 on 2xx, 22 on 4xx, 1 on 5xx/network. Sets TFC_HTTP_CODE. +tfc_http() { + local path="$1" host token tmp code + shift + host="${TFC_HOST:-$(tfc_hostname)}" + token="$(tfc_token "$host")" + tmp="$(mktemp)" + code="$(curl -sS -o "$tmp" -w '%{http_code}' -H "Authorization: Bearer $token" \ + -H "Content-Type: application/vnd.api+json" "$@" "https://$host/api/v2/$path" 2>/dev/null)" || true + case "$code" in [0-9][0-9][0-9]) ;; *) code=000 ;; esac + # shellcheck disable=SC2034 # read by callers + TFC_HTTP_CODE="$code" + case "$code" in + 2*) cat "$tmp"; rm -f "$tmp"; return 0 ;; + 4*) rm -f "$tmp"; return 22 ;; + *) rm -f "$tmp"; return 1 ;; + esac +} + +# tfc_get_all — GET a paginated JSON:API collection; prints the merged +# .data array. Fails (non-zero) if any page fails. +tfc_get_all() { + local path="$1" page=1 next acc sep body jqb + jqb="$(sg_resolve jq sg_ensure_jq)" + acc="$(mktemp)" + : >"$acc" + case "$path" in *\?*) sep='&' ;; *) sep='?' ;; esac + while :; do + if ! body="$(tfc_http "${path}${sep}page%5Bsize%5D=100&page%5Bnumber%5D=$page")"; then + sg_err "TFC API request failed (HTTP ${TFC_HTTP_CODE:-000}): $path" + rm -f "$acc" + return 1 + fi + printf '%s' "$body" | "$jqb" -c '.data[]?' >>"$acc" + next="$(printf '%s' "$body" | "$jqb" -r '.meta.pagination."next-page" // empty' 2>/dev/null || true)" + [ -z "$next" ] && break + page="$next" + done + "$jqb" -s '.' "$acc" + rm -f "$acc" +} + +# tfc_list_orgs — ["org-name", ...] the token can see. +tfc_list_orgs() { tfc_get_all "organizations" | "$(sg_resolve jq sg_ensure_jq)" -c '[.[].id]'; } + +# tfc_list_projects — [{id, name}, ...] +tfc_list_projects() { tfc_get_all "organizations/$1/projects" | "$(sg_resolve jq sg_ensure_jq)" -c '[.[] | {id: .id, name: .attributes.name}]'; } + +# tfc_list_workspaces — [{name, id, project, tags, terraform_version, execution_mode}, ...] +tfc_list_workspaces() { + tfc_get_all "organizations/$1/workspaces" | "$(sg_resolve jq sg_ensure_jq)" -c \ + '[.[] | {name: .attributes.name, id: .id, project: (.relationships.project.data.id // ""), tags: (.attributes."tag-names" // []), terraform_version: .attributes."terraform-version", execution_mode: .attributes."execution-mode"}]' +} + +# require_tfc_auth — fail fast when the tfe provider would not be able to +# authenticate: no credential at all, or one that TFC/TFE rejects (e.g. an +# expired `terraform login` session). Verified with GET /account/details. +require_tfc_auth() { + local host token + host="$(tfc_hostname)" + token="$(tfc_token "$host")" + [ -n "$token" ] || die "no Terraform Cloud/Enterprise credentials found for $host. Set TFE_TOKEN= (recommended) or run 'terraform login' before '$PROG apply'." + command -v curl >/dev/null 2>&1 || return 0 + if tfc_http "account/details" >/dev/null; then + sg_log "TFC/TFE credentials verified ($host)" + return 0 + fi + case "$TFC_HTTP_CODE" in + 401 | 403) die "Terraform Cloud/Enterprise rejected $(tfc_token_source) for $host (HTTP $TFC_HTTP_CODE). Set TFE_TOKEN= or re-run 'terraform login'." ;; + 000) die "could not reach https://$host to verify TFC/TFE credentials (network/proxy?)" ;; + *) sg_warn "unexpected HTTP $TFC_HTTP_CODE verifying TFC/TFE credentials at $host; continuing" ;; + esac +} diff --git a/scripts/lib/tfvars.sh b/scripts/lib/tfvars.sh new file mode 100644 index 0000000..d6073e6 --- /dev/null +++ b/scripts/lib/tfvars.sh @@ -0,0 +1,36 @@ +#!/bin/bash +# Read access to transformer/terraform-cloud/terraform.tfvars (sourced; needs +# tools.sh and TFVARS to be set). The file is converted once with hcl2json and +# cached for the life of the process. + +_TFVARS_JSON="" +_TFVARS_JSON_FOR="" + +# tfvars_json — the whole tfvars file as JSON (empty object when missing). +tfvars_json() { + if [ -z "$_TFVARS_JSON" ] || [ "$_TFVARS_JSON_FOR" != "$TFVARS" ]; then + if [ -f "$TFVARS" ]; then + _TFVARS_JSON="$("$(sg_resolve hcl2json sg_ensure_hcl2json)" "$TFVARS" 2>/dev/null || echo '{}')" + else + _TFVARS_JSON='{}' + fi + _TFVARS_JSON_FOR="$TFVARS" + fi + printf '%s' "$_TFVARS_JSON" +} + +# tfvars_get [default] — raw value of an expression over the tfvars +# JSON, e.g. tfvars_get '.tfOrg'. Prints the default when null/missing. +tfvars_get() { + local expr="$1" def="${2-}" v + v="$(tfvars_json | "$(sg_resolve jq sg_ensure_jq)" -r "$expr // empty" 2>/dev/null || true)" + printf '%s' "${v:-$def}" +} + +# tfvars_get_json — compact JSON value of an expression (or null). +tfvars_get_json() { + tfvars_json | "$(sg_resolve jq sg_ensure_jq)" -c "$1 // null" 2>/dev/null || echo null +} + +# tfvars_invalidate — forget the cached conversion (after writing the file). +tfvars_invalidate() { _TFVARS_JSON=""; } diff --git a/scripts/migrate.sh b/scripts/migrate.sh index 4d1f3a3..5f7439b 100755 --- a/scripts/migrate.sh +++ b/scripts/migrate.sh @@ -9,6 +9,14 @@ set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # shellcheck source=tools.sh source "$SCRIPT_DIR/tools.sh" +# shellcheck source=lib/prompt.sh +source "$SCRIPT_DIR/lib/prompt.sh" +# shellcheck source=lib/tfvars.sh +source "$SCRIPT_DIR/lib/tfvars.sh" +# shellcheck source=lib/tfc_api.sh +source "$SCRIPT_DIR/lib/tfc_api.sh" +# shellcheck source=lib/sg_api.sh +source "$SCRIPT_DIR/lib/sg_api.sh" # SCRIPT_DIR holds the sibling scripts; SG_REPO_ROOT (from tools.sh) is the repo # root used for all repo-relative paths. @@ -148,7 +156,7 @@ completion_hint() { sg_log "tab completion for this shell session: source <(./sg-migrate.sh completion $sh)" } -# --- StackGuardian API helpers (workflow groups) --------------------------- +# --- workflow-group mapping (API helpers live in lib/sg_api.sh) ------------- # group_for -> the SG workflow group for a project segment: an entry # from the optional override map, else the default tfc-. @@ -160,28 +168,6 @@ group_for() { [ -n "$override" ] && echo "$override" || echo "tfc-$seg" } -# wfgroup_http_code -> HTTP status of GET (200 exists, 404 missing). -wfgroup_http_code() { - curl -sS -o /dev/null -w '%{http_code}' \ - -H "Authorization: apikey $SG_API_TOKEN" \ - "$SG_BASE_URL/api/v1/orgs/$ORG/wfgrps/$1/" -} - -# wfgroup_create — create a workflow group (idempotent at call sites). -wfgroup_create() { - local body - body="$("$JQ_BIN" -nc --arg n "$1" '{ResourceName:$n, Description:"Created by stackguardian-migrator (Terraform Cloud import)"}')" - sg_retry "$RETRIES" "$RETRY_BASE" -- \ - curl -fsS -X POST \ - -H "Authorization: apikey $SG_API_TOKEN" -H "Content-Type: application/json" \ - -d "$body" "$SG_BASE_URL/api/v1/orgs/$ORG/wfgrps/" >/dev/null -} - -# wf_triggers_endpoint -> the VCS-triggers webhook URL for a workflow. -wf_triggers_endpoint() { - echo "$SG_BASE_URL/api/v1/orgs/$ORG/wfgrps/$1/wfs/$2/webhooks/vcs_triggers/" -} - # do_set_triggers — for each workflow in the file with a non-null # VCSTriggers block, register its VCS triggers via the dedicated webhooks # endpoint (the bulk create API silently drops VCSTriggers; this second pass is @@ -200,7 +186,7 @@ do_set_triggers() { fi wf="$("$JQ_BIN" -r --argjson i "$i" '.[$i].ResourceName' "$f")" # A workflow that failed to import has nothing to attach triggers to. - if [ "$(sg_http_code GET "$SG_BASE_URL/api/v1/orgs/$ORG/wfgrps/$grp/wfs/$wf/")" != "200" ]; then + if ! sg_workflow_exists "$grp" "$wf"; then sg_warn " $grp/$wf does not exist in SG (import failed?) — skipping triggers" rc=1 continue @@ -218,37 +204,6 @@ do_set_triggers() { return "$rc" } -# sg_http_code — prints the HTTP status (000 on network error). -sg_http_code() { - curl -sS -o /dev/null -w '%{http_code}' -X "$1" -H "Authorization: apikey $SG_API_TOKEN" "$2" 2>/dev/null || echo 000 -} - -# sg_api_post — POST to the SG API. Exit 0 on 2xx, 22 on a -# definitive 4xx (not retryable; response echoed), 1 on 5xx/network (retryable). -sg_api_post() { - local url="$1" body="$2" tmp code - tmp="$(mktemp)" - code="$(curl -sS -o "$tmp" -w '%{http_code}' -X POST \ - -H "Authorization: apikey $SG_API_TOKEN" -H "Content-Type: application/json" \ - -d "$body" "$url" 2>/dev/null || echo 000)" - case "$code" in - 2*) - rm -f "$tmp" - return 0 - ;; - 4*) - sg_err " HTTP $code from ${url#"$SG_BASE_URL"}: $(head -c 400 "$tmp")" - rm -f "$tmp" - return 22 - ;; - *) - sg_warn " HTTP $code from ${url#"$SG_BASE_URL"}" - rm -f "$tmp" - return 1 - ;; - esac -} - # set_triggers_pass — run do_set_triggers over all payload files (assumes # JQ_BIN/SG_API_TOKEN/ORG are already set up by the caller). set_triggers_pass() { @@ -294,48 +249,6 @@ cmd_clean() { sg_success "clean complete" } -# tfc_hostname — TFC/TFE host from terraform.tfvars (tfHostname), default app.terraform.io. -tfc_hostname() { - local h="" - if [ -f "$TFVARS" ]; then - h="$("$(sg_resolve hcl2json sg_ensure_hcl2json)" "$TFVARS" 2>/dev/null | "$(sg_resolve jq sg_ensure_jq)" -r '.tfHostname // empty' 2>/dev/null || true)" - fi - echo "${h:-app.terraform.io}" -} - -# tfc_token — resolve the TFC/TFE token the tfe provider will use, in -# terraform's own precedence: TFE_TOKEN, TF_TOKEN_ ('.'->'_', '-'->'__'), -# then the `terraform login` credentials file. Prints nothing if none is set. -tfc_token() { - local host="$1" var creds - if [ -n "${TFE_TOKEN:-}" ]; then echo "$TFE_TOKEN"; return; fi - var="TF_TOKEN_$(echo "$host" | sed 's/-/__/g; s/\./_/g')" - if [ -n "${!var:-}" ]; then echo "${!var}"; return; fi - creds="$HOME/.terraform.d/credentials.tfrc.json" - [ -f "$creds" ] && "$(sg_resolve jq sg_ensure_jq)" -r --arg h "$host" '.credentials[$h].token // empty' "$creds" 2>/dev/null || true -} - -# require_tfc_auth — fail fast, before the (long) apply, when the tfe provider -# would not be able to authenticate: no credential at all, or a credential -# that TFC/TFE rejects (e.g. an expired `terraform login` session). Verified -# with GET /api/v2/account/details. Without this, terraform fails later with a -# confusing "Invalid provider configuration" error. -require_tfc_auth() { - local host token code src - host="$(tfc_hostname)" - token="$(tfc_token "$host")" - if [ -n "${TFE_TOKEN:-}" ]; then src="TFE_TOKEN"; elif env | grep -q '^TF_TOKEN_'; then src="the TF_TOKEN_* variable"; else src="the 'terraform login' session (likely expired)"; fi - [ -n "$token" ] || die "no Terraform Cloud/Enterprise credentials found for $host. Set TFE_TOKEN= (recommended) or run 'terraform login' before '$PROG apply'." - command -v curl >/dev/null 2>&1 || return 0 - code="$(curl -sS -o /dev/null -w '%{http_code}' -H "Authorization: Bearer $token" "https://$host/api/v2/account/details" 2>/dev/null || echo 000)" - case "$code" in - 200) sg_log "TFC/TFE credentials verified ($host)" ;; - 401 | 403) die "Terraform Cloud/Enterprise rejected $src for $host (HTTP $code). Set TFE_TOKEN= or re-run 'terraform login'." ;; - 000) die "could not reach https://$host to verify TFC/TFE credentials (network/proxy?)" ;; - *) sg_warn "unexpected HTTP $code verifying TFC/TFE credentials at $host; continuing" ;; - esac -} - cmd_apply() { sg_step "Phase: apply (terraform)" command -v terraform >/dev/null 2>&1 || die "terraform not found on PATH" @@ -381,13 +294,10 @@ cmd_enrich() { payload_files [ "${#PF[@]}" -gt 0 ] || die "No payload files in $(sg_rel "$EXPORT_DIR") (run 'apply' first)." # Variable sets belong to the TFC org (tfOrg in terraform.tfvars), not SG_ORG. - local jqb h2j tforg tfhost - jqb="$(sg_resolve jq sg_ensure_jq)" - h2j="$(sg_resolve hcl2json sg_ensure_hcl2json)" - tforg="$("$h2j" "$TFVARS" | "$jqb" -r '.tfOrg // empty')" + local tforg + tforg="$(tfvars_get '.tfOrg')" [ -n "$tforg" ] || die "tfOrg not found in $(sg_rel "$TFVARS")" - tfhost="$("$h2j" "$TFVARS" | "$jqb" -r '.tfHostname // empty')" - SG_TFC_HOSTNAME="${tfhost:-app.terraform.io}" "$SCRIPT_DIR/enrich_variable_sets.sh" "$tforg" "${PF[@]}" + SG_TFC_HOSTNAME="$(tfc_hostname)" "$SCRIPT_DIR/enrich_variable_sets.sh" "$tforg" "${PF[@]}" } do_convert() { "$SCRIPT_DIR/convert_hcl_to_json.sh" "$1"; } @@ -581,11 +491,7 @@ cmd_import() { SGCLI_BIN="$(sg_resolve sg-cli sg_ensure_sgcli)" # Fallback Terraform version for workflows the API rejects as above the # managed ceiling: SGDefaultTerraformVersion from terraform.tfvars, else 1.5.7. - SG_DEFAULT_TF_VERSION="${SG_DEFAULT_TF_VERSION:-}" - if [ -z "$SG_DEFAULT_TF_VERSION" ] && [ -f "$TFVARS" ]; then - SG_DEFAULT_TF_VERSION="$("$(sg_resolve hcl2json sg_ensure_hcl2json)" "$TFVARS" | "$JQ_BIN" -r '.SGDefaultTerraformVersion // empty')" - fi - SG_DEFAULT_TF_VERSION="${SG_DEFAULT_TF_VERSION:-TERRAFORM-1.5.7}" + SG_DEFAULT_TF_VERSION="${SG_DEFAULT_TF_VERSION:-$(tfvars_get '.SGDefaultTerraformVersion' TERRAFORM-1.5.7)}" rm -f "$EXPORT_DIR/terraform-version-fallbacks.log" sg_log "importing ${#PF[@]} payload(s), up to $CONC in parallel (retries: $RETRIES)" From 80fb77e761d2435b4ed5d546f6a8b4434071cc98 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adis=20Halilovi=C4=87?= Date: Mon, 7 Sep 2026 09:18:23 +0200 Subject: [PATCH 14/71] feat: interactive init wizard with TFC and SG discovery 'init' now walks through the configuration instead of copying the example file: it lists the TFC organisations, projects and workspaces the token can see, the SG VCS and cloud connectors and runner groups, and derives the source kind and repo prefix from the chosen connector. Every step falls back to free text when a token is missing or an API call fails, and non-interactive runs keep the template behaviour. The generated terraform.tfvars keeps the example's order and comments; re-running the wizard prefills the current values and keeps a .bak. --- scripts/lib/tfvars.sh | 79 +++++++++++++++ scripts/lib/wizard.sh | 230 ++++++++++++++++++++++++++++++++++++++++++ scripts/migrate.sh | 24 ++++- sg-migrate.sh | 2 +- 4 files changed, 330 insertions(+), 5 deletions(-) create mode 100644 scripts/lib/wizard.sh diff --git a/scripts/lib/tfvars.sh b/scripts/lib/tfvars.sh index d6073e6..8bfc766 100644 --- a/scripts/lib/tfvars.sh +++ b/scripts/lib/tfvars.sh @@ -34,3 +34,82 @@ tfvars_get_json() { # tfvars_invalidate — forget the cached conversion (after writing the file). tfvars_invalidate() { _TFVARS_JSON=""; } + +# tfvars_write — render terraform.tfvars from W_* variables set by the +# wizard (lists/objects are passed as compact JSON, which HCL accepts). Keeps +# the same order and comments as terraform.tfvars.example so the file stays +# hand-editable afterwards. +# W_TFORG W_TFHOST W_WSNAMES_JSON W_TAGS_JSON W_IGNORE_TAGS_JSON W_EXPORT_STATE +# W_APPROVERS_JSON W_REPO_PREFIX W_VCS_INTEGRATION W_DPC_JSON W_RUNNER_JSON +# W_DEST_KIND W_TF_VERSION W_TRIGGERS +tfvars_write() { + local dest="$1" host_line="" + if [ "${W_TFHOST:-app.terraform.io}" != "app.terraform.io" ]; then + host_line="$(printf '\n# Terraform Enterprise hostname (omit for Terraform Cloud)\ntfHostname = "%s"\n' "$W_TFHOST")" + fi + cat >"$dest" <) +SGDefaultVCSAuthIntegrationID = "$W_VCS_INTEGRATION" + +# Cloud connector the workflows deploy with +SGDefaultDeploymentPlatformConfig = $W_DPC_JSON + +# Runners for every workflow: { type = "shared" } for SG-hosted runners, or +# { type = "private", names = [""] } for a private runner group. +SGDefaultRunnerConstraints = $W_RUNNER_JSON + +# Choose from: GITHUB_COM, BITBUCKET_ORG, GITLAB_COM, AZURE_DEVOPS, GIT_OTHER +SGDefaultSourceConfigDestKind = "$W_DEST_KIND" + +# SG Terraform version used when a workspace's version is not a pinned semver, or +# when the SG API rejects a pinned version as above the managed ceiling (1.5.7, +# the last MPL/FOSS release; newer versions are BSL and not bundled). +SGDefaultTerraformVersion = "$W_TF_VERSION" + +# Pre-configure VCS triggers on each workflow from the workspace's TFC settings +SGDefaultEnableVCSTriggers = $W_TRIGGERS + +# Re-pull state for every workspace on each apply (default: idempotent) +forceStateRefresh = false + +# Per-workspace overrides, keyed by workspace name. Any field set here wins over +# the SGDefault* value above, for that workspace only. See terraform.tfvars.example +# for every supported field. +# workspaceOverrides = { +# "prod-networking" = { +# RunnerConstraints = { "type" : "private", "names" : ["sg-runner"] } +# Approvers = ["lead@example.com"] +# terraformVersion = "TERRAFORM-1.5.7" +# } +# } +TFVARS + tfvars_invalidate +} diff --git a/scripts/lib/wizard.sh b/scripts/lib/wizard.sh new file mode 100644 index 0000000..5096092 --- /dev/null +++ b/scripts/lib/wizard.sh @@ -0,0 +1,230 @@ +#!/bin/bash +# Interactive 'init' wizard (sourced; needs tools.sh, prompt.sh, tfvars.sh, +# tfc_api.sh, sg_api.sh). Discovers TFC orgs/workspaces and SG integrations / +# runner groups with the tokens already in the environment and writes +# terraform.tfvars. Every step degrades to free-text entry when a token is +# missing or an API call fails, so the wizard always completes. + +# _w_default — current tfvars value (re-run) or fallback. +_w_default() { local v; v="$(tfvars_get "$1")"; printf '%s' "${v:-$2}"; } + +# _w_csv_json — "a, b" -> ["a","b"]; empty -> []. +_w_csv_json() { + local jqb + jqb="$(sg_resolve jq sg_ensure_jq)" + printf '%s' "$1" | tr ',' '\n' | sed 's/^ *//; s/ *$//' | grep -v '^$' | "$jqb" -R . | "$jqb" -sc . 2>/dev/null || echo '[]' +} + +# _w_repo_prefix_for — proposed repo URL prefix. +_w_repo_prefix_for() { + case "$1" in + GITHUB_COM) printf 'https://github.com' ;; + GITLAB_COM) printf 'https://gitlab.com' ;; + BITBUCKET_ORG) printf 'https://bitbucket.org' ;; + AZURE_DEVOPS) printf 'https://dev.azure.com' ;; + *) printf 'https://VCS_PROVIDER_DOMAIN' ;; + esac +} + +# --- step 1: Terraform Cloud ------------------------------------------------ +wizard_tfc() { + local host token orgs n ws projects wsn prn scope tags alltags + local jqb + jqb="$(sg_resolve jq sg_ensure_jq)" + sg_step "1/4 Terraform Cloud / Enterprise" + host="$(sg_ask "TFC/TFE hostname" "$(_w_default .tfHostname app.terraform.io)")" || return 1 + W_TFHOST="$host" + # shellcheck disable=SC2034 # consumed by tfc_http (lib/tfc_api.sh) + TFC_HOST="$host" + token="$(tfc_token "$host")" + W_TFC_DISCOVERY=0 + if [ -z "$token" ]; then + sg_warn "no TFC credentials found for $host (TFE_TOKEN or 'terraform login'); organisations and workspaces cannot be listed" + elif ! tfc_http "account/details" >/dev/null; then + sg_warn "TFC rejected $(tfc_token_source) for $host (HTTP $TFC_HTTP_CODE); continuing without discovery" + else + W_TFC_DISCOVERY=1 + fi + + if [ "$W_TFC_DISCOVERY" -eq 1 ] && orgs="$(tfc_list_orgs 2>/dev/null)" && [ "$(printf '%s' "$orgs" | "$jqb" 'length')" -gt 0 ]; then + n="$(printf '%s' "$orgs" | "$jqb" 'length')" + if [ "$n" -eq 1 ]; then + W_TFORG="$(printf '%s' "$orgs" | "$jqb" -r '.[0]')" + sg_log "organisation: $W_TFORG (the only one this token can see)" + else + # shellcheck disable=SC2046 + W_TFORG="$(SG_SELECT_OTHER=1 sg_select "Which TFC organisation do you want to migrate?" $(printf '%s' "$orgs" | "$jqb" -r '.[]'))" || return 1 + fi + else + W_TFORG="$(sg_ask "TFC organisation name" "$(_w_default .tfOrg '')")" || return 1 + [ -n "$W_TFORG" ] || W_TFORG="$(sg_ask_required "TFC organisation name")" || return 1 + fi + + W_WSNAMES_JSON='["*"]' + W_TAGS_JSON=null + W_IGNORE_TAGS_JSON=null + if [ "$W_TFC_DISCOVERY" -eq 1 ] && ws="$(tfc_list_workspaces "$W_TFORG" 2>/dev/null)"; then + wsn="$(printf '%s' "$ws" | "$jqb" 'length')" + projects="$(tfc_list_projects "$W_TFORG" 2>/dev/null || echo '[]')" + prn="$(printf '%s' "$projects" | "$jqb" 'length')" + sg_log "found $wsn workspace(s) in $prn project(s); each project becomes SG workflow group tfc-" + W_WS_COUNT="$wsn" + W_WS_ABOVE_CEILING="$(printf '%s' "$ws" | "$jqb" '[.[] | select((.terraform_version // "") | test("^[0-9]+\\.[0-9]+\\.[0-9]+$")) | select(((.terraform_version | split(".") | map(tonumber)) as $v | ($v[0] > 1) or ($v[0] == 1 and $v[1] > 5) or ($v[0] == 1 and $v[1] == 5 and $v[2] > 7)))] | length')" + alltags="$(printf '%s' "$ws" | "$jqb" -r '[.[].tags[]?] | unique | join(", ")')" + else + alltags="" + fi + scope="$(sg_select "Which workspaces should be migrated?" \ + "all|every workspace in the organisation" \ + "tags|only workspaces carrying certain tags" \ + "exclude|all workspaces except those carrying certain tags" \ + "names|specific workspace names")" || return 1 + case "$scope" in + tags) + [ -n "$alltags" ] && sg_dim "tags in use: $alltags" + tags="$(sg_ask_required "Tags to include (comma-separated)")" || return 1 + W_TAGS_JSON="$(_w_csv_json "$tags")" + ;; + exclude) + [ -n "$alltags" ] && sg_dim "tags in use: $alltags" + tags="$(sg_ask_required "Tags to exclude (comma-separated)")" || return 1 + W_IGNORE_TAGS_JSON="$(_w_csv_json "$tags")" + ;; + names) + tags="$(sg_ask_required "Workspace names (comma-separated, * wildcards allowed)")" || return 1 + W_WSNAMES_JSON="$(_w_csv_json "$tags")" + ;; + esac +} + +# --- step 2: StackGuardian --------------------------------------------------- +wizard_sg() { + local ints vcs cloud pick kind name runner groups jqb + jqb="$(sg_resolve jq sg_ensure_jq)" + sg_step "2/4 StackGuardian" + if [ -z "$ORG" ]; then + ORG="$(sg_ask_required "StackGuardian organisation")" || return 1 + else + sg_log "organisation: $ORG (from --org / SG_ORG)" + fi + W_SG_DISCOVERY=0 + if [ -z "${SG_API_TOKEN:-}" ]; then + sg_warn "SG_API_TOKEN is not set; connectors and runner groups cannot be listed (export it and re-run 'init' to pick from a list)" + elif ! ints="$(sg_list_integrations)"; then + case "$SG_HTTP_CODE" in + 401 | 403) sg_warn "StackGuardian rejected SG_API_TOKEN for org '$ORG' (HTTP $SG_HTTP_CODE); continuing without discovery" ;; + 404) sg_warn "StackGuardian org '$ORG' not found at $SG_BASE_URL (HTTP 404); continuing without discovery" ;; + *) sg_warn "could not list connectors (HTTP $SG_HTTP_CODE); continuing without discovery" ;; + esac + else + W_SG_DISCOVERY=1 + fi + + # VCS connector -> integration id, source kind, repo prefix. + vcs="" + [ "$W_SG_DISCOVERY" -eq 1 ] && vcs="$(printf '%s' "$ints" | "$jqb" -r '.[] | select(.type | test("^(GITHUB_COM|GITHUB_APP_CUSTOM|GITLAB_COM|BITBUCKET_ORG|AZURE_DEVOPS|GIT_OTHER)$")) | "\(.name)|\(.type)"')" + if [ -n "$vcs" ]; then + # shellcheck disable=SC2046 + pick="$(SG_SELECT_OTHER=1 sg_select "Which VCS connector should clone the repositories?" $(printf '%s\n' "$vcs" | tr '\n' ' '))" || return 1 + kind="$(printf '%s' "$ints" | "$jqb" -r --arg n "$pick" '.[] | select(.name == $n) | .type' | head -1)" + else + pick="$(sg_ask "VCS connector name (as in StackGuardian, e.g. github_com)" "$(_w_default .SGDefaultVCSAuthIntegrationID '' | sed 's#^/integrations/##')")" || return 1 + [ -n "$pick" ] || pick="$(sg_ask_required "VCS connector name")" || return 1 + kind="" + fi + W_VCS_INTEGRATION="/integrations/${pick#/integrations/}" + case "$kind" in + GITHUB_APP_CUSTOM) W_DEST_KIND=GITHUB_COM ;; + GITHUB_COM | GITLAB_COM | BITBUCKET_ORG | AZURE_DEVOPS | GIT_OTHER) W_DEST_KIND="$kind" ;; + *) W_DEST_KIND="$(sg_select "VCS provider kind" GITHUB_COM GITLAB_COM BITBUCKET_ORG AZURE_DEVOPS GIT_OTHER)" || return 1 ;; + esac + W_REPO_PREFIX="$(sg_ask "Repository URL prefix" "$(_w_default .SGDefaultIACVCSRepoPrefix "$(_w_repo_prefix_for "$W_DEST_KIND")")")" || return 1 + + # Cloud connector -> DeploymentPlatformConfig. + cloud="" + [ "$W_SG_DISCOVERY" -eq 1 ] && cloud="$(printf '%s' "$ints" | "$jqb" -r '.[] | select(.type | test("^(AWS|AZURE|GCP)_")) | "\(.name)|\(.type)"')" + if [ -n "$cloud" ]; then + # shellcheck disable=SC2046 + pick="$(SG_SELECT_OTHER=1 sg_select "Which cloud connector should the workflows deploy with?" $(printf '%s\n' "$cloud" | tr '\n' ' ') "skip|decide later (leaves a placeholder to edit)")" || return 1 + kind="$(printf '%s' "$ints" | "$jqb" -r --arg n "$pick" '.[] | select(.name == $n) | .type' | head -1)" + else + pick="$(sg_ask "Cloud connector name (as in StackGuardian; empty to decide later)" "$(_w_default .SGDefaultDeploymentPlatformConfig[0].config.integrationId '' | sed 's#^/integrations/##')")" || return 1 + kind="" + fi + if [ -z "$pick" ] || [ "$pick" = "skip" ]; then + W_DPC_JSON='[{"kind":"AWS_RBAC","config":{"integrationId":"/integrations/CHANGE_ME","profileName":"default"}}]' + W_DPC_PLACEHOLDER=1 + else + [ -n "$kind" ] || kind="$(sg_select "Connector kind" AWS_RBAC AWS_STATIC AWS_OIDC AZURE_STATIC AZURE_OIDC AZURE_MANAGED_ID_OIDC GCP_STATIC GCP_OIDC)" || return 1 + name="$(sg_ask "Profile name" "$(_w_default .SGDefaultDeploymentPlatformConfig[0].config.profileName default)")" || return 1 + W_DPC_JSON="$("$jqb" -nc --arg k "$kind" --arg i "/integrations/${pick#/integrations/}" --arg p "$name" '[{kind:$k, config:{integrationId:$i, profileName:$p}}]')" + W_DPC_PLACEHOLDER=0 + fi + + # Runner constraints. + runner="$(sg_select "Where should the workflows run?" \ + "shared|StackGuardian-hosted shared runners" \ + "private|a private runner group in your own network")" || return 1 + if [ "$runner" = "private" ]; then + groups="" + [ "$W_SG_DISCOVERY" -eq 1 ] && groups="$(sg_list_runnergroups 2>/dev/null | "$jqb" -r '.[]' 2>/dev/null || true)" + if [ -n "$groups" ]; then + # shellcheck disable=SC2046 + name="$(SG_SELECT_OTHER=1 sg_select "Which runner group?" $(printf '%s\n' "$groups" | tr '\n' ' '))" || return 1 + else + name="$(sg_ask_required "Runner group name")" || return 1 + fi + if [ "$W_SG_DISCOVERY" -eq 1 ] && ! sg_runnergroup_exists "$name"; then + sg_warn "runner group '$name' was not found in org '$ORG' — preflight will fail until it exists" + fi + W_RUNNER_JSON="$("$jqb" -nc --arg n "$name" '{type:"private", names:[$n]}')" + else + W_RUNNER_JSON='{"type":"shared"}' + fi +} + +# --- step 3: policy ------------------------------------------------------------ +wizard_policy() { + local approvers + sg_step "3/4 Workflow defaults" + approvers="$(sg_ask "Approver emails for plans (comma-separated, empty for none)" "$(tfvars_get_json .SGDefaultWfApprovers | "$(sg_resolve jq sg_ensure_jq)" -r 'if . == null then "" else join(", ") end')")" || return 1 + W_APPROVERS_JSON="$(_w_csv_json "$approvers")" + if sg_confirm "Export Terraform state for each workspace?" "$([ "$(_w_default .exportStateFiles true)" = "false" ] && echo N || echo Y)"; then W_EXPORT_STATE=true; else W_EXPORT_STATE=false; fi + if sg_confirm "Pre-configure VCS triggers (push / pull-request runs) from the TFC settings?" "$([ "$(_w_default .SGDefaultEnableVCSTriggers true)" = "false" ] && echo N || echo Y)"; then W_TRIGGERS=true; else W_TRIGGERS=false; fi + sg_dim "StackGuardian bundles managed Terraform only up to 1.5.7 (last MPL/FOSS release);" + sg_dim "workspaces pinned above it are imported with the fallback version below." + while :; do + W_TF_VERSION="$(sg_ask "Fallback Terraform version" "$(_w_default .SGDefaultTerraformVersion TERRAFORM-1.5.7)")" || return 1 + [[ "$W_TF_VERSION" =~ ^TERRAFORM-[0-9]+\.[0-9]+\.[0-9]+$ ]] && break + sg_warn "use the SG format, e.g. TERRAFORM-1.5.7" + done +} + +# --- step 4: review + write ---------------------------------------------------- +wizard_review() { + sg_step "4/4 Review" + row() { printf ' %s%-28s%s %s\n' "$C_BOLD" "$1" "$C_RESET" "$2" >&2; } + row "TFC host / org" "$W_TFHOST / $W_TFORG" + row "Workspaces" "$W_WSNAMES_JSON tags=$W_TAGS_JSON ignore=$W_IGNORE_TAGS_JSON${W_WS_COUNT:+ ($W_WS_COUNT found)}" + row "SG org" "$ORG" + row "VCS connector" "$W_VCS_INTEGRATION ($W_DEST_KIND, $W_REPO_PREFIX)" + row "Cloud connector" "$W_DPC_JSON" + row "Runners" "$W_RUNNER_JSON" + row "Approvers" "$W_APPROVERS_JSON" + row "State export / triggers" "$W_EXPORT_STATE / $W_TRIGGERS" + row "Fallback Terraform" "$W_TF_VERSION${W_WS_ABOVE_CEILING:+ ($W_WS_ABOVE_CEILING workspace(s) above 1.5.7 will use it)}" + [ "${W_DPC_PLACEHOLDER:-0}" -eq 1 ] && sg_warn "cloud connector left as a placeholder — edit SGDefaultDeploymentPlatformConfig in $(sg_rel "$TFVARS") before 'apply'" + sg_confirm "Write $(sg_rel "$TFVARS")?" Y +} + +# wizard_run — the whole flow; returns non-zero when aborted. +wizard_run() { + wizard_tfc && wizard_sg && wizard_policy || { sg_err "init aborted"; return 1; } + wizard_review || { sg_log "nothing written"; return 1; } + if [ -f "$TFVARS" ]; then + cp "$TFVARS" "$TFVARS.bak" + sg_log "previous file kept as $(sg_rel "$TFVARS.bak")" + fi + tfvars_write "$TFVARS" + sg_success "wrote $(sg_rel "$TFVARS")" +} diff --git a/scripts/migrate.sh b/scripts/migrate.sh index 5f7439b..5cb69a7 100755 --- a/scripts/migrate.sh +++ b/scripts/migrate.sh @@ -17,6 +17,8 @@ source "$SCRIPT_DIR/lib/tfvars.sh" source "$SCRIPT_DIR/lib/tfc_api.sh" # shellcheck source=lib/sg_api.sh source "$SCRIPT_DIR/lib/sg_api.sh" +# shellcheck source=lib/wizard.sh +source "$SCRIPT_DIR/lib/wizard.sh" # SCRIPT_DIR holds the sibling scripts; SG_REPO_ROOT (from tools.sh) is the repo # root used for all repo-relative paths. @@ -136,13 +138,27 @@ seg_of() { cmd_init() { sg_step "Phase: init" mkdir -p "$SG_REPO_ROOT/.sg" "$SG_CACHE_BIN" - if [ ! -f "$TFVARS" ]; then - cp "$TRANSFORMER_DIR/terraform.tfvars.example" "$TFVARS" - sg_log "created $(sg_rel "$TFVARS") — edit it before 'apply'" + if ! sg_interactive; then + # Non-interactive (CI, no TTY, -y): fall back to the template. + if [ ! -f "$TFVARS" ]; then + cp "$TRANSFORMER_DIR/terraform.tfvars.example" "$TFVARS" + sg_log "created $(sg_rel "$TFVARS") from the template — edit it before 'apply' (run 'init' in a terminal for the guided setup)" + else + sg_log "$(sg_rel "$TFVARS") already exists" + fi else - sg_log "$(sg_rel "$TFVARS") already exists" + if [ -f "$TFVARS" ]; then + case "$(sg_select "$(sg_rel "$TFVARS") already exists" "keep|leave it unchanged" "rerun|run the wizard again (current values become the defaults; a .bak copy is kept)")" in + keep) sg_log "keeping $(sg_rel "$TFVARS")" ;; + rerun) wizard_run || return 1 ;; + esac + else + sg_log "answer a few questions to generate $(sg_rel "$TFVARS"); tokens are read from the environment and never written to disk" + wizard_run || return 1 + fi fi sg_log "workflow groups are created automatically as tfc-; no mapping needed" + sg_log "next: ./sg-migrate.sh all" completion_hint sg_success "init complete" } diff --git a/sg-migrate.sh b/sg-migrate.sh index b43c84e..7d4a1f1 100755 --- a/sg-migrate.sh +++ b/sg-migrate.sh @@ -54,7 +54,7 @@ fi DOCKER_ARGS=(--rm -i -v "$SCRIPT_DIR:/app" -w /app -e SG_API_TOKEN -e SG_ORG -e SG_BASE_URL -e SG_CONCURRENCY -e SG_RETRIES -e SG_TF_PARALLELISM - -e TFE_TOKEN -e SG_PROG) + -e TFE_TOKEN -e SG_PROG -e SG_NONINTERACTIVE -e SG_UI_URL) # Interactive TTY only when attached to one (so the confirmation prompt works, # but CI/non-tty invocations still run — use -y there). From aeb8303eb6779f55f22d9ab95564948337ba13bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adis=20Halilovi=C4=87?= Date: Mon, 7 Sep 2026 09:21:19 +0200 Subject: [PATCH 15/71] feat: preflight checks before apply and import Verify up front what used to fail minutes later inside terraform or as API 400s: TFC token and org, that the workspace selection matches anything, SG token and org, every connector / secret / runner group referenced in terraform.tfvars (defaults and workspaceOverrides), SGDefaultSourceConfigDestKind and SGDefaultTerraformVersion format, the exportPath vs export-dir mismatch, and override keys that match no workspace. Runs automatically for apply, import and all; also available as 'preflight'; --skip-preflight bypasses it. --- scripts/lib/preflight.sh | 221 +++++++++++++++++++++++++++++++++++++++ scripts/migrate.sh | 38 +++++-- sg-migrate.sh | 2 +- 3 files changed, 249 insertions(+), 12 deletions(-) create mode 100644 scripts/lib/preflight.sh diff --git a/scripts/lib/preflight.sh b/scripts/lib/preflight.sh new file mode 100644 index 0000000..c03e0cc --- /dev/null +++ b/scripts/lib/preflight.sh @@ -0,0 +1,221 @@ +#!/bin/bash +# Preflight checks (sourced; needs tools.sh, tfvars.sh, tfc_api.sh, sg_api.sh). +# +# Verifies, before the long terraform apply or a bulk import, that the tokens +# work and that everything terraform.tfvars refers to actually exists in TFC +# and StackGuardian. Prints one line per check (✓ ok, ! warning, ✗ failure) and +# fails the run when any check fails. Skipped with --skip-preflight. + +PF_FAIL=0 +PF_WARN=0 +pf_ok() { printf ' %s✓%s %s\n' "$C_GREEN" "$C_RESET" "$*" >&2; } +pf_warn() { printf ' %s!%s %s\n' "$C_YELLOW" "$C_RESET" "$*" >&2; PF_WARN=$((PF_WARN + 1)); } +pf_fail() { printf ' %s✗%s %s\n' "$C_RED$C_BOLD" "$C_RESET" "$*" >&2; PF_FAIL=$((PF_FAIL + 1)); } + +# --- Terraform Cloud ----------------------------------------------------------- +preflight_tfc() { + local host token org body n names_json tags_json ignore_json jqb + jqb="$(sg_resolve jq sg_ensure_jq)" + host="$(tfc_hostname)" + org="$(tfvars_get .tfOrg)" + [ -n "$org" ] || { pf_fail "tfOrg is not set in $(sg_rel "$TFVARS")"; return 0; } + token="$(tfc_token "$host")" + if [ -z "$token" ]; then + pf_fail "no TFC/TFE credentials for $host — set TFE_TOKEN (recommended) or run 'terraform login'" + return 0 + fi + if ! tfc_http "account/details" >/dev/null; then + case "$TFC_HTTP_CODE" in + 401 | 403) pf_fail "TFC rejected $(tfc_token_source) for $host (HTTP $TFC_HTTP_CODE) — set TFE_TOKEN or re-run 'terraform login'" ;; + 000) pf_fail "cannot reach https://$host (network/proxy?)" ;; + *) pf_warn "unexpected HTTP $TFC_HTTP_CODE from $host while verifying credentials" ;; + esac + return 0 + fi + pf_ok "TFC credentials valid ($host, via $(tfc_token_source))" + if tfc_http "organizations/$org" >/dev/null; then + pf_ok "TFC organisation '$org' accessible" + else + case "$TFC_HTTP_CODE" in + 404 | 403) pf_fail "TFC organisation '$org' not found or not accessible with this token (HTTP $TFC_HTTP_CODE) — check tfOrg" ;; + *) pf_warn "could not verify TFC organisation '$org' (HTTP $TFC_HTTP_CODE)" ;; + esac + return 0 + fi + # Workspace selection: mirror the module's filters (names glob, include/exclude tags). + if body="$(tfc_list_workspaces "$org" 2>/dev/null)"; then + names_json="$(tfvars_get_json .workspacenames)" + tags_json="$(tfvars_get_json .tfWorkspaceTags)" + ignore_json="$(tfvars_get_json .tfWorkspaceIgnoreTags)" + n="$(printf '%s' "$body" | "$jqb" --argjson names "${names_json:-null}" --argjson tags "${tags_json:-null}" --argjson ignore "${ignore_json:-null}" ' + def glob($p): ("^" + ($p | gsub("\\*"; ".*")) + "$"); + [ .[] + | select(($names == null) or ($names == ["*"]) or ([.name] | inside([]) | not) and ([$names[] as $p | (.name | test(glob($p)))] | any)) + | select(($tags == null) or (($tags | length) == 0) or ([.tags[]?] | inside($tags) | not) or (([.tags[]?] | map(select(. as $t | $tags | index($t) != null)) | length) > 0)) + | select(($ignore == null) or (($ignore | length) == 0) or (([.tags[]?] | map(select(. as $t | $ignore | index($t) != null)) | length) == 0)) + ] | length')" + if [ "$n" -gt 0 ]; then + pf_ok "$n workspace(s) match the selection (of $(printf '%s' "$body" | "$jqb" 'length') in the org)" + else + pf_warn "no workspace matches workspacenames/tfWorkspaceTags/tfWorkspaceIgnoreTags — apply would export nothing" + fi + PF_TFC_WORKSPACES="$body" + else + pf_warn "could not list workspaces for '$org' (HTTP $TFC_HTTP_CODE)" + fi +} + +# --- StackGuardian ------------------------------------------------------------ +# preflight_sg +preflight_sg() { + local required="$1" ints names id n jqb rg + jqb="$(sg_resolve jq sg_ensure_jq)" + if [ -z "${SG_API_TOKEN:-}" ] || [ -z "$ORG" ]; then + if [ "$required" -eq 1 ]; then + [ -n "${SG_API_TOKEN:-}" ] || pf_fail "SG_API_TOKEN is not set" + [ -n "$ORG" ] || pf_fail "StackGuardian org not set (use --org or SG_ORG)" + else + pf_warn "SG_API_TOKEN/SG_ORG not set — StackGuardian references are not verified (they will be at import)" + fi + return 0 + fi + if ! ints="$(sg_list_integrations 2>/dev/null)"; then + case "$SG_HTTP_CODE" in + 401 | 403) pf_fail "StackGuardian rejected SG_API_TOKEN for org '$ORG' (HTTP $SG_HTTP_CODE)" ;; + 404) pf_fail "StackGuardian org '$ORG' not found at $SG_BASE_URL (HTTP 404) — check --org / SG_ORG" ;; + 000) pf_fail "cannot reach $SG_BASE_URL (network/proxy? SG_BASE_URL?)" ;; + *) pf_fail "could not list StackGuardian connectors (HTTP $SG_HTTP_CODE)" ;; + esac + return 0 + fi + pf_ok "StackGuardian credentials valid (org '$ORG', $SG_BASE_URL)" + + # Every integration id referenced anywhere in tfvars must exist. + names="$(printf '%s' "$ints" | "$jqb" -c '[.[].name]')" + while IFS= read -r id; do + [ -n "$id" ] || continue + case "$id" in + *CHANGE_ME* | *INTEGRATION_ID*) pf_fail "placeholder connector id '$id' in $(sg_rel "$TFVARS") — run '$PROG init' or edit the file" ;; + /integrations/*) + if printf '%s' "$names" | "$jqb" -e --arg n "${id#/integrations/}" 'index($n) != null' >/dev/null; then + pf_ok "connector $id exists" + else + pf_fail "connector $id not found in org '$ORG' (available: $(printf '%s' "$names" | "$jqb" -r 'join(", ")'))" + fi + ;; + /secrets/*) + if sg_secret_exists "${id#/secrets/}"; then pf_ok "secret $id exists"; else pf_fail "secret $id not found in org '$ORG'"; fi + ;; + *) pf_warn "unrecognised integration id '$id' (expected /integrations/ or /secrets/)" ;; + esac + done < <(tfvars_json | "$jqb" -r ' + [ .SGDefaultVCSAuthIntegrationID, + (.SGDefaultDeploymentPlatformConfig // [])[]?.config.integrationId, + ((.workspaceOverrides // {}) | to_entries[]? | .value | (.vcsAuthIntegrationID, ((.DeploymentPlatformConfig // [])[]?.config.integrationId))) ] + | map(select(. != null and . != "")) | unique | .[]') + + # Every private runner group must exist. + while IFS= read -r rg; do + [ -n "$rg" ] || continue + if sg_runnergroup_exists "$rg"; then pf_ok "runner group '$rg' exists"; else pf_fail "runner group '$rg' not found in org '$ORG' — check SGDefaultRunnerConstraints / workspaceOverrides"; fi + done < <(tfvars_json | "$jqb" -r ' + [ (.SGDefaultRunnerConstraints // {} | select(.type == "private") | .names[]?), + ((.workspaceOverrides // {}) | to_entries[]? | .value.RunnerConstraints // {} | select(.type == "private") | .names[]?) ] + | unique | .[]') + n="$(tfvars_json | "$jqb" -r '(.SGDefaultRunnerConstraints // {}).type // "shared"')" + [ "$n" = "shared" ] && pf_ok "runners: StackGuardian shared runners" +} + +# --- static config -------------------------------------------------------------- +preflight_config() { + local v ceiling="1.5.7" export_path want jqb + jqb="$(sg_resolve jq sg_ensure_jq)" + v="$(tfvars_get .SGDefaultSourceConfigDestKind GIT_OTHER)" + case "$v" in + GITHUB_COM | GITHUB_APP_CUSTOM | GIT_OTHER | INLINE | BITBUCKET_ORG | GITLAB_COM | AZURE_DEVOPS) pf_ok "SGDefaultSourceConfigDestKind = $v" ;; + *) pf_fail "SGDefaultSourceConfigDestKind '$v' is not one of GITHUB_COM, GITLAB_COM, BITBUCKET_ORG, AZURE_DEVOPS, GIT_OTHER" ;; + esac + v="$(tfvars_get .SGDefaultTerraformVersion TERRAFORM-1.5.7)" + if [[ "$v" =~ ^TERRAFORM-([0-9]+)\.([0-9]+)\.([0-9]+)$ ]]; then + if [ "$(printf '%03d%03d%03d' "${BASH_REMATCH[1]}" "${BASH_REMATCH[2]}" "${BASH_REMATCH[3]}")" -gt "001005007" ]; then + pf_warn "SGDefaultTerraformVersion $v is above SG's managed ceiling ($ceiling); fallbacks would fail too unless a private runner ships that binary" + else + pf_ok "SGDefaultTerraformVersion = $v" + fi + else + pf_fail "SGDefaultTerraformVersion '$v' must look like TERRAFORM-1.5.7" + fi + export_path="$(tfvars_get .exportPath export)" + case "$export_path" in /*) want="$export_path" ;; *) want="$SG_REPO_ROOT/$export_path" ;; esac + if [ "$(cd "$(dirname "$want")" 2>/dev/null && pwd)/$(basename "$want")" = "$(cd "$(dirname "$EXPORT_DIR")" 2>/dev/null && pwd)/$(basename "$EXPORT_DIR")" ]; then + pf_ok "export directory: $(sg_rel "$EXPORT_DIR")" + else + pf_warn "exportPath in tfvars ($export_path) differs from the orchestrator's export dir ($(sg_rel "$EXPORT_DIR")) — payloads would be written where later phases don't look" + fi + if [ -n "${PF_TFC_WORKSPACES:-}" ]; then + while IFS= read -r v; do + [ -n "$v" ] || continue + if printf '%s' "$PF_TFC_WORKSPACES" | "$jqb" -e --arg n "$v" '[.[].name] | index($n) != null' >/dev/null; then + pf_ok "workspaceOverrides['$v'] matches a workspace" + else + pf_warn "workspaceOverrides['$v'] does not match any workspace in the org (typo?)" + fi + done < <(tfvars_json | "$jqb" -r '(.workspaceOverrides // {}) | keys[]') + fi +} + +# --- import inputs -------------------------------------------------------------- +preflight_import_inputs() { + payload_files + if [ "${#PF[@]}" -gt 0 ]; then + pf_ok "${#PF[@]} payload file(s) in $(sg_rel "$EXPORT_DIR")" + else + pf_fail "no payload files in $(sg_rel "$EXPORT_DIR") — run '$PROG apply' first" + fi + if sg_resolve sg-cli sg_ensure_sgcli >/dev/null 2>&1; then pf_ok "sg-cli available"; else pf_fail "sg-cli not found and could not be downloaded"; fi +} + +# preflight_run — run the checks for a context; dies on +# failures. Runs at most once per process (PREFLIGHT_DONE). +preflight_run() { + local ctx="$1" + [ "${SKIP_PREFLIGHT:-0}" -eq 1 ] && { sg_warn "preflight skipped (--skip-preflight)"; return 0; } + [ "${PREFLIGHT_DONE:-0}" -eq 1 ] && return 0 + sg_step "Preflight ($ctx)" + [ -f "$TFVARS" ] || die "Missing $(sg_rel "$TFVARS"). Run: $PROG init" + PF_FAIL=0 + PF_WARN=0 + PF_TFC_WORKSPACES="" + case "$ctx" in + apply) + preflight_tfc + preflight_sg 0 + preflight_config + ;; + import) + preflight_sg 1 + preflight_config + preflight_import_inputs + ;; + all) + preflight_tfc + preflight_sg 1 + preflight_config + ;; + *) die "preflight: unknown context '$ctx'" ;; + esac + PREFLIGHT_DONE=1 + if [ "$PF_FAIL" -gt 0 ]; then + die "preflight found $PF_FAIL problem(s); fix them (or re-run '$PROG init') and try again. Use --skip-preflight to bypass." + fi + if [ "$PF_WARN" -gt 0 ]; then + sg_warn "preflight passed with $PF_WARN warning(s)" + else + sg_success "preflight passed" + fi +} + +cmd_preflight() { + PREFLIGHT_DONE=0 + preflight_run all +} diff --git a/scripts/migrate.sh b/scripts/migrate.sh index 5cb69a7..3c90ef1 100755 --- a/scripts/migrate.sh +++ b/scripts/migrate.sh @@ -19,6 +19,8 @@ source "$SCRIPT_DIR/lib/tfc_api.sh" source "$SCRIPT_DIR/lib/sg_api.sh" # shellcheck source=lib/wizard.sh source "$SCRIPT_DIR/lib/wizard.sh" +# shellcheck source=lib/preflight.sh +source "$SCRIPT_DIR/lib/preflight.sh" # SCRIPT_DIR holds the sibling scripts; SG_REPO_ROOT (from tools.sh) is the repo # root used for all repo-relative paths. @@ -34,6 +36,10 @@ PURGE=0 CREATE_GROUPS=1 ENRICH_VARSETS=1 VCS_TRIGGERS=1 +# shellcheck disable=SC2034 # consumed by lib/preflight.sh +SKIP_PREFLIGHT=0 +# shellcheck disable=SC2034 +PREFLIGHT_DONE=0 VERBOSE="${SG_VERBOSE:-0}" CONC="${SG_CONCURRENCY:-4}" TF_PARALLELISM="${SG_TF_PARALLELISM:-20}" @@ -46,7 +52,9 @@ usage() { Usage: $PROG [options] Commands: - init Create terraform.tfvars from the template + init Guided setup: discovers TFC/SG resources and writes terraform.tfvars + preflight Verify tokens and every connector/runner/org referenced in tfvars + (runs automatically before apply, import and all) apply Run the transformer (terraform apply) to generate payloads + state enrich Merge TFC Variable Set variables into the payloads (via the TFC API) convert Convert HCL-string variables to JSON in each payload (parallel) @@ -72,6 +80,7 @@ Options: --no-create-groups Do not create missing workflow groups; require them to exist --no-variable-sets Skip merging TFC Variable Set variables in the 'all' flow --no-vcs-triggers Skip registering VCS triggers after import + --skip-preflight Skip the preflight checks (not recommended) --all With 'clean': also remove config (terraform.tfvars, mapping, .sg) -v, --verbose Show full terraform/tool output (default: concise) -y, --yes Skip the import confirmation prompt @@ -268,8 +277,8 @@ cmd_clean() { cmd_apply() { sg_step "Phase: apply (terraform)" command -v terraform >/dev/null 2>&1 || die "terraform not found on PATH" - [ -f "$TFVARS" ] || die "Missing $(sg_rel "$TFVARS"). Run: $PROG init (then edit it)." - require_tfc_auth + [ -f "$TFVARS" ] || die "Missing $(sg_rel "$TFVARS"). Run: $PROG init" + preflight_run apply # State export (TFC API) calls curl + jq from terraform's local-exec; make sure # both are on PATH for the apply (jq from cache if not already installed). command -v curl >/dev/null 2>&1 || die "curl is required for state export" @@ -445,6 +454,7 @@ cmd_import() { # so the sg-cli child process inherits the same target. export SG_API_TOKEN SG_BASE_URL JQ_BIN="$(sg_resolve jq sg_ensure_jq)" + preflight_run import payload_files [ "${#PF[@]}" -gt 0 ] || die "No payload files in $(sg_rel "$EXPORT_DIR")." @@ -530,8 +540,8 @@ cmd_import() { # Single source of truth for shell completion (keep in sync with the parser below # and the host-only flags in sg-migrate.sh). -SG_COMMANDS="init apply enrich convert validate import triggers all clean completion" -SG_OPTIONS="--org --export-dir --mapping --concurrency --no-create-groups --no-variable-sets --no-vcs-triggers --all -v --verbose -y --yes -h --help --native --local --build" +SG_COMMANDS="init preflight apply enrich convert validate import triggers all clean completion" +SG_OPTIONS="--org --export-dir --mapping --concurrency --no-create-groups --no-variable-sets --no-vcs-triggers --skip-preflight --all -v --verbose -y --yes -h --help --native --local --build" # cmd_completion — print a completion script for sg-migrate.sh / # migrate.sh to stdout. Both shells fall back to the basename when the command @@ -572,7 +582,8 @@ BASH _sg_migrate() { local -a cmds cmds=( - 'init:Create terraform.tfvars from the template' + 'init:Guided setup — generates terraform.tfvars' + 'preflight:Verify tokens and every id in terraform.tfvars before running' 'apply:Run the transformer (terraform apply)' 'enrich:Merge TFC Variable Set variables into the payloads' 'convert:Convert HCL-string variables to JSON' @@ -591,6 +602,7 @@ _sg_migrate() { '--no-create-groups[Require workflow groups to pre-exist]' \\ '--no-variable-sets[Skip merging TFC Variable Sets]' \\ '--no-vcs-triggers[Skip registering VCS triggers after import]' \\ + '--skip-preflight[Skip the preflight checks]' \\ '--all[With clean: also remove config]' \\ '(-v --verbose)'{-v,--verbose}'[Show full terraform/tool output]' \\ '(-y --yes)'{-y,--yes}'[Skip the import confirmation prompt]' \\ @@ -638,13 +650,14 @@ while [ $# -gt 0 ]; do --no-create-groups) CREATE_GROUPS=0 ;; --no-variable-sets) ENRICH_VARSETS=0 ;; --no-vcs-triggers) VCS_TRIGGERS=0 ;; + --skip-preflight) export SKIP_PREFLIGHT=1 ;; -v | --verbose) VERBOSE=1 ;; --all) PURGE=1 ;; -h | --help) usage exit 0 ;; - init | apply | enrich | convert | validate | import | triggers | all | clean) CMD="$1" ;; + init | apply | enrich | convert | validate | import | triggers | all | clean | preflight) CMD="$1" ;; completion) cmd_completion "${2:-}" exit 0 @@ -673,15 +686,18 @@ convert) cmd_convert ;; validate) cmd_validate ;; import) cmd_import ;; triggers) cmd_triggers ;; +preflight) + export SG_API_TOKEN SG_BASE_URL + cmd_preflight + ;; all) if [ ! -f "$TFVARS" ]; then cmd_init die "Edit $(sg_rel "$TFVARS"), then re-run '$PROG all'." fi - # Fail fast on prerequisites before the (long) apply. - require_tfc_auth - [ -n "${SG_API_TOKEN:-}" ] || die "SG_API_TOKEN is not set (needed for import)." - [ -n "$ORG" ] || die "StackGuardian org not set (use --org or SG_ORG)." + # Fail fast on everything the whole pipeline needs, before the (long) apply. + export SG_API_TOKEN SG_BASE_URL + preflight_run all cmd_apply if [ "$ENRICH_VARSETS" -eq 1 ]; then cmd_enrich; fi cmd_convert diff --git a/sg-migrate.sh b/sg-migrate.sh index 7d4a1f1..5a56ba4 100755 --- a/sg-migrate.sh +++ b/sg-migrate.sh @@ -33,7 +33,7 @@ HAS_CMD=0 for a in ${ARGS[@]+"${ARGS[@]}"}; do case "$a" in clean | completion | -h | --help) NATIVE=1; HAS_CMD=1 ;; - init | apply | enrich | convert | validate | import | triggers | all) HAS_CMD=1 ;; + init | preflight | apply | enrich | convert | validate | import | triggers | all) HAS_CMD=1 ;; esac done [ "$HAS_CMD" -eq 1 ] || NATIVE=1 From b39c4d7d4736053719098647805c8da03b70152e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adis=20Halilovi=C4=87?= Date: Mon, 7 Sep 2026 09:24:32 +0200 Subject: [PATCH 16/71] feat: resumable runs and project/workspace filters 'all' now records each phase's inputs in .sg/state.json and skips phases whose inputs have not changed, so a failed run resumes instead of re-running terraform apply. Import records per payload file which workflows landed and which failed; unchanged, fully imported files are skipped and files with failures are re-imported (sg-cli updates existing workflows in place). --fresh discards the state. --project and --workspace restrict every phase to a subset; apply maps --workspace to the workspacenames variable. --- scripts/lib/preflight.sh | 6 +- scripts/lib/state.sh | 65 +++++++++++++++++ scripts/migrate.sh | 146 +++++++++++++++++++++++++++++++++++---- 3 files changed, 203 insertions(+), 14 deletions(-) create mode 100644 scripts/lib/state.sh diff --git a/scripts/lib/preflight.sh b/scripts/lib/preflight.sh index c03e0cc..c0bb2ad 100644 --- a/scripts/lib/preflight.sh +++ b/scripts/lib/preflight.sh @@ -63,6 +63,7 @@ preflight_tfc() { else pf_warn "could not list workspaces for '$org' (HTTP $TFC_HTTP_CODE)" fi + return 0 } # --- StackGuardian ------------------------------------------------------------ @@ -123,7 +124,8 @@ preflight_sg() { ((.workspaceOverrides // {}) | to_entries[]? | .value.RunnerConstraints // {} | select(.type == "private") | .names[]?) ] | unique | .[]') n="$(tfvars_json | "$jqb" -r '(.SGDefaultRunnerConstraints // {}).type // "shared"')" - [ "$n" = "shared" ] && pf_ok "runners: StackGuardian shared runners" + if [ "$n" = "shared" ]; then pf_ok "runners: StackGuardian shared runners"; fi + return 0 } # --- static config -------------------------------------------------------------- @@ -162,6 +164,7 @@ preflight_config() { fi done < <(tfvars_json | "$jqb" -r '(.workspaceOverrides // {}) | keys[]') fi + return 0 } # --- import inputs -------------------------------------------------------------- @@ -173,6 +176,7 @@ preflight_import_inputs() { pf_fail "no payload files in $(sg_rel "$EXPORT_DIR") — run '$PROG apply' first" fi if sg_resolve sg-cli sg_ensure_sgcli >/dev/null 2>&1; then pf_ok "sg-cli available"; else pf_fail "sg-cli not found and could not be downloaded"; fi + return 0 } # preflight_run — run the checks for a context; dies on diff --git a/scripts/lib/state.sh b/scripts/lib/state.sh new file mode 100644 index 0000000..c69040d --- /dev/null +++ b/scripts/lib/state.sh @@ -0,0 +1,65 @@ +#!/bin/bash +# Run state for resumable 'all' runs (sourced; needs tools.sh). +# +# .sg/state.json records, per phase, a hash of the inputs it last ran with and +# when. 'all' skips a phase whose inputs are unchanged; --fresh wipes the file. +# Import results are recorded per payload file (hash, imported/failed +# workflows) so a re-run only touches files that changed or had failures, and +# the post-import checklist knows what actually landed. +# +# Layout: { "phases": { "": {"at": iso, "input_sha": sha} }, +# "import": { "": {"at": iso, "payload_sha": sha, "group": g, +# "imported": [...], "failed": [...]} } } + +STATE_FILE="${SG_STATE_FILE:-$SG_REPO_ROOT/.sg/state.json}" + +# sg_sha — short sha256 of a string. +sg_sha() { + if command -v shasum >/dev/null 2>&1; then printf '%s' "$1" | shasum -a 256 | cut -c1-16 + else printf '%s' "$1" | sha256sum | cut -c1-16; fi +} + +# sg_sha_files ... — sha256 over the contents of the given files (sorted). +sg_sha_files() { + local f + { + for f in "$@"; do [ -f "$f" ] && cat "$f"; done + } | if command -v shasum >/dev/null 2>&1; then shasum -a 256; else sha256sum; fi | cut -c1-16 +} + +state_read() { [ -f "$STATE_FILE" ] && cat "$STATE_FILE" || echo '{}'; } + +# state_update [jq args...] — apply a filter to the state document. +state_update() { + local filter="$1" tmp + shift + mkdir -p "$(dirname "$STATE_FILE")" + tmp="$(mktemp)" + state_read | "$(sg_resolve jq sg_ensure_jq)" "$@" "$filter" >"$tmp" && mv -f "$tmp" "$STATE_FILE" +} + +state_now() { date -u +%Y-%m-%dT%H:%M:%SZ; } + +# state_phase_done — exit 0 when the phase last completed +# with exactly these inputs. Prints the completion time on stdout. +state_phase_done() { + local at + at="$(state_read | "$(sg_resolve jq sg_ensure_jq)" -r --arg p "$1" --arg s "$2" '.phases[$p] | select(.input_sha == $s) | .at // empty')" + [ -n "$at" ] && printf '%s' "$at" +} + +# state_mark_phase +state_mark_phase() { state_update '.phases[$p] = {at: $at, input_sha: $s}' --arg p "$1" --arg s "$2" --arg at "$(state_now)"; } + +# state_import_done — exit 0 when this payload file was +# fully imported (no failures) with exactly this content. +state_import_done() { + state_read | "$(sg_resolve jq sg_ensure_jq)" -e --arg s "$1" --arg sha "$2" \ + '.import[$s] | select(.payload_sha == $sha and ((.failed // []) | length) == 0)' >/dev/null 2>&1 +} + +# state_record_import — merge a do_import result +# ({group, payload_sha, imported, failed}) for a payload file. +state_record_import() { state_update '.import[$s] = ($r + {at: $at})' --arg s "$1" --argjson r "$2" --arg at "$(state_now)"; } + +state_reset() { rm -f "$STATE_FILE"; } diff --git a/scripts/migrate.sh b/scripts/migrate.sh index 3c90ef1..dd78765 100755 --- a/scripts/migrate.sh +++ b/scripts/migrate.sh @@ -21,6 +21,8 @@ source "$SCRIPT_DIR/lib/sg_api.sh" source "$SCRIPT_DIR/lib/wizard.sh" # shellcheck source=lib/preflight.sh source "$SCRIPT_DIR/lib/preflight.sh" +# shellcheck source=lib/state.sh +source "$SCRIPT_DIR/lib/state.sh" # SCRIPT_DIR holds the sibling scripts; SG_REPO_ROOT (from tools.sh) is the repo # root used for all repo-relative paths. @@ -40,6 +42,9 @@ VCS_TRIGGERS=1 SKIP_PREFLIGHT=0 # shellcheck disable=SC2034 PREFLIGHT_DONE=0 +FRESH=0 +PROJECT_FILTER=() +WS_FILTER=() VERBOSE="${SG_VERBOSE:-0}" CONC="${SG_CONCURRENCY:-4}" TF_PARALLELISM="${SG_TF_PARALLELISM:-20}" @@ -81,6 +86,9 @@ Options: --no-variable-sets Skip merging TFC Variable Set variables in the 'all' flow --no-vcs-triggers Skip registering VCS triggers after import --skip-preflight Skip the preflight checks (not recommended) + --fresh Ignore the saved run state: redo every phase and re-import everything + --project SEG Only handle this TFC project (repeatable; matches sg-payload..json) + --workspace NAME Only handle this workspace (repeatable; apply exports only it) --all With 'clean': also remove config (terraform.tfvars, mapping, .sg) -v, --verbose Show full terraform/tool output (default: concise) -y, --yes Skip the import confirmation prompt @@ -130,11 +138,54 @@ run_parallel() { return "$rc" } +# payload_sha — hash of the current payload files (the input of enrich/convert/validate). +payload_sha() { + payload_files + sg_sha_files ${PF[@]+"${PF[@]}"} +} + +# run_phase — in 'all', skip a phase that already ran +# with identical inputs; otherwise run it and record the inputs it ran with. +# Phases that rewrite the payloads (enrich, convert) record the post-run hash, +# so an unchanged export is recognised on the next run. +run_phase() { + local name="$1" sha="$2" fn="$3" at + if at="$(state_phase_done "$name" "$sha")" && [ -n "$at" ]; then + sg_log "skipping $name — unchanged since $at (--fresh to redo)" + return 0 + fi + "$fn" || return $? + case "$name" in + apply) state_mark_phase "$name" "$sha" ;; + *) state_mark_phase "$name" "$(payload_sha)" ;; + esac +} + # Populate PF with the generated payload files. payload_files() { + local f seg keep shopt -s nullglob PF=("$EXPORT_DIR"/sg-payload.*.json) shopt -u nullglob + if [ "${#PROJECT_FILTER[@]}" -gt 0 ]; then + keep=() + for f in "${PF[@]}"; do + seg="$(seg_of "$f")" + case " ${PROJECT_FILTER[*]} " in *" $seg "*) keep+=("$f") ;; esac + done + PF=(${keep[@]+"${keep[@]}"}) + fi +} + +# ws_filter_json — the --workspace names as a JSON array (empty array = no filter). +ws_filter_json() { + if [ "${#WS_FILTER[@]}" -eq 0 ]; then echo '[]'; else names_json "${WS_FILTER[@]}"; fi +} + +# ws_selected — exit 0 when no --workspace filter is set or it lists . +ws_selected() { + [ "${#WS_FILTER[@]}" -eq 0 ] && return 0 + case " ${WS_FILTER[*]} " in *" $1 "*) return 0 ;; *) return 1 ;; esac } seg_of() { @@ -210,6 +261,7 @@ do_set_triggers() { continue fi wf="$("$JQ_BIN" -r --argjson i "$i" '.[$i].ResourceName' "$f")" + ws_selected "$wf" || continue # A workflow that failed to import has nothing to attach triggers to. if ! sg_workflow_exists "$grp" "$wf"; then sg_warn " $grp/$wf does not exist in SG (import failed?) — skipping triggers" @@ -266,6 +318,7 @@ cmd_clean() { rm -rf "$TRANSFORMER_DIR/.terraform" "$TRANSFORMER_DIR/.terraform.lock.hcl" \ "$TRANSFORMER_DIR/terraform.tfstate" "$TRANSFORMER_DIR/terraform.tfstate.backup" rm -rf "$SG_CACHE_DIR" + state_reset if [ "$PURGE" -eq 1 ]; then rm -f "$TFVARS" "$MAPPING" rm -rf "$SG_REPO_ROOT/.sg" @@ -283,12 +336,18 @@ cmd_apply() { # both are on PATH for the apply (jq from cache if not already installed). command -v curl >/dev/null 2>&1 || die "curl is required for state export" local jqdir tflog rc=0 + local -a tfvar_args=() jqdir="$(dirname "$(sg_resolve jq sg_ensure_jq)")" + if [ "${#WS_FILTER[@]}" -gt 0 ]; then + JQ_BIN="${JQ_BIN:-$(sg_resolve jq sg_ensure_jq)}" + tfvar_args=(-var "workspacenames=$(ws_filter_json)") + sg_log "limiting apply to workspace(s): ${WS_FILTER[*]}" + fi if [ "$VERBOSE" -eq 1 ]; then (cd "$TRANSFORMER_DIR" && export PATH="$jqdir:$PATH" TF_IN_AUTOMATION=1 && terraform init -input=false && - terraform apply -auto-approve -compact-warnings -parallelism="$TF_PARALLELISM" -var-file=terraform.tfvars) || rc=$? + terraform apply -auto-approve -compact-warnings -parallelism="$TF_PARALLELISM" -var-file=terraform.tfvars "${tfvar_args[@]}") || rc=$? else # Quiet: capture terraform's verbose plan/output; surface only progress, the # final summary, and (on failure) the captured log. @@ -298,7 +357,7 @@ cmd_apply() { if [ "$rc" -eq 0 ]; then sg_log "reading workspaces, generating payloads, exporting state..." (cd "$TRANSFORMER_DIR" && export PATH="$jqdir:$PATH" TF_IN_AUTOMATION=1 && - terraform apply -auto-approve -compact-warnings -no-color -parallelism="$TF_PARALLELISM" -var-file=terraform.tfvars) >"$tflog" 2>&1 || rc=$? + terraform apply -auto-approve -compact-warnings -no-color -parallelism="$TF_PARALLELISM" -var-file=terraform.tfvars "${tfvar_args[@]}") >"$tflog" 2>&1 || rc=$? fi if [ "$rc" -ne 0 ]; then sg_err "terraform failed (rc=$rc):" @@ -374,12 +433,24 @@ names_json() { printf '%s\n' "$@" | "$JQ_BIN" -R . | "$JQ_BIN" -s .; } # the trigger pass see what was actually imported); each fallback is appended to # terraform-version-fallbacks.log. Any other per-workflow failure fails the file. do_import() { - local f="$1" seg grp out rc=0 ceiling="" failed=() fb=() name line tmp names + local f="$1" seg grp out rc=0 ceiling="" failed=() fb=() name line tmp names work all_names seg="$(seg_of "$f")" grp="$(group_for "$seg")" + # With --workspace, import only the selected workflows (a filtered copy). + work="$f" + if [ "${#WS_FILTER[@]}" -gt 0 ]; then + work="$(mktemp "$EXPORT_DIR/.subset.$seg.XXXXXX")" + "$JQ_BIN" --argjson names "$(ws_filter_json)" 'map(select(.ResourceName as $n | $names | index($n) != null))' "$f" >"$work" + if [ "$("$JQ_BIN" 'length' "$work")" -eq 0 ]; then + sg_log "$(basename "$f"): no selected workflows — skipped" + rm -f "$work" + return 0 + fi + fi sg_log "importing $(basename "$f") -> $grp" out="$(mktemp)" - sg_retry "$RETRIES" "$RETRY_BASE" -- sgcli_bulk "$grp" "$f" "$out" || rc=1 + sg_retry "$RETRIES" "$RETRY_BASE" -- sgcli_bulk "$grp" "$work" "$out" || rc=1 + all_names="$("$JQ_BIN" -c '[.[].ResourceName]' "$work")" while IFS= read -r line; do if [[ "$line" =~ $TF_CEILING_RE ]]; then @@ -419,6 +490,16 @@ do_import() { rm -f "$tmp" fi + [ "$work" != "$f" ] && rm -f "$work" + + # Per-file result for the state file and the post-import checklist + # (run_parallel jobs run in subshells, so the caller merges these). + "$JQ_BIN" -nc --arg g "$grp" --arg sha "$(sg_sha_files "$f")" --argjson all "$all_names" \ + --argjson failed "$([ "${#failed[@]}" -gt 0 ] && names_json "${failed[@]}" || echo '[]')" \ + --argjson fallback "$([ "${#fb[@]}" -gt 0 ] && names_json "${fb[@]}" || echo '[]')" \ + '{group: $g, payload_sha: $sha, imported: ($all - $failed), failed: $failed, tf_fallback: ($fallback - $failed)}' \ + >"$EXPORT_DIR/.import-result.$seg.json" + if [ "${#failed[@]}" -gt 0 ]; then sg_err "$(basename "$f"): ${#failed[@]} workflow(s) failed to import: ${failed[*]}" return 1 @@ -520,12 +601,35 @@ cmd_import() { SG_DEFAULT_TF_VERSION="${SG_DEFAULT_TF_VERSION:-$(tfvars_get '.SGDefaultTerraformVersion' TERRAFORM-1.5.7)}" rm -f "$EXPORT_DIR/terraform-version-fallbacks.log" - sg_log "importing ${#PF[@]} payload(s), up to $CONC in parallel (retries: $RETRIES)" + # Resume: skip payload files already imported in full with identical content + # (a changed payload or a previous failure re-imports the whole file; + # sg-cli updates existing workflows in place). + local -a todo=() + local skipped=0 seg + for f in "${PF[@]}"; do + seg="$(seg_of "$f")" + if [ "$FRESH" -eq 0 ] && [ "${#WS_FILTER[@]}" -eq 0 ] && state_import_done "$seg" "$(sg_sha_files "$f")"; then + skipped=$((skipped + 1)) + continue + fi + todo+=("$f") + done + [ "$skipped" -gt 0 ] && sg_log "skipping $skipped payload(s) already imported and unchanged (use --fresh to re-import)" local import_rc=0 - run_parallel do_import "$CONC" "${PF[@]}" || import_rc=1 + if [ "${#todo[@]}" -gt 0 ]; then + sg_log "importing ${#todo[@]} payload(s), up to $CONC in parallel (retries: $RETRIES)" + run_parallel do_import "$CONC" "${todo[@]}" || import_rc=1 + for f in "${todo[@]}"; do + seg="$(seg_of "$f")" + if [ -f "$EXPORT_DIR/.import-result.$seg.json" ]; then + state_record_import "$seg" "$(cat "$EXPORT_DIR/.import-result.$seg.json")" + rm -f "$EXPORT_DIR/.import-result.$seg.json" + fi + done + fi tf_fallback_notice if [ "$import_rc" -eq 0 ]; then - sg_success "import complete (${#PF[@]} payload(s))" + sg_success "import complete (${#todo[@]} payload(s) imported, $skipped skipped)" else sg_err "one or more workflows failed to import (see above); VCS triggers are still registered for the ones that succeeded" fi @@ -541,7 +645,7 @@ cmd_import() { # Single source of truth for shell completion (keep in sync with the parser below # and the host-only flags in sg-migrate.sh). SG_COMMANDS="init preflight apply enrich convert validate import triggers all clean completion" -SG_OPTIONS="--org --export-dir --mapping --concurrency --no-create-groups --no-variable-sets --no-vcs-triggers --skip-preflight --all -v --verbose -y --yes -h --help --native --local --build" +SG_OPTIONS="--org --export-dir --mapping --concurrency --no-create-groups --no-variable-sets --no-vcs-triggers --skip-preflight --fresh --project --workspace --all -v --verbose -y --yes -h --help --native --local --build" # cmd_completion — print a completion script for sg-migrate.sh / # migrate.sh to stdout. Both shells fall back to the basename when the command @@ -560,7 +664,7 @@ _sg_migrate() { case "\$prev" in --export-dir) COMPREPLY=(\$(compgen -d -- "\$cur")); return ;; --mapping) COMPREPLY=(\$(compgen -f -- "\$cur")); return ;; - --org | --concurrency) COMPREPLY=(); return ;; + --org | --concurrency | --project | --workspace) COMPREPLY=(); return ;; completion) COMPREPLY=(\$(compgen -W "bash zsh" -- "\$cur")); return ;; esac for w in "\${COMP_WORDS[@]:1:COMP_CWORD-1}"; do @@ -603,6 +707,9 @@ _sg_migrate() { '--no-variable-sets[Skip merging TFC Variable Sets]' \\ '--no-vcs-triggers[Skip registering VCS triggers after import]' \\ '--skip-preflight[Skip the preflight checks]' \\ + '--fresh[Ignore saved run state: redo every phase]' \\ + '*--project[Only this TFC project segment]:segment' \\ + '*--workspace[Only this workspace]:name' \\ '--all[With clean: also remove config]' \\ '(-v --verbose)'{-v,--verbose}'[Show full terraform/tool output]' \\ '(-y --yes)'{-y,--yes}'[Skip the import confirmation prompt]' \\ @@ -651,6 +758,17 @@ while [ $# -gt 0 ]; do --no-variable-sets) ENRICH_VARSETS=0 ;; --no-vcs-triggers) VCS_TRIGGERS=0 ;; --skip-preflight) export SKIP_PREFLIGHT=1 ;; + --fresh) FRESH=1 ;; + --project) + PROJECT_FILTER+=("$2") + shift + ;; + --project=*) PROJECT_FILTER+=("${1#*=}") ;; + --workspace) + WS_FILTER+=("$2") + shift + ;; + --workspace=*) WS_FILTER+=("${1#*=}") ;; -v | --verbose) VERBOSE=1 ;; --all) PURGE=1 ;; -h | --help) @@ -698,10 +816,12 @@ all) # Fail fast on everything the whole pipeline needs, before the (long) apply. export SG_API_TOKEN SG_BASE_URL preflight_run all - cmd_apply - if [ "$ENRICH_VARSETS" -eq 1 ]; then cmd_enrich; fi - cmd_convert - cmd_validate + [ "$FRESH" -eq 1 ] && { state_reset; sg_log "--fresh: previous run state discarded"; } + JQ_BIN="$(sg_resolve jq sg_ensure_jq)" + run_phase apply "$(sg_sha "$(sg_sha_files "$TFVARS")|$(ws_filter_json)")" cmd_apply + if [ "$ENRICH_VARSETS" -eq 1 ]; then run_phase enrich "$(payload_sha)" cmd_enrich; fi + run_phase convert "$(payload_sha)" cmd_convert + run_phase validate "$(payload_sha)" cmd_validate cmd_import ;; esac From 08bb39fe6eea1f15ab3ff393cab019e55bb7e241 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adis=20Halilovi=C4=87?= Date: Mon, 7 Sep 2026 09:26:39 +0200 Subject: [PATCH 17/71] feat: show the migration summary after apply and an import plan apply now prints a condensed migration summary (workflows per project, sensitive variables skipped, unpinned Terraform versions, non-remote execution modes, renames, failed state exports) instead of leaving it in a file nobody opens. import shows a per-workflow table before the confirmation prompt: create/update, Terraform version with the 1.5.7 fallback marked, runner, triggers, variable and secret counts. 'import --dry-run' stops after the table without touching anything. --- scripts/lib/report.sh | 89 +++++++++++++++++++++++++++++++++++++++++++ scripts/migrate.sh | 21 ++++++++-- 2 files changed, 106 insertions(+), 4 deletions(-) create mode 100644 scripts/lib/report.sh diff --git a/scripts/lib/report.sh b/scripts/lib/report.sh new file mode 100644 index 0000000..a670516 --- /dev/null +++ b/scripts/lib/report.sh @@ -0,0 +1,89 @@ +#!/bin/bash +# Reporting helpers (sourced; needs tools.sh, sg_api.sh): the migration +# summary shown after apply, and the per-workflow import plan shown before the +# confirmation prompt (and by 'import --dry-run'). + +# show_migration_summary — condensed view of export/migration-summary.json plus +# state-export-failures.log, so nobody has to know to open the files. +show_migration_summary() { + local f="$EXPORT_DIR/migration-summary.json" jqb n + [ -f "$f" ] || return 0 + jqb="$(sg_resolve jq sg_ensure_jq)" + sg_step "Migration summary" + printf ' %s%-30s%s %s\n' "$C_BOLD" "TFC organisation" "$C_RESET" "$("$jqb" -r '.organization' "$f")" >&2 + printf ' %s%-30s%s %s\n' "$C_BOLD" "Workspaces exported" "$C_RESET" "$("$jqb" -r '.workspaceCount' "$f")" >&2 + "$jqb" -r '.projectWorkspaceCounts | to_entries[] | " tfc-\(.key | ascii_downcase | gsub("[^a-z0-9-]+"; "-")): \(.value) workflow(s)"' "$f" >&2 + + _summary_section "$f" '.skippedSensitiveVars' "Sensitive variables skipped (TFC never exposes them)" \ + 'to_entries[] | "\(.key): \(.value | join(", "))"' "recreated as SG secrets after import" + _summary_section "$f" '.terraformVersionFallbacks' "Terraform version not pinned (SGDefaultTerraformVersion used)" \ + 'to_entries[] | "\(.key): \"\(.value)\""' "" + _summary_section "$f" '.nonRemoteExecutionModes' "Non-remote execution mode (state may not be in TFC)" \ + 'to_entries[] | "\(.key): \(.value)"' "" + _summary_section "$f" '.renamedWorkspaces' "Renamed to a valid SG workflow name" \ + 'to_entries[] | "\(.key) -> \(.value)"' "" + if [ -s "$EXPORT_DIR/state-export-failures.log" ]; then + n="$(wc -l <"$EXPORT_DIR/state-export-failures.log" | tr -d ' ')" + printf ' %s!%s %s\n' "$C_YELLOW" "$C_RESET" "state could not be exported for $n workspace(s):" >&2 + sed 's/^/ /' "$EXPORT_DIR/state-export-failures.log" | head -10 >&2 + [ "$n" -gt 10 ] && sg_dim " ... see $(sg_rel "$EXPORT_DIR")/state-export-failures.log" + fi + sg_dim "full report: $(sg_rel "$EXPORT_DIR")/migration-summary.md" +} + +# _summary_section <jq-line-filter> <hint> +_summary_section() { + local f="$1" path="$2" title="$3" lines="$4" hint="$5" jqb n + jqb="$(sg_resolve jq sg_ensure_jq)" + n="$("$jqb" -r "$path | length" "$f")" + [ "$n" -gt 0 ] || return 0 + printf ' %s!%s %s (%s)%s\n' "$C_YELLOW" "$C_RESET" "$title" "$n" "${hint:+ — $hint}" >&2 + "$jqb" -r "$path | $lines" "$f" | head -10 | sed 's/^/ /' >&2 + [ "$n" -gt 10 ] && sg_dim " ... $((n - 10)) more in migration-summary.md" + return 0 +} + +# tf_version_above_ceiling <TERRAFORM-x.y.z> — exit 0 when above 1.5.7. +tf_version_above_ceiling() { + [[ "$1" =~ ^TERRAFORM-([0-9]+)\.([0-9]+)\.([0-9]+)$ ]] || return 1 + [ "$(printf '%03d%03d%03d' "${BASH_REMATCH[1]}" "${BASH_REMATCH[2]}" "${BASH_REMATCH[3]}")" -gt "001005007" ] +} + +# show_import_plan <payload>... — per-workflow table: what will be created or +# updated, with the Terraform version (and fallback), runner, triggers and the +# number of variables / skipped secrets. Needs JQ_BIN, ORG, SG_API_TOKEN. +show_import_plan() { + local f seg grp existing summary rows + summary="$EXPORT_DIR/migration-summary.json" + [ -f "$summary" ] || summary="" + printf '\n %s%-28s %-24s %-7s %-28s %-8s %-8s %-5s %s%s\n' "$C_BOLD" "WORKFLOW" "GROUP" "ACTION" "TERRAFORM" "RUNNER" "TRIGGERS" "VARS" "SECRETS" "$C_RESET" >&2 + for f in "$@"; do + seg="$(seg_of "$f")" + grp="$(group_for "$seg")" + existing="$(sg_list_workflows "$grp")" + rows="$("$JQ_BIN" -r --argjson ex "$existing" --arg def "$SG_DEFAULT_TF_VERSION" --argjson ws "$(ws_filter_json)" \ + --slurpfile sum "${summary:-/dev/null}" ' + ($sum[0] // {}) as $S + | .[] + | select(($ws | length) == 0 or (.ResourceName as $n | $ws | index($n) != null)) + | ((.CLIConfiguration.TfStateFilePath // "") | sub(".*/"; "") | sub("\\.tfstate$"; "")) as $wsName + | .ResourceName as $n + | [ $n, + (if ($ex | index($n)) != null then "update" else "create" end), + (.TerraformConfig.terraformVersion // "-"), + ((.RunnerConstraints // {}) | if .type == "private" then "private" else "shared" end), + (if (.VCSTriggers // null) != null then "yes" else "no" end), + ((.VCSConfig.iacInputData.data // {}) | length), + (($S.skippedSensitiveVars // {})[$wsName] // [] | length) + ] | @tsv' "$f")" + while IFS=$'\t' read -r name action tfv runner trig vars secrets; do + [ -n "$name" ] || continue + if tf_version_above_ceiling "$tfv"; then tfv="${tfv#TERRAFORM-} -> ${SG_DEFAULT_TF_VERSION#TERRAFORM-} (fallback)"; else tfv="${tfv#TERRAFORM-}"; fi + [ "$secrets" = "0" ] && secrets="-" + case "$action" in create) action="${C_GREEN}create ${C_RESET}" ;; update) action="${C_YELLOW}update ${C_RESET}" ;; esac + printf ' %-28s %-24s %s %-28s %-8s %-8s %-5s %s\n' "$name" "$grp" "$action" "$tfv" "$runner" "$trig" "$vars" "$secrets" >&2 + done <<<"$rows" + done + echo >&2 + sg_dim "TERRAFORM '-> fallback' = pinned above SG's managed ceiling (1.5.7, last FOSS release); SECRETS = sensitive vars to recreate" +} diff --git a/scripts/migrate.sh b/scripts/migrate.sh index dd78765..0fb52ee 100755 --- a/scripts/migrate.sh +++ b/scripts/migrate.sh @@ -23,6 +23,8 @@ source "$SCRIPT_DIR/lib/wizard.sh" source "$SCRIPT_DIR/lib/preflight.sh" # shellcheck source=lib/state.sh source "$SCRIPT_DIR/lib/state.sh" +# shellcheck source=lib/report.sh +source "$SCRIPT_DIR/lib/report.sh" # SCRIPT_DIR holds the sibling scripts; SG_REPO_ROOT (from tools.sh) is the repo # root used for all repo-relative paths. @@ -43,6 +45,7 @@ SKIP_PREFLIGHT=0 # shellcheck disable=SC2034 PREFLIGHT_DONE=0 FRESH=0 +DRY_RUN=0 PROJECT_FILTER=() WS_FILTER=() VERBOSE="${SG_VERBOSE:-0}" @@ -86,6 +89,7 @@ Options: --no-variable-sets Skip merging TFC Variable Set variables in the 'all' flow --no-vcs-triggers Skip registering VCS triggers after import --skip-preflight Skip the preflight checks (not recommended) + --dry-run With 'import': show the per-workflow plan and stop (nothing is created) --fresh Ignore the saved run state: redo every phase and re-import everything --project SEG Only handle this TFC project (repeatable; matches sg-payload.<SEG>.json) --workspace NAME Only handle this workspace (repeatable; apply exports only it) @@ -370,6 +374,7 @@ cmd_apply() { [ "$rc" -eq 0 ] || return "$rc" sg_success "apply complete — payloads in $(sg_rel "$EXPORT_DIR")" + show_migration_summary } cmd_enrich() { @@ -583,6 +588,15 @@ cmd_import() { die "some groups are missing (override groups are not auto-created; create them or remove the override)." fi + # Fallback Terraform version for workflows the API rejects as above the + # managed ceiling: SGDefaultTerraformVersion from terraform.tfvars, else 1.5.7. + SG_DEFAULT_TF_VERSION="${SG_DEFAULT_TF_VERSION:-$(tfvars_get '.SGDefaultTerraformVersion' TERRAFORM-1.5.7)}" + show_import_plan "${PF[@]}" + if [ "$DRY_RUN" -eq 1 ]; then + sg_success "dry run — nothing was created or changed" + return 0 + fi + if [ "$ASSUME_YES" -ne 1 ]; then printf '%sProceed?%s This imports to %s and creates any "create" groups. [y/N] ' "$C_BOLD$C_YELLOW" "$C_RESET" "$ORG" >&2 read -r ans || ans="" @@ -596,9 +610,6 @@ cmd_import() { done SGCLI_BIN="$(sg_resolve sg-cli sg_ensure_sgcli)" - # Fallback Terraform version for workflows the API rejects as above the - # managed ceiling: SGDefaultTerraformVersion from terraform.tfvars, else 1.5.7. - SG_DEFAULT_TF_VERSION="${SG_DEFAULT_TF_VERSION:-$(tfvars_get '.SGDefaultTerraformVersion' TERRAFORM-1.5.7)}" rm -f "$EXPORT_DIR/terraform-version-fallbacks.log" # Resume: skip payload files already imported in full with identical content @@ -645,7 +656,7 @@ cmd_import() { # Single source of truth for shell completion (keep in sync with the parser below # and the host-only flags in sg-migrate.sh). SG_COMMANDS="init preflight apply enrich convert validate import triggers all clean completion" -SG_OPTIONS="--org --export-dir --mapping --concurrency --no-create-groups --no-variable-sets --no-vcs-triggers --skip-preflight --fresh --project --workspace --all -v --verbose -y --yes -h --help --native --local --build" +SG_OPTIONS="--org --export-dir --mapping --concurrency --no-create-groups --no-variable-sets --no-vcs-triggers --skip-preflight --dry-run --fresh --project --workspace --all -v --verbose -y --yes -h --help --native --local --build" # cmd_completion <bash|zsh> — print a completion script for sg-migrate.sh / # migrate.sh to stdout. Both shells fall back to the basename when the command @@ -707,6 +718,7 @@ _sg_migrate() { '--no-variable-sets[Skip merging TFC Variable Sets]' \\ '--no-vcs-triggers[Skip registering VCS triggers after import]' \\ '--skip-preflight[Skip the preflight checks]' \\ + '--dry-run[With import: show the plan and stop]' \\ '--fresh[Ignore saved run state: redo every phase]' \\ '*--project[Only this TFC project segment]:segment' \\ '*--workspace[Only this workspace]:name' \\ @@ -759,6 +771,7 @@ while [ $# -gt 0 ]; do --no-vcs-triggers) VCS_TRIGGERS=0 ;; --skip-preflight) export SKIP_PREFLIGHT=1 ;; --fresh) FRESH=1 ;; + --dry-run) DRY_RUN=1 ;; --project) PROJECT_FILTER+=("$2") shift From 497612032a5415225c6828b7ac697aa5ce5075b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adis=20Halilovi=C4=87?= <adis.halilovic@stackguardian.io> Date: Mon, 7 Sep 2026 09:29:47 +0200 Subject: [PATCH 18/71] feat: explain known StackGuardian API errors Map the API messages users actually hit (unknown connector, missing runner group, repository not reachable, bad approvers, invalid names or VCS kind, rejected token, Terraform ceiling) to a one-line hint naming the terraform.tfvars field to fix. Shown once per distinct hint, both for direct API calls and for sg-cli's per-workflow failures; unknown messages still show the raw response. --- scripts/lib/errors.sh | 42 ++++++++++++++++++++++++++++++++++++++++++ scripts/migrate.sh | 5 +++++ 2 files changed, 47 insertions(+) create mode 100644 scripts/lib/errors.sh diff --git a/scripts/lib/errors.sh b/scripts/lib/errors.sh new file mode 100644 index 0000000..bed4439 --- /dev/null +++ b/scripts/lib/errors.sh @@ -0,0 +1,42 @@ +#!/bin/bash +# Friendly explanations for StackGuardian API errors (sourced; needs tools.sh). +# +# explain_api_error <response-body-or-message> prints, at most once per +# distinct hint per run, what the error usually means and which +# terraform.tfvars field to change. Unknown messages print nothing (the raw +# response is already shown by the caller). Patterns are case-insensitive +# extended regexes matched against the message text. + +_SG_HINTS_SHOWN=" " + +# Table: "<regex> => <hint>" — keep hints to one line each. +SG_API_HINTS=( + 'above the highest managed version => Terraform version above SG'"'"'s managed ceiling (1.5.7, last MPL/FOSS release). The importer retries with SGDefaultTerraformVersion automatically; to keep the newer version use a private runner binary path or a custom runtime template (workspaceOverrides[<ws>].terraformVersion).' + 'integration.*(not found|does not exist|invalid)|(not found|does not exist|invalid).*integration => Connector/integration id not found in this org. Check SGDefaultVCSAuthIntegrationID and SGDefaultDeploymentPlatformConfig[].config.integrationId (or the workspaceOverrides equivalents); '"'"'preflight'"'"' lists the ids that exist.' + 'runner ?group.*(not found|does not exist|invalid)|(not found|does not exist|invalid).*runner => Runner group not found. Check SGDefaultRunnerConstraints.names / workspaceOverrides[<ws>].RunnerConstraints against the runner groups in the SG org.' + '(repo|repository).*(not found|access|permission|denied|unable|could not|cannot)|(clone|checkout).*(fail|denied|unable) => The VCS connector cannot reach the repository. Check SGDefaultIACVCSRepoPrefix + the workspace repo path, and that the connector (SGDefaultVCSAuthIntegrationID) has access to that repository.' + 'approver => Approvers must be e-mail addresses of existing SG users. Check SGDefaultWfApprovers / workspaceOverrides[<ws>].Approvers.' + 'already exists => A resource with that name already exists. Re-running the import updates workflows in place; for workflow groups this is harmless.' + 'ResourceName => Invalid workflow name: 1-100 chars, letters/digits/-/_ only. The transformer sanitizes names (see renamedWorkspaces in migration-summary.md); adjust local.resourceNames if a rule is missing.' + 'sourceConfigDestKind => Invalid VCS kind. SGDefaultSourceConfigDestKind must be one of GITHUB_COM, GITLAB_COM, BITBUCKET_ORG, AZURE_DEVOPS, GIT_OTHER.' + '(unauthori[sz]ed|forbidden|invalid token|authentication) => SG_API_TOKEN was rejected or lacks permission for this org. Regenerate it under Org settings -> API keys and re-export SG_API_TOKEN.' +) + +explain_api_error() { + local msg="$1" entry re hint + # Prefer the API's "msg" field when the body is JSON. + if printf '%s' "$msg" | grep -q '^{'; then + msg="$(printf '%s' "$msg" | "$(sg_resolve jq sg_ensure_jq)" -r '.msg // .message // .error // .' 2>/dev/null || printf '%s' "$msg")" + fi + for entry in "${SG_API_HINTS[@]}"; do + re="${entry%% => *}" + hint="${entry#* => }" + if printf '%s' "$msg" | grep -Eiq -- "$re"; then + case "$_SG_HINTS_SHOWN" in *" $re "*) return 0 ;; esac + _SG_HINTS_SHOWN="$_SG_HINTS_SHOWN$re " + printf ' %s→ %s%s\n' "$C_YELLOW" "$hint" "$C_RESET" >&2 + return 0 + fi + done + return 0 +} diff --git a/scripts/migrate.sh b/scripts/migrate.sh index 0fb52ee..0527d28 100755 --- a/scripts/migrate.sh +++ b/scripts/migrate.sh @@ -25,6 +25,8 @@ source "$SCRIPT_DIR/lib/preflight.sh" source "$SCRIPT_DIR/lib/state.sh" # shellcheck source=lib/report.sh source "$SCRIPT_DIR/lib/report.sh" +# shellcheck source=lib/errors.sh +source "$SCRIPT_DIR/lib/errors.sh" # SCRIPT_DIR holds the sibling scripts; SG_REPO_ROOT (from tools.sh) is the repo # root used for all repo-relative paths. @@ -461,6 +463,9 @@ do_import() { if [[ "$line" =~ $TF_CEILING_RE ]]; then fb+=("${BASH_REMATCH[1]}") ceiling="${BASH_REMATCH[2]}" + elif [[ "$line" =~ Failed\ to\ create\ ([^:]+):\ [0-9]+:\ (.*)$ ]]; then + failed+=("${BASH_REMATCH[1]}") + explain_api_error "${BASH_REMATCH[2]}" elif [[ "$line" =~ Failed\ to\ create\ ([^:]+): ]]; then failed+=("${BASH_REMATCH[1]}") fi From 797dcb9fc47dd787c16e33c2ddac0c612f521a5c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adis=20Halilovi=C4=87?= <adis.halilovic@stackguardian.io> Date: Mon, 7 Sep 2026 09:31:17 +0200 Subject: [PATCH 19/71] feat: post-import checklist and placeholder secrets After import (or via 'checklist'), write export/post-import-checklist.md listing everything the migration could not do by itself: secrets to fill in, workflows that failed to import, Terraform version fallbacks to verify, VCS triggers that failed, state that could not be exported, non-remote execution modes and renamed workflows, with deep links into the SG UI (SG_UI_URL). For every sensitive variable TFC would not expose, create an SG secret with the value CHANGE_ME, reference it from the workflow (env var or IaC input, ${secret::<name>}) and patch the payload to match; --no-secret-stubs opts out. Trigger registration results are now recorded in the run state. --- scripts/lib/checklist.sh | 162 +++++++++++++++++++++++++++++++++++++++ scripts/migrate.sh | 40 ++++++++-- sg-migrate.sh | 2 +- 3 files changed, 198 insertions(+), 6 deletions(-) create mode 100644 scripts/lib/checklist.sh diff --git a/scripts/lib/checklist.sh b/scripts/lib/checklist.sh new file mode 100644 index 0000000..748683c --- /dev/null +++ b/scripts/lib/checklist.sh @@ -0,0 +1,162 @@ +#!/bin/bash +# Post-import checklist and secret stubs (sourced; needs tools.sh, sg_api.sh, +# state.sh). Turns everything the migration could not do automatically into +# export/post-import-checklist.md, and creates placeholder SG secrets for the +# sensitive variables TFC never exposes so the workflows are wired up and the +# user only has to fill in values. + +SG_UI_URL="${SG_UI_URL:-https://app.stackguardian.io}" +wf_ui_url() { printf '%s/orchestrator/orgs/%s/wfgrps/%s/wfs/%s' "$SG_UI_URL" "$ORG" "$1" "$2"; } +secrets_ui_url() { printf '%s/orchestrator/orgs/%s/settings?tab=secrets' "$SG_UI_URL" "$ORG"; } + +# secret_name_for <workflow> <var> — SG secret name for a stubbed variable. +secret_name_for() { printf 'tfc-%s-%s' "$1" "$2" | tr -c 'A-Za-z0-9_-\n' '-' | cut -c1-100; } + +# workflow_for_workspace <tfc-workspace-name> — "<seg>\t<group>\t<ResourceName>" +# for the payload entry whose TfStateFilePath basename is the workspace. +workflow_for_workspace() { + local f seg wf + for f in "${PF[@]}"; do + wf="$("$JQ_BIN" -r --arg ws "$1" '.[] | select(((.CLIConfiguration.TfStateFilePath // "") | sub(".*/"; "") | sub("\\.tfstate$"; "")) == $ws) | .ResourceName' "$f" | head -1)" + if [ -n "$wf" ]; then + seg="$(seg_of "$f")" + printf '%s\t%s\t%s\n' "$seg" "$(group_for "$seg")" "$wf" + return 0 + fi + done + return 1 +} + +# create_secret_stubs — for every skipped sensitive variable: create the SG +# secret (placeholder value) if missing, reference it from the workflow (env +# var or IaC input), patch the payload file to match, and record it in state. +create_secret_stubs() { + local summary="$EXPORT_DIR/migration-summary.json" ws entry cat var f seg grp wf name ref created=0 reused=0 patched=0 skipped=0 body imported + [ -f "$summary" ] || return 0 + [ "$("$JQ_BIN" '.skippedSensitiveVars | length' "$summary")" -gt 0 ] || return 0 + sg_log "creating placeholder secrets for sensitive variables (value: CHANGE_ME)..." + while IFS=$'\t' read -r ws entry; do + [ -n "$ws" ] || continue + cat="${entry%%:*}" + var="${entry#*:}" + if ! IFS=$'\t' read -r seg grp wf < <(workflow_for_workspace "$ws"); then + sg_warn " $ws: no payload entry found for $entry — listed in the checklist only" + skipped=$((skipped + 1)) + continue + fi + ws_selected "$wf" || continue + imported="$(state_read | "$JQ_BIN" -r --arg s "$seg" --arg w "$wf" '.import[$s].imported // [] | index($w) != null')" + if [ "$imported" != "true" ] || ! sg_workflow_exists "$grp" "$wf"; then + sg_warn " $grp/$wf is not in StackGuardian (import failed?) — $entry listed in the checklist only" + skipped=$((skipped + 1)) + continue + fi + name="$(secret_name_for "$wf" "$var")" + ref="\${secret::$name}" + if sg_secret_exists "$name"; then + reused=$((reused + 1)) + elif sg_create_secret "$name" "CHANGE_ME" "Placeholder for TFC sensitive variable $entry of workspace $ws — set the real value"; then + created=$((created + 1)) + else + sg_warn " could not create secret $name — $entry listed in the checklist only" + skipped=$((skipped + 1)) + continue + fi + # Reference the secret from the workflow: env var or IaC input value. + f="$EXPORT_DIR/sg-payload.$seg.json" + if [ "$cat" = "env" ]; then + "$JQ_BIN" --arg w "$wf" --arg v "$var" --arg r "$ref" 'map(if .ResourceName == $w then + .EnvironmentVariables = ((.EnvironmentVariables // []) | map(select(.config.varName != $v)) + [{kind:"PLAIN_TEXT", config:{varName:$v, textValue:$r}}]) else . end)' "$f" >"$f.tmp" && mv -f "$f.tmp" "$f" + body="$("$JQ_BIN" -c --arg w "$wf" '.[] | select(.ResourceName == $w) | {EnvironmentVariables}' "$f")" + else + "$JQ_BIN" --arg w "$wf" --arg v "$var" --arg r "$ref" 'map(if .ResourceName == $w then .VCSConfig.iacInputData.data[$v] = $r else . end)' "$f" >"$f.tmp" && mv -f "$f.tmp" "$f" + body="$("$JQ_BIN" -c --arg w "$wf" '.[] | select(.ResourceName == $w) | {VCSConfig}' "$f")" + fi + if SG_NO_RETRY_RC=22 sg_retry "$RETRIES" "$RETRY_BASE" -- sg_patch_workflow "$grp" "$wf" "$body"; then + patched=$((patched + 1)) + else + sg_warn " secret $name created but $grp/$wf could not be updated to reference it" + fi + state_update '.secrets[$n] = {workflow: $w, group: $g, var: $v, category: $c, workspace: $ws, at: $at}' \ + --arg n "$name" --arg w "$wf" --arg g "$grp" --arg v "$var" --arg c "$cat" --arg ws "$ws" --arg at "$(state_now)" + done < <("$JQ_BIN" -r '.skippedSensitiveVars | to_entries[] | .key as $ws | .value[] | "\($ws)\t\(.)"' "$summary") + sg_log "secret stubs: $created created, $reused already existed, $patched workflow(s) now reference them, $skipped skipped" + return 0 +} + +# write_checklist — render export/post-import-checklist.md and print it. +write_checklist() { + local out="$EXPORT_DIR/post-import-checklist.md" summary="$EXPORT_DIR/migration-summary.json" st items + st="$(state_read)" + { + printf '# Post-import checklist — StackGuardian org %s\n\n' "$ORG" + printf 'Generated %s by stackguardian-migrator. Tick items as you complete them.\n\n' "$(state_now)" + + # 1. secrets + printf '## 1. Set the real values of the placeholder secrets\n\n' + printf 'TFC never exposes sensitive variable values, so each one was recreated as an SG secret with the value `CHANGE_ME` and referenced from its workflow as `${secret::<name>}`. Set the real values under [Org settings → Secrets](%s).\n\n' "$(secrets_ui_url)" + items="$(printf '%s' "$st" | "$JQ_BIN" -r --arg ui "$SG_UI_URL" --arg org "$ORG" '.secrets // {} | to_entries[] | "- [ ] `\(.key)` — \(.value.category) var `\(.value.var)` of [\(.value.group)/\(.value.workflow)](\($ui)/orchestrator/orgs/\($org)/wfgrps/\(.value.group)/wfs/\(.value.workflow))"')" + if [ -n "$items" ]; then printf '%s\n\n' "$items"; else printf -- '- None.\n\n'; fi + if [ -f "$summary" ]; then + items="$("$JQ_BIN" -r --argjson stubbed "$(printf '%s' "$st" | "$JQ_BIN" -c '[.secrets // {} | .[] | "\(.workspace)|\(.category):\(.var)"]')" \ + '.skippedSensitiveVars | to_entries[] | .key as $ws | .value[] | select(($ws + "|" + .) as $k | $stubbed | index($k) == null) | "- [ ] `\(.)` of workspace `\($ws)` — no stub created (workflow missing or --no-secret-stubs); create the secret and reference it by hand"' "$summary")" + [ -n "$items" ] && printf '%s\n\n' "$items" + fi + + # 2. failed imports + printf '## 2. Workflows that failed to import\n\n' + items="$(printf '%s' "$st" | "$JQ_BIN" -r '.import // {} | to_entries[] | .value.group as $g | .value.failed[]? | "- [ ] `\($g)/\(.)` — see the import output for the API error; fix terraform.tfvars (or workspaceOverrides) and re-run `./sg-migrate.sh import`"')" + if [ -n "$items" ]; then printf '%s\n\n' "$items"; else printf -- '- None.\n\n'; fi + + # 3. terraform version fallbacks + printf '## 3. Verify workflows moved to a different Terraform version\n\n' + printf 'StackGuardian bundles managed Terraform only up to 1.5.7 (the last MPL/FOSS release). These workflows were pinned higher in TFC and now run the fallback version; run a plan and check for incompatibilities. To keep the newer version, point `workspaceOverrides[<ws>].terraformVersion` at a binary on a private runner and re-import.\n\n' + items="" + [ -s "$EXPORT_DIR/terraform-version-fallbacks.log" ] && items="$(sed 's/^/- [ ] /' "$EXPORT_DIR/terraform-version-fallbacks.log")" + if [ -f "$summary" ]; then + items="$items$(printf '%s' "$items" | grep -q . && echo)$("$JQ_BIN" -r '.terraformVersionFallbacks | to_entries[] | "- [ ] `\(.key)` was not pinned in TFC (\"\(.value)\"); SGDefaultTerraformVersion was used — confirm it is compatible"' "$summary")" + fi + if [ -n "$items" ]; then printf '%s\n\n' "$items"; else printf -- '- None.\n\n'; fi + + # 4. VCS triggers + printf '## 4. VCS triggers\n\n' + items="$(printf '%s' "$st" | "$JQ_BIN" -r '.triggers // {} | to_entries[] | .value.group as $g | (.value.failed[]? | "- [ ] `\($g)/\(.)` — trigger registration failed; check the connector has admin/webhook rights on the repository, then re-run `./sg-migrate.sh triggers`"), (.value.missing[]? | "- [ ] `\($g)/\(.)` — workflow was not imported, so no trigger was registered")')" + if [ -n "$items" ]; then printf '%s\n\n' "$items"; else printf -- '- None failed. Push to a tracked branch or open a pull request to confirm the webhooks fire.\n\n'; fi + + # 5. state + printf '## 5. Terraform state\n\n' + if [ -s "$EXPORT_DIR/state-export-failures.log" ]; then + printf 'State could not be pulled from TFC for these workspaces; upload it manually (Workflow → Settings → State) or run an import in the new workflow.\n\n' + sed 's/^/- [ ] /' "$EXPORT_DIR/state-export-failures.log"; echo + else + printf -- '- All selected workspaces had their state exported.\n\n' + fi + if [ -f "$summary" ]; then + items="$("$JQ_BIN" -r '.nonRemoteExecutionModes | to_entries[] | "- [ ] `\(.key)` used `\(.value)` execution in TFC — its state may live outside TFC; verify the exported state is current"' "$summary")" + [ -n "$items" ] && printf '%s\n\n' "$items" + fi + + # 6. renames + if [ -f "$summary" ] && [ "$("$JQ_BIN" '.renamedWorkspaces | length' "$summary")" -gt 0 ]; then + printf '## 6. Renamed workflows\n\nThese TFC workspace names were not valid StackGuardian workflow names and were adjusted:\n\n' + "$JQ_BIN" -r '.renamedWorkspaces | to_entries[] | "- `\(.key)` → `\(.value)`"' "$summary"; echo + fi + + printf '## Finally\n\n- [ ] Run a plan on one workflow per project and compare with the last TFC run.\n- [ ] Disable auto-apply / triggers on the TFC workspaces once StackGuardian owns the deployments.\n' + } >"$out" + sg_step "Post-import checklist" + sed 's/^/ /' "$out" >&2 + sg_dim "saved to $(sg_rel "$out")" +} + +cmd_checklist() { + sg_step "Phase: checklist" + [ -n "${SG_API_TOKEN:-}" ] || die "SG_API_TOKEN is not set." + [ -n "$ORG" ] || die "StackGuardian org not set (use --org or SG_ORG)." + export SG_API_TOKEN SG_BASE_URL + JQ_BIN="$(sg_resolve jq sg_ensure_jq)" + payload_files + [ "${#PF[@]}" -gt 0 ] || die "No payload files in $(sg_rel "$EXPORT_DIR")." + [ "$SECRET_STUBS" -eq 1 ] && create_secret_stubs + write_checklist +} diff --git a/scripts/migrate.sh b/scripts/migrate.sh index 0527d28..2ad0bf1 100755 --- a/scripts/migrate.sh +++ b/scripts/migrate.sh @@ -27,6 +27,8 @@ source "$SCRIPT_DIR/lib/state.sh" source "$SCRIPT_DIR/lib/report.sh" # shellcheck source=lib/errors.sh source "$SCRIPT_DIR/lib/errors.sh" +# shellcheck source=lib/checklist.sh +source "$SCRIPT_DIR/lib/checklist.sh" # SCRIPT_DIR holds the sibling scripts; SG_REPO_ROOT (from tools.sh) is the repo # root used for all repo-relative paths. @@ -48,6 +50,7 @@ SKIP_PREFLIGHT=0 PREFLIGHT_DONE=0 FRESH=0 DRY_RUN=0 +SECRET_STUBS=1 PROJECT_FILTER=() WS_FILTER=() VERBOSE="${SG_VERBOSE:-0}" @@ -72,6 +75,8 @@ Commands: import Import each payload to StackGuardian (parallel, with confirmation), then register VCS triggers (unless --no-vcs-triggers) triggers Register VCS triggers for already-imported workflows (second pass) + checklist Create placeholder secrets for skipped sensitive variables and write + export/post-import-checklist.md (runs automatically after import) all apply -> enrich -> convert -> validate -> import clean Remove local working artifacts for a fresh start (export/, TF state, tool cache). Add --all to also remove config (terraform.tfvars, mapping). @@ -92,6 +97,7 @@ Options: --no-vcs-triggers Skip registering VCS triggers after import --skip-preflight Skip the preflight checks (not recommended) --dry-run With 'import': show the per-workflow plan and stop (nothing is created) + --no-secret-stubs Do not create placeholder SG secrets for sensitive variables --fresh Ignore the saved run state: redo every phase and re-import everything --project SEG Only handle this TFC project (repeatable; matches sg-payload.<SEG>.json) --workspace NAME Only handle this workspace (repeatable; apply exports only it) @@ -105,6 +111,7 @@ Environment: SG_ORG StackGuardian org (alternative to --org) SG_RETRIES Import retry attempts on failure (default: 4) SG_TF_PARALLELISM terraform apply -parallelism (default: 20) + SG_UI_URL StackGuardian UI base for checklist links (default: https://app.stackguardian.io) EOF } @@ -257,7 +264,7 @@ group_for() { # straight from the (converted) payload. Per-workflow failures are surfaced but # do not abort the rest of the file. do_set_triggers() { - local f="$1" seg grp n i wf body rc=0 set=0 skip=0 + local f="$1" seg grp n i wf body rc=0 set=0 skip=0 ok=() failed=() missing=() seg="$(seg_of "$f")" grp="$(group_for "$seg")" n="$("$JQ_BIN" 'length' "$f")" @@ -271,6 +278,7 @@ do_set_triggers() { # A workflow that failed to import has nothing to attach triggers to. if ! sg_workflow_exists "$grp" "$wf"; then sg_warn " $grp/$wf does not exist in SG (import failed?) — skipping triggers" + missing+=("$wf") rc=1 continue fi @@ -278,12 +286,19 @@ do_set_triggers() { if SG_NO_RETRY_RC=22 sg_retry "$RETRIES" "$RETRY_BASE" -- \ sg_api_post "$(wf_triggers_endpoint "$grp" "$wf")" "$body"; then set=$((set + 1)) + ok+=("$wf") else sg_warn " vcs triggers failed: $grp/$wf" + failed+=("$wf") rc=1 fi done sg_log "$(basename "$f"): set triggers on $set workflow(s) (skipped $skip without triggers)" + "$JQ_BIN" -nc --arg g "$grp" \ + --argjson ok "$([ "${#ok[@]}" -gt 0 ] && names_json "${ok[@]}" || echo '[]')" \ + --argjson failed "$([ "${#failed[@]}" -gt 0 ] && names_json "${failed[@]}" || echo '[]')" \ + --argjson missing "$([ "${#missing[@]}" -gt 0 ] && names_json "${missing[@]}" || echo '[]')" \ + '{group: $g, set: $ok, failed: $failed, missing: $missing}' >"$EXPORT_DIR/.triggers-result.$seg.json" return "$rc" } @@ -297,7 +312,16 @@ set_triggers_pass() { return 0 fi sg_log "registering VCS triggers for $total workflow(s), up to $CONC in parallel (retries: $RETRIES)" - if run_parallel do_set_triggers "$CONC" "${PF[@]}"; then + local rc=0 f seg + run_parallel do_set_triggers "$CONC" "${PF[@]}" || rc=1 + for f in "${PF[@]}"; do + seg="$(seg_of "$f")" + if [ -f "$EXPORT_DIR/.triggers-result.$seg.json" ]; then + state_update '.triggers[$s] = ($r + {at: $at})' --arg s "$seg" --argjson r "$(cat "$EXPORT_DIR/.triggers-result.$seg.json")" --arg at "$(state_now)" + rm -f "$EXPORT_DIR/.triggers-result.$seg.json" + fi + done + if [ "$rc" -eq 0 ]; then sg_success "vcs triggers registered" else sg_err "one or more VCS trigger registrations failed (re-run: $PROG triggers)" @@ -655,13 +679,15 @@ cmd_import() { if [ "$VCS_TRIGGERS" -eq 1 ]; then set_triggers_pass || import_rc=1 fi + [ "$SECRET_STUBS" -eq 1 ] && create_secret_stubs + write_checklist return "$import_rc" } # Single source of truth for shell completion (keep in sync with the parser below # and the host-only flags in sg-migrate.sh). -SG_COMMANDS="init preflight apply enrich convert validate import triggers all clean completion" -SG_OPTIONS="--org --export-dir --mapping --concurrency --no-create-groups --no-variable-sets --no-vcs-triggers --skip-preflight --dry-run --fresh --project --workspace --all -v --verbose -y --yes -h --help --native --local --build" +SG_COMMANDS="init preflight apply enrich convert validate import triggers checklist all clean completion" +SG_OPTIONS="--org --export-dir --mapping --concurrency --no-create-groups --no-variable-sets --no-vcs-triggers --skip-preflight --dry-run --no-secret-stubs --fresh --project --workspace --all -v --verbose -y --yes -h --help --native --local --build" # cmd_completion <bash|zsh> — print a completion script for sg-migrate.sh / # migrate.sh to stdout. Both shells fall back to the basename when the command @@ -710,6 +736,7 @@ _sg_migrate() { 'validate:Validate payloads against the SG schema' 'import:Import payloads to StackGuardian, then register VCS triggers' 'triggers:Register VCS triggers for already-imported workflows' + 'checklist:Write the post-import checklist (and create secret stubs)' 'all:apply -> enrich -> convert -> validate -> import' 'clean:Remove local working artifacts' 'completion:Print a shell completion script' @@ -724,6 +751,7 @@ _sg_migrate() { '--no-vcs-triggers[Skip registering VCS triggers after import]' \\ '--skip-preflight[Skip the preflight checks]' \\ '--dry-run[With import: show the plan and stop]' \\ + '--no-secret-stubs[Do not create placeholder SG secrets for sensitive vars]' \\ '--fresh[Ignore saved run state: redo every phase]' \\ '*--project[Only this TFC project segment]:segment' \\ '*--workspace[Only this workspace]:name' \\ @@ -777,6 +805,7 @@ while [ $# -gt 0 ]; do --skip-preflight) export SKIP_PREFLIGHT=1 ;; --fresh) FRESH=1 ;; --dry-run) DRY_RUN=1 ;; + --no-secret-stubs) SECRET_STUBS=0 ;; --project) PROJECT_FILTER+=("$2") shift @@ -793,7 +822,7 @@ while [ $# -gt 0 ]; do usage exit 0 ;; - init | apply | enrich | convert | validate | import | triggers | all | clean | preflight) CMD="$1" ;; + init | apply | enrich | convert | validate | import | triggers | all | clean | preflight | checklist) CMD="$1" ;; completion) cmd_completion "${2:-}" exit 0 @@ -826,6 +855,7 @@ preflight) export SG_API_TOKEN SG_BASE_URL cmd_preflight ;; +checklist) cmd_checklist ;; all) if [ ! -f "$TFVARS" ]; then cmd_init diff --git a/sg-migrate.sh b/sg-migrate.sh index 5a56ba4..0d96281 100755 --- a/sg-migrate.sh +++ b/sg-migrate.sh @@ -33,7 +33,7 @@ HAS_CMD=0 for a in ${ARGS[@]+"${ARGS[@]}"}; do case "$a" in clean | completion | -h | --help) NATIVE=1; HAS_CMD=1 ;; - init | preflight | apply | enrich | convert | validate | import | triggers | all) HAS_CMD=1 ;; + init | preflight | apply | enrich | convert | validate | import | triggers | checklist | all) HAS_CMD=1 ;; esac done [ "$HAS_CMD" -eq 1 ] || NATIVE=1 From 07296d6260e1d4fe02e96a5ee92b8988e9b5e1e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adis=20Halilovi=C4=87?= <adis.halilovic@stackguardian.io> Date: Mon, 7 Sep 2026 09:32:26 +0200 Subject: [PATCH 20/71] docs: describe the guided init, preflight, resume, dry-run and checklist --- CLAUDE.md | 3 ++- README.md | 30 +++++++++++++++++------------- scripts/migrate.sh | 5 +++-- 3 files changed, 22 insertions(+), 16 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index ad9f762..ed6fc39 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -24,7 +24,8 @@ The migration is a user-driven pipeline, not a single program: The five phases are wrapped by an orchestrator so users don't run them by hand: - `sg-migrate.sh` (host entrypoint, repo root) — runs `scripts/migrate.sh` **inside the Docker image** (`Dockerfile`), bind-mounting the repo at `/app` and forwarding `SG_API_TOKEN`/`SG_ORG`/`SG_BASE_URL`/`TFE_TOKEN`/`TF_TOKEN_*`. For TFC auth it prefers a long-lived `TFE_TOKEN` (env); otherwise it mounts `~/.terraform.d/credentials.tfrc.json` read-only. Runs natively instead when `--native`/`--local` is passed, `SG_NATIVE=1` is set, the command is `clean`/`completion`/help (or no command is given), or Docker is absent. Exports `SG_PROG` so `migrate.sh` shows `./sg-migrate.sh` in its usage/hints. -- `scripts/migrate.sh` — the actual orchestrator. Subcommands `init|apply|enrich|convert|validate|import|triggers|all|clean|completion` (no command prints the help menu; `enrich` runs in `all` unless `--no-variable-sets`). Resolves each tool via `sg_resolve` (PATH first — the image installs them — else `sg_ensure_*` cache). `apply`/`all` first run `require_tfc_auth`, which resolves the token the tfe provider will use (`TFE_TOKEN`, `TF_TOKEN_<host>`, or the `terraform login` file for `tfHostname`) and verifies it with `GET /api/v2/account/details`, failing fast on a missing or rejected (expired) credential. Runs `convert` and `import` in parallel (`--concurrency`, default 4), raises `terraform apply -parallelism`, and retries `sg-cli` imports with backoff (`sg_retry`, `SG_RETRIES`). Import resolves each project's workflow group, checks existence via the SG API, shows a plan (`exists`/`create`), prompts (skip with `-y`), **creates the missing `tfc-<project>` groups via the API**, then imports. sg-cli exits 0 even when individual workflows fail, so `do_import` parses its output: a workflow rejected as above SG's managed Terraform ceiling (1.5.7, last MPL/FOSS release) is re-imported with `SGDefaultTerraformVersion` (read from `terraform.tfvars`), the payload patched in place, and the case logged to `export/terraform-version-fallbacks.log` plus a printed notice; any other per-workflow failure fails the run. The trigger pass skips workflows that do not exist in SG, and `sg_api_post` returns 22 on 4xx so `sg_retry` (via `SG_NO_RETRY_RC`) does not retry definitive errors. `clean` removes local artifacts (`export/`, TF state, tool cache); `clean --all` also removes config. `completion bash|zsh` prints a completion script (`cmd_completion`; commands/options come from `SG_COMMANDS`/`SG_OPTIONS`, keep them in sync with the parser and the host flags); like `clean` it always runs natively. Output is concise by default — `apply` captures terraform's init/plan output and shows only progress + the final summary (or the full log on failure, `TF_IN_AUTOMATION=1`/`-no-color`); `-v`/`--verbose` (`SG_VERBOSE`) streams everything and un-gates the convert detail lines. Colored logging via `sg_step`/`sg_log`/`sg_success`/`sg_warn`/`sg_err`; paths shown relative via `sg_rel` (all auto-off when not a TTY / `NO_COLOR`). +- `scripts/migrate.sh` — the actual orchestrator (sources `scripts/lib/*.sh`, see below). Subcommands `init|preflight|apply|enrich|convert|validate|import|triggers|checklist|all|clean|completion` (no command prints the help menu; `enrich` runs in `all` unless `--no-variable-sets`). Flags beyond the basics: `--dry-run` (import: plan only), `--fresh` (ignore run state), `--project SEG` / `--workspace NAME` (repeatable filters; apply maps `--workspace` to `-var workspacenames=[...]`, import works on a jq-filtered temp copy), `--skip-preflight`, `--no-secret-stubs`. Resolves each tool via `sg_resolve` (PATH first — the image installs them — else `sg_ensure_*` cache). `apply`/`all` first run `require_tfc_auth`, which resolves the token the tfe provider will use (`TFE_TOKEN`, `TF_TOKEN_<host>`, or the `terraform login` file for `tfHostname`) and verifies it with `GET /api/v2/account/details`, failing fast on a missing or rejected (expired) credential. Runs `convert` and `import` in parallel (`--concurrency`, default 4), raises `terraform apply -parallelism`, and retries `sg-cli` imports with backoff (`sg_retry`, `SG_RETRIES`). Import resolves each project's workflow group, checks existence via the SG API, shows a plan (`exists`/`create`), prompts (skip with `-y`), **creates the missing `tfc-<project>` groups via the API**, then imports. sg-cli exits 0 even when individual workflows fail, so `do_import` parses its output: a workflow rejected as above SG's managed Terraform ceiling (1.5.7, last MPL/FOSS release) is re-imported with `SGDefaultTerraformVersion` (read from `terraform.tfvars`), the payload patched in place, and the case logged to `export/terraform-version-fallbacks.log` plus a printed notice; any other per-workflow failure fails the run. The trigger pass skips workflows that do not exist in SG, and `sg_api_post` returns 22 on 4xx so `sg_retry` (via `SG_NO_RETRY_RC`) does not retry definitive errors. `clean` removes local artifacts (`export/`, TF state, tool cache); `clean --all` also removes config. `completion bash|zsh` prints a completion script (`cmd_completion`; commands/options come from `SG_COMMANDS`/`SG_OPTIONS`, keep them in sync with the parser and the host flags); like `clean` it always runs natively. Output is concise by default — `apply` captures terraform's init/plan output and shows only progress + the final summary (or the full log on failure, `TF_IN_AUTOMATION=1`/`-no-color`); `-v`/`--verbose` (`SG_VERBOSE`) streams everything and un-gates the convert detail lines. Colored logging via `sg_step`/`sg_log`/`sg_success`/`sg_warn`/`sg_err`; paths shown relative via `sg_rel` (all auto-off when not a TTY / `NO_COLOR`). +- `scripts/lib/` — sourced libraries (a `lib/` gitignore rule is negated for this dir): `prompt.sh` (`sg_ask`/`sg_confirm`/`sg_select`; tty or `SG_ANSWERS_FILE` for tests; `SG_NONINTERACTIVE=1`/`-y`/no TTY → defaults), `tfvars.sh` (`tfvars_get` cached hcl2json reads, `tfvars_write` renders the wizard's `W_*` vars), `tfc_api.sh` (`tfc_http`/`tfc_get_all` paginated, `tfc_list_{orgs,projects,workspaces}`, `tfc_token` with provider precedence, `require_tfc_auth`), `sg_api.sh` (`_sg_api` 0/22/1 contract + `SG_HTTP_CODE`, `sg_list_integrations`, `sg_*_exists`, `sg_list_workflows`, `sg_create_secret`, `sg_patch_workflow`, `wfgroup_*`), `wizard.sh` (`wizard_run`: 4 steps TFC → SG → defaults → review), `preflight.sh` (`preflight_run <apply|import|all>`, once per process), `state.sh` (`.sg/state.json`: `phases.<name>.input_sha`, `import.<seg>.{imported,failed,tf_fallback}`, `triggers.<seg>`, `secrets.<name>`; `run_phase` in migrate.sh skips unchanged phases), `report.sh` (`show_migration_summary` after apply, `show_import_plan` per-workflow table), `errors.sh` (`explain_api_error` regex→hint table, `SG_API_HINTS`), `checklist.sh` (`create_secret_stubs` → SG secret `tfc-<wf>-<VAR>` = `CHANGE_ME`, referenced as `${secret::<name>}` via PATCH + payload patch; `write_checklist` → `export/post-import-checklist.md`). Parallel jobs (`run_parallel`) run in subshells, so `do_import`/`do_set_triggers` write `.import-result.<seg>.json` / `.triggers-result.<seg>.json` into the export dir and the caller merges them into state. - `scripts/tools.sh` — sourced by the other scripts (sets `SG_REPO_ROOT` to its parent dir). Provides `sg_resolve` (PATH-or-cache), `sg_ensure_{jq,hcl2json,yajsv,sgcli}` (pinned downloads into `.sg/cached/`; `sg-cli` is the Go binary), `sg_retry`, the color vars + log helpers. The Docker image bundles these tools at build time (`yajsv` built from source so arm64 works), so in-container runs never download. - **Variable sets** — `scripts/enrich_variable_sets.sh <tfc-org> <payload...>` (run by `cmd_enrich`, which reads `tfOrg`/`tfHostname` from `terraform.tfvars` via `hcl2json` — variable sets live in the **TFC** org, not `SG_ORG`). Lists all sets + vars via the TFC API (Bearer token from the `terraform login` creds file or `TFE_TOKEN`), computes per-workspace effective vars (scope: global/project/workspace; precedence: priority sets > workspace vars > non-priority sets), and merges them into each workflow (matched by workspace name parsed from `TfStateFilePath`). Sensitive set vars and key conflicts are reported. Skipped by `--no-variable-sets`. - **Workflow groups** — each TFC project maps to an SG workflow group `tfc-<project-segment>`, created via `POST /api/v1/orgs/{org}/wfgrps/` (auth `Authorization: apikey <token>`) if missing. (Both the API calls and sg-cli honor the undocumented `SG_BASE_URL` for non-prod targets.) Groups are addressed by name, so no generated ID is tracked. `.sg/workflow-groups.json` (gitignored, optional) overrides the target group per segment (`{"<segment>": "<existing-group>"}`); override groups are not auto-created. `--no-create-groups` requires all groups to pre-exist. diff --git a/README.md b/README.md index b47ab17..d823216 100644 --- a/README.md +++ b/README.md @@ -21,23 +21,25 @@ export TFE_TOKEN=<TFC/TFE token> # long-lived API token (User/Team/Org token export SG_API_TOKEN=<your SG token> export SG_ORG=<your SG org> -./sg-migrate.sh init # scaffolds terraform.tfvars -# edit transformer/terraform-cloud/terraform.tfvars (org, integrations, workspaceOverrides) - -./sg-migrate.sh all # apply -> enrich -> convert -> validate -> import (prompts before importing) +./sg-migrate.sh init # guided setup: picks TFC org/workspaces, SG connectors, runners -> terraform.tfvars +./sg-migrate.sh all # preflight -> apply -> enrich -> convert -> validate -> import (shows a plan, asks before importing) ``` -That's it — no workflow-group mapping to fill in. Each TFC project is imported into an SG workflow group named `tfc-<project>`, **created automatically via the API** if it doesn't exist. The import prompt shows each group as `exists` or `create` before anything is written. +That's it — no IDs to look up and no workflow-group mapping to fill in. `init` lists what the tokens can see (TFC organisations and workspaces, SG VCS/cloud connectors and runner groups) and writes `terraform.tfvars` from your picks; `all` verifies every reference **before** running terraform (preflight), prints a migration summary after the export, shows a per-workflow import plan (create/update, Terraform version, runner, triggers, secrets), and ends with a **post-import checklist** of what still needs a human. Each TFC project is imported into an SG workflow group named `tfc-<project>`, **created automatically via the API** if it doesn't exist. -- Single phase: `./sg-migrate.sh apply|enrich|convert|validate|import|triggers`. Running `./sg-migrate.sh` with no command prints the help menu. +- Single phase: `./sg-migrate.sh preflight|apply|enrich|convert|validate|import|triggers|checklist`. Running `./sg-migrate.sh` with no command prints the help menu. +- **Resume.** `all` remembers what it completed (`.sg/state.json`) and skips phases whose inputs have not changed, so after a failure you just re-run it; files already imported in full are skipped and files with failures are retried. `--fresh` redoes everything. +- **Scope.** `--project <segment>` and `--workspace <name>` (repeatable) limit every phase to a subset — migrate one team first, then the rest. +- **Dry run.** `./sg-migrate.sh import --dry-run` prints the per-workflow plan and stops; nothing is created. +- **Sensitive variables** (which TFC never exposes) are recreated as SG secrets with the value `CHANGE_ME` and referenced from the workflows as `${secret::<name>}`; the checklist lists each one to fill in. Opt out with `--no-secret-stubs`. - TFC **Variable Set** variables are merged into the payloads automatically (the `enrich` phase, via the TFC API); skip it with `--no-variable-sets`. -- `./sg-migrate.sh clean` removes local working artifacts (`export/`, Terraform state, tool cache) for a fresh start; add `--all` to also remove config. `clean` always runs locally. +- `./sg-migrate.sh clean` removes local working artifacts (`export/`, Terraform state, run state, tool cache) for a fresh start; add `--all` to also remove config. `clean` always runs locally. - **Override** a project's target group (to reuse an existing group) in `.sg/workflow-groups.json`: `{"<project-segment>": "<existing-group>"}`. Override groups must already exist (they're not auto-created). -- Output is concise by default (terraform's plan/init noise is hidden; shown on error). Add `-v`/`--verbose` for full output. -- Flags: `-y` skip the import prompt (CI), `--concurrency N` parallel jobs, `--org NAME`, `--no-create-groups` require groups to pre-exist, `--build` rebuild the image, `--native`/`--local` force a local run even when Docker is available. -- Tuning via env: `SG_RETRIES`, `SG_TF_PARALLELISM`, `SG_NATIVE=1`. +- Output is concise by default (terraform's plan/init noise is hidden; shown on error). Add `-v`/`--verbose` for full output. Known API errors come with a hint naming the `terraform.tfvars` field to fix. +- Flags: `-y` skip the import prompt (CI; also makes `init` non-interactive), `--concurrency N` parallel jobs, `--org NAME`, `--no-create-groups` require groups to pre-exist, `--skip-preflight`, `--build` rebuild the image, `--native`/`--local` force a local run even when Docker is available. +- Tuning via env: `SG_RETRIES`, `SG_TF_PARALLELISM`, `SG_NATIVE=1`, `SG_UI_URL` (base URL for the checklist's links, default `https://app.stackguardian.io`). - Tab completion for the current shell session: `source <(./sg-migrate.sh completion zsh)` (or `bash`). `init` prints this line for your shell. -- TFC auth: set `TFE_TOKEN` (recommended — a long-lived token avoids re-running `terraform login`); otherwise the `terraform login` credentials file is mounted read-only into the container. SG/TFC tokens are passed as env vars. +- TFC auth: set `TFE_TOKEN` (recommended — a long-lived token avoids re-running `terraform login`); otherwise the `terraform login` credentials file is mounted read-only into the container. Tokens are only ever read from the environment; `init` never writes them to disk. The manual, step-by-step flow below remains supported for fine-grained control and is what each phase runs under the hood (the helper scripts live in `scripts/`). @@ -162,7 +164,9 @@ To update workflows with different details, re-run the sg-cli command with the m - **Workflow groups.** Each TFC project imports into an SG workflow group `tfc-<project>`, created via the API if missing (disable with `--no-create-groups`). Override the target group per project in `.sg/workflow-groups.json`; override groups must already exist. - **Variable Sets are migrated** (the `enrich` phase) — global, project-, and workspace-scoped sets are resolved per workspace with TFC precedence (priority sets override workspace vars; otherwise workspace vars win). **Sensitive** set variables can't be read from the API, so they're skipped and reported — recreate them as StackGuardian secrets. -- **Sensitive variables are skipped.** TFC never returns sensitive values via the API, so they are omitted from the payload and listed in `migration-summary.md`. Recreate them as StackGuardian secrets. +- **Sensitive variables become placeholder secrets.** TFC never returns sensitive values via the API. The export omits them (listed in `migration-summary.md`); after import the orchestrator creates an SG secret `tfc-<workflow>-<VAR>` with the value `CHANGE_ME` for each, references it from the workflow (`${secret::<name>}`, as an environment variable or IaC input) and lists it in `export/post-import-checklist.md`. Set the real values in the SG UI. `--no-secret-stubs` leaves SG secrets untouched. - **Terraform version fallback (FOSS ceiling).** Workspaces set to `latest` or a version constraint use `SGDefaultTerraformVersion` at export time. Pinned versions are carried over as-is and tried first at import, so a custom runtime image or private runner that ships that binary keeps working. StackGuardian's _managed_ runtimes only go up to **1.5.7**, the last MPL-licensed (FOSS) Terraform release; newer versions are BSL-licensed and are not bundled. When the API rejects a workflow for that reason, the importer **automatically re-imports it with `SGDefaultTerraformVersion`**, patches the payload file to match, prints a notice, and records each case in `export/terraform-version-fallbacks.log`. Those workflows run a different Terraform than they did in TFC, so check compatibility before the first run. To keep a newer version, set `workspaceOverrides[<name>].terraformVersion` to a binary path mounted from a private runner (or use a custom runtime container template) and re-import. - **State export is idempotent.** Re-running `terraform apply` only pulls state for workspaces not yet exported. Set `forceStateRefresh = true` to re-pull everything. Workspaces not using `remote` execution may export incomplete state — see the summary. -- **Workflow naming.** `ResourceName` currently mirrors the TFC workspace name. Confirm it satisfies StackGuardian's naming rules before import. +- **Workflow naming.** `ResourceName` mirrors the TFC workspace name, sanitized to StackGuardian's rules (1-100 chars, `[-a-zA-Z0-9_]`); any rename is listed in the summary and the checklist. +- **Preflight.** `apply`, `import` and `all` first verify the TFC and SG tokens, the TFC org and workspace selection, and that every connector, secret and runner group referenced in `terraform.tfvars` exists. Fix what it reports (or re-run `init`); `--skip-preflight` bypasses it. +- **Post-import checklist.** `export/post-import-checklist.md` (also printed) collects the secrets to fill in, failed imports, Terraform version fallbacks, failed VCS triggers, missing state exports and renames, with links into the SG UI. diff --git a/scripts/migrate.sh b/scripts/migrate.sh index 2ad0bf1..48ad44d 100755 --- a/scripts/migrate.sh +++ b/scripts/migrate.sh @@ -77,11 +77,12 @@ Commands: triggers Register VCS triggers for already-imported workflows (second pass) checklist Create placeholder secrets for skipped sensitive variables and write export/post-import-checklist.md (runs automatically after import) - all apply -> enrich -> convert -> validate -> import + all preflight -> apply -> enrich -> convert -> validate -> import -> checklist + (resumes where a previous run stopped; --fresh to redo everything) clean Remove local working artifacts for a fresh start (export/, TF state, tool cache). Add --all to also remove config (terraform.tfvars, mapping). completion Print a shell completion script for the current session: - \`source <($0 completion zsh)\` (or bash) + \`source <($PROG completion zsh)\` (or bash) Each TFC project maps to an SG workflow group named tfc-<project>, created via the API if missing. Override a project's target group in .sg/workflow-groups.json From 734656104740fffa1e2ccdc89165278a663b1549 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adis=20Halilovi=C4=87?= <adis.halilovic@stackguardian.io> Date: Mon, 7 Sep 2026 09:38:14 +0200 Subject: [PATCH 21/71] fix: read connector kind from Settings.kind in the init wizard The integrations list has no ResourceType; the kind (GITHUB_COM, AWS_RBAC, ...) is Settings.kind. Map it from there, make the type filters null-safe, and only treat msg/data as the item list when it is actually an array. --- scripts/lib/sg_api.sh | 9 +++++---- scripts/lib/wizard.sh | 4 ++-- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/scripts/lib/sg_api.sh b/scripts/lib/sg_api.sh index 9937a4b..28f24d4 100644 --- a/scripts/lib/sg_api.sh +++ b/scripts/lib/sg_api.sh @@ -81,7 +81,7 @@ sg_workflow_exists() { [ "$(sg_http_code GET "$(wf_url "$1" "$2")")" = "200" ]; sg_list_workflows() { local body if body="$(sg_api_get "$(sg_org_url)/wfgrps/$1/wfs/listall/" 2>/dev/null)"; then - printf '%s' "$body" | "$(sg_resolve jq sg_ensure_jq)" -c '[(.msg // .data // [])[] | .ResourceName] | map(select(. != null))' + printf '%s' "$body" | "$(sg_resolve jq sg_ensure_jq)" -c '[(if (.msg | type) == "array" then .msg elif (.data | type) == "array" then .data elif type == "array" then . else [] end)[] | .ResourceName] | map(select(. != null))' else echo '[]' fi @@ -93,10 +93,11 @@ sg_patch_workflow() { sg_api_patch "$(wf_url "$1" "$2")" "$3"; } # --- integrations (connectors) -------------------------------------------- # sg_list_integrations — [{name, type}, ...] for the org (fails on error). +# The connector kind (GITHUB_COM, AWS_RBAC, ...) is Settings.kind in the API. sg_list_integrations() { local body body="$(sg_api_get "$(sg_org_url)/integrations/listall/")" || return $? - printf '%s' "$body" | "$(sg_resolve jq sg_ensure_jq)" -c '[(.msg // .data // [])[] | {name: .ResourceName, type: .ResourceType}] | map(select(.name != null))' + printf '%s' "$body" | "$(sg_resolve jq sg_ensure_jq)" -c '[(if (.msg | type) == "array" then .msg elif (.data | type) == "array" then .data elif type == "array" then . else [] end)[] | {name: (.ResourceName // .Id // ""), type: (.Settings.kind // .kind // .ResourceType // "")}] | map(select(.name != ""))' } # sg_integration_exists <name-or-/integrations/name> — exit 0 when it exists. @@ -115,7 +116,7 @@ sg_runnergroup_exists() { [ "$(sg_http_code GET "$(sg_org_url)/runnergroups/$1/" sg_list_runnergroups() { local body body="$(sg_api_get "$(sg_org_url)/runnergroups/listall/" 2>/dev/null)" || return 1 - printf '%s' "$body" | "$(sg_resolve jq sg_ensure_jq)" -c '[(.msg // .data // [])[] | .ResourceName] | map(select(. != null))' + printf '%s' "$body" | "$(sg_resolve jq sg_ensure_jq)" -c '[(if (.msg | type) == "array" then .msg elif (.data | type) == "array" then .data elif type == "array" then . else [] end)[] | .ResourceName] | map(select(. != null))' } # --- secrets --------------------------------------------------------------- @@ -124,7 +125,7 @@ sg_list_runnergroups() { sg_secret_exists() { local body body="$(sg_api_get "$(sg_org_url)/secrets/listall/" 2>/dev/null)" || return 1 - printf '%s' "$body" | "$(sg_resolve jq sg_ensure_jq)" -e --arg n "$1" '[(.msg // .data // [])[] | .ResourceName] | index($n) != null' >/dev/null + printf '%s' "$body" | "$(sg_resolve jq sg_ensure_jq)" -e --arg n "$1" '[(if (.msg | type) == "array" then .msg elif (.data | type) == "array" then .data elif type == "array" then . else [] end)[] | .ResourceName] | index($n) != null' >/dev/null } # sg_create_secret <name> <value> [description] diff --git a/scripts/lib/wizard.sh b/scripts/lib/wizard.sh index 5096092..f388ac4 100644 --- a/scripts/lib/wizard.sh +++ b/scripts/lib/wizard.sh @@ -122,7 +122,7 @@ wizard_sg() { # VCS connector -> integration id, source kind, repo prefix. vcs="" - [ "$W_SG_DISCOVERY" -eq 1 ] && vcs="$(printf '%s' "$ints" | "$jqb" -r '.[] | select(.type | test("^(GITHUB_COM|GITHUB_APP_CUSTOM|GITLAB_COM|BITBUCKET_ORG|AZURE_DEVOPS|GIT_OTHER)$")) | "\(.name)|\(.type)"')" + [ "$W_SG_DISCOVERY" -eq 1 ] && vcs="$(printf '%s' "$ints" | "$jqb" -r '.[] | select((.type // "") | test("^(GITHUB_COM|GITHUB_APP_CUSTOM|GITLAB_COM|BITBUCKET_ORG|AZURE_DEVOPS|GIT_OTHER)$")) | "\(.name)|\(.type)"')" if [ -n "$vcs" ]; then # shellcheck disable=SC2046 pick="$(SG_SELECT_OTHER=1 sg_select "Which VCS connector should clone the repositories?" $(printf '%s\n' "$vcs" | tr '\n' ' '))" || return 1 @@ -142,7 +142,7 @@ wizard_sg() { # Cloud connector -> DeploymentPlatformConfig. cloud="" - [ "$W_SG_DISCOVERY" -eq 1 ] && cloud="$(printf '%s' "$ints" | "$jqb" -r '.[] | select(.type | test("^(AWS|AZURE|GCP)_")) | "\(.name)|\(.type)"')" + [ "$W_SG_DISCOVERY" -eq 1 ] && cloud="$(printf '%s' "$ints" | "$jqb" -r '.[] | select((.type // "") | test("^(AWS|AZURE|GCP)_")) | "\(.name)|\(.type)"')" if [ -n "$cloud" ]; then # shellcheck disable=SC2046 pick="$(SG_SELECT_OTHER=1 sg_select "Which cloud connector should the workflows deploy with?" $(printf '%s\n' "$cloud" | tr '\n' ' ') "skip|decide later (leaves a placeholder to edit)")" || return 1 From 1c03650c8d4dd537626f67741bffe73cc2baad81 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adis=20Halilovi=C4=87?= <adis.halilovic@stackguardian.io> Date: Mon, 7 Sep 2026 09:40:45 +0200 Subject: [PATCH 22/71] fix: init wrote invalid tfvars for an empty approver list _w_csv_json emitted "[]" twice when the list was empty (grep -v on empty input trips the fallback under pipefail), so the generated file was not valid HCL and preflight reported every field as missing. Validate the file right after writing it, report an unparsable tfvars as such in preflight, remember the SG org / API host chosen in the wizard so a new shell does not need SG_ORG, pretty-print objects in the generated file and separate menu values from their descriptions. --- scripts/lib/preflight.sh | 5 +++++ scripts/lib/prompt.sh | 2 +- scripts/lib/tfvars.sh | 15 +++++++++++++-- scripts/lib/wizard.sh | 13 +++++++++++-- scripts/migrate.sh | 9 +++++++++ 5 files changed, 39 insertions(+), 5 deletions(-) diff --git a/scripts/lib/preflight.sh b/scripts/lib/preflight.sh index c0bb2ad..4e8725f 100644 --- a/scripts/lib/preflight.sh +++ b/scripts/lib/preflight.sh @@ -190,6 +190,11 @@ preflight_run() { PF_FAIL=0 PF_WARN=0 PF_TFC_WORKSPACES="" + local parse_err + if ! parse_err="$(tfvars_valid)"; then + pf_fail "$(sg_rel "$TFVARS") is not valid HCL: ${parse_err:-parse error}" + die "fix the file (or re-run '$PROG init') and try again." + fi case "$ctx" in apply) preflight_tfc diff --git a/scripts/lib/prompt.sh b/scripts/lib/prompt.sh index 3ca5540..7e754e5 100644 --- a/scripts/lib/prompt.sh +++ b/scripts/lib/prompt.sh @@ -95,7 +95,7 @@ sg_select() { printf '%s? %s%s\n' "$C_BOLD" "$q" "$C_RESET" >&2 for ((i = 0; i < n; i++)); do if [ -n "${descs[i]}" ]; then - printf ' %s%2d)%s %s %s%s%s\n' "$C_CYAN" "$((i + 1))" "$C_RESET" "${vals[i]}" "$C_DIM" "${descs[i]}" "$C_RESET" >&2 + printf ' %s%2d)%s %-10s %s%s%s\n' "$C_CYAN" "$((i + 1))" "$C_RESET" "${vals[i]}" "$C_DIM" "— ${descs[i]}" "$C_RESET" >&2 else printf ' %s%2d)%s %s\n' "$C_CYAN" "$((i + 1))" "$C_RESET" "${vals[i]}" >&2 fi diff --git a/scripts/lib/tfvars.sh b/scripts/lib/tfvars.sh index 8bfc766..f37f8b6 100644 --- a/scripts/lib/tfvars.sh +++ b/scripts/lib/tfvars.sh @@ -35,6 +35,17 @@ tfvars_get_json() { # tfvars_invalidate — forget the cached conversion (after writing the file). tfvars_invalidate() { _TFVARS_JSON=""; } +# tfvars_valid — exit 0 when the file exists and hcl2json can parse it; +# the parser's message is printed on stdout otherwise. +tfvars_valid() { + [ -f "$TFVARS" ] || return 1 + # shellcheck disable=SC2069 # stderr is the result; stdout (the JSON) is discarded + "$(sg_resolve hcl2json sg_ensure_hcl2json)" "$TFVARS" 2>&1 >/dev/null +} + +# _tfvars_hcl <json> — pretty-print a JSON value so it reads like HCL in the file. +_tfvars_hcl() { printf '%s' "$1" | "$(sg_resolve jq sg_ensure_jq)" --indent 2 '.' 2>/dev/null || printf '%s' "$1"; } + # tfvars_write <dest> — render terraform.tfvars from W_* variables set by the # wizard (lists/objects are passed as compact JSON, which HCL accepts). Keeps # the same order and comments as terraform.tfvars.example so the file stays @@ -80,11 +91,11 @@ SGDefaultIACVCSRepoPrefix = "$W_REPO_PREFIX" SGDefaultVCSAuthIntegrationID = "$W_VCS_INTEGRATION" # Cloud connector the workflows deploy with -SGDefaultDeploymentPlatformConfig = $W_DPC_JSON +SGDefaultDeploymentPlatformConfig = $(_tfvars_hcl "$W_DPC_JSON") # Runners for every workflow: { type = "shared" } for SG-hosted runners, or # { type = "private", names = ["<runner-group>"] } for a private runner group. -SGDefaultRunnerConstraints = $W_RUNNER_JSON +SGDefaultRunnerConstraints = $(_tfvars_hcl "$W_RUNNER_JSON") # Choose from: GITHUB_COM, BITBUCKET_ORG, GITLAB_COM, AZURE_DEVOPS, GIT_OTHER SGDefaultSourceConfigDestKind = "$W_DEST_KIND" diff --git a/scripts/lib/wizard.sh b/scripts/lib/wizard.sh index f388ac4..edbbffe 100644 --- a/scripts/lib/wizard.sh +++ b/scripts/lib/wizard.sh @@ -10,9 +10,11 @@ _w_default() { local v; v="$(tfvars_get "$1")"; printf '%s' "${v:-$2}"; } # _w_csv_json <csv> — "a, b" -> ["a","b"]; empty -> []. _w_csv_json() { - local jqb + local jqb out jqb="$(sg_resolve jq sg_ensure_jq)" - printf '%s' "$1" | tr ',' '\n' | sed 's/^ *//; s/ *$//' | grep -v '^$' | "$jqb" -R . | "$jqb" -sc . 2>/dev/null || echo '[]' + # (grep -v exits 1 on empty input; never let that trigger the fallback twice) + out="$(printf '%s' "$1" | tr ',' '\n' | sed 's/^ *//; s/ *$//' | { grep -v '^$' || true; } | "$jqb" -R . | "$jqb" -sc . 2>/dev/null)" + printf '%s' "${out:-[]}" } # _w_repo_prefix_for <sourceConfigDestKind> — proposed repo URL prefix. @@ -226,5 +228,12 @@ wizard_run() { sg_log "previous file kept as $(sg_rel "$TFVARS.bak")" fi tfvars_write "$TFVARS" + if ! tfvars_valid; then + sg_err "the generated $(sg_rel "$TFVARS") is not valid HCL — this is a bug in the wizard; the file was kept for inspection" + return 1 + fi + # Remember the SG org (and API host) for later phases, so users don't have to + # export SG_ORG again in a new shell. Tokens are never stored. + state_update '.config = ((.config // {}) + {sg_org: $o, sg_base_url: $u})' --arg o "$ORG" --arg u "$SG_BASE_URL" sg_success "wrote $(sg_rel "$TFVARS")" } diff --git a/scripts/migrate.sh b/scripts/migrate.sh index 48ad44d..a7df492 100755 --- a/scripts/migrate.sh +++ b/scripts/migrate.sh @@ -30,6 +30,7 @@ source "$SCRIPT_DIR/lib/errors.sh" # shellcheck source=lib/checklist.sh source "$SCRIPT_DIR/lib/checklist.sh" + # SCRIPT_DIR holds the sibling scripts; SG_REPO_ROOT (from tools.sh) is the repo # root used for all repo-relative paths. TRANSFORMER_DIR="$SG_REPO_ROOT/transformer/terraform-cloud" @@ -38,6 +39,7 @@ PROG="${SG_PROG:-$0}" EXPORT_DIR="${SG_EXPORT_DIR:-$SG_REPO_ROOT/export}" MAPPING="${SG_WFGROUP_MAP:-$SG_REPO_ROOT/.sg/workflow-groups.json}" ORG="${SG_ORG:-}" +SG_BASE_URL_SET="${SG_BASE_URL:-}" SG_BASE_URL="${SG_BASE_URL:-https://api.app.stackguardian.io}" ASSUME_YES=0 PURGE=0 @@ -59,6 +61,13 @@ TF_PARALLELISM="${SG_TF_PARALLELISM:-20}" RETRIES="${SG_RETRIES:-4}" RETRY_BASE="${SG_RETRY_BASE:-2}" PF=() +# The init wizard remembers the SG org / API host in .sg/state.json so a new +# shell without SG_ORG still works; flags and env always win. +if [ -z "$ORG" ] && [ -f "$STATE_FILE" ]; then + ORG="$(state_read | "$(sg_resolve jq sg_ensure_jq)" -r '.config.sg_org // empty' 2>/dev/null || true)" + [ -n "$ORG" ] && [ -z "${SG_BASE_URL_SET:-}" ] && SG_BASE_URL="$(state_read | "$(sg_resolve jq sg_ensure_jq)" -r --arg d "$SG_BASE_URL" '.config.sg_base_url // $d' 2>/dev/null || echo "$SG_BASE_URL")" +fi + usage() { cat >&2 <<EOF From 0f071fa506d3b2839fdf5c9296412a2223c8122b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adis=20Halilovi=C4=87?= <adis.halilovic@stackguardian.io> Date: Mon, 7 Sep 2026 09:45:41 +0200 Subject: [PATCH 23/71] fix: completion follows the shell you are actually running The hint used $SHELL, which is the login shell and was wrong for a bash login shell running zsh, so the bash script got sourced into zsh and broke on the first Tab. sg-migrate.sh now detects the parent shell and 'completion' with no argument prints the matching script; each script also hands off to the other shell's version when sourced by mistake. --- README.md | 2 +- scripts/migrate.sh | 29 ++++++++++++++++++++++------- sg-migrate.sh | 7 ++++++- 3 files changed, 29 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index d823216..2cd8050 100644 --- a/README.md +++ b/README.md @@ -38,7 +38,7 @@ That's it — no IDs to look up and no workflow-group mapping to fill in. `init` - Output is concise by default (terraform's plan/init noise is hidden; shown on error). Add `-v`/`--verbose` for full output. Known API errors come with a hint naming the `terraform.tfvars` field to fix. - Flags: `-y` skip the import prompt (CI; also makes `init` non-interactive), `--concurrency N` parallel jobs, `--org NAME`, `--no-create-groups` require groups to pre-exist, `--skip-preflight`, `--build` rebuild the image, `--native`/`--local` force a local run even when Docker is available. - Tuning via env: `SG_RETRIES`, `SG_TF_PARALLELISM`, `SG_NATIVE=1`, `SG_UI_URL` (base URL for the checklist's links, default `https://app.stackguardian.io`). -- Tab completion for the current shell session: `source <(./sg-migrate.sh completion zsh)` (or `bash`). `init` prints this line for your shell. +- Tab completion for the current shell session: `source <(./sg-migrate.sh completion)` (bash/zsh detected; or pass `bash`/`zsh`). `init` prints this line. - TFC auth: set `TFE_TOKEN` (recommended — a long-lived token avoids re-running `terraform login`); otherwise the `terraform login` credentials file is mounted read-only into the container. Tokens are only ever read from the environment; `init` never writes them to disk. The manual, step-by-step flow below remains supported for fine-grained control and is what each phase runs under the hood (the helper scripts live in `scripts/`). diff --git a/scripts/migrate.sh b/scripts/migrate.sh index a7df492..24ad72a 100755 --- a/scripts/migrate.sh +++ b/scripts/migrate.sh @@ -90,8 +90,8 @@ Commands: (resumes where a previous run stopped; --fresh to redo everything) clean Remove local working artifacts for a fresh start (export/, TF state, tool cache). Add --all to also remove config (terraform.tfvars, mapping). - completion Print a shell completion script for the current session: - \`source <($PROG completion zsh)\` (or bash) + completion Print a completion script for your shell (bash/zsh auto-detected): + \`source <($PROG completion)\` Each TFC project maps to an SG workflow group named tfc-<project>, created via the API if missing. Override a project's target group in .sg/workflow-groups.json @@ -250,9 +250,14 @@ cmd_init() { # A child process cannot register completions in the parent shell, so this only # prints the one-liner (nothing is written to the user's rc files). completion_hint() { - local sh - case "$(basename "${SHELL:-}")" in bash) sh=bash ;; *) sh=zsh ;; esac - sg_log "tab completion for this shell session: source <(./sg-migrate.sh completion $sh)" + sg_log "tab completion for this shell session: source <(./sg-migrate.sh completion)" +} + +# current_shell — bash|zsh: the shell the user is typing in (detected by +# sg-migrate.sh from its parent process), else the login shell. +current_shell() { + case "${SG_SHELL:-}" in bash | zsh) printf '%s' "$SG_SHELL"; return ;; esac + case "$(basename "${SHELL:-}")" in bash) printf 'bash' ;; *) printf 'zsh' ;; esac } # --- workflow-group mapping (API helpers live in lib/sg_api.sh) ------------- @@ -703,11 +708,16 @@ SG_OPTIONS="--org --export-dir --mapping --concurrency --no-create-groups --no-v # migrate.sh to stdout. Both shells fall back to the basename when the command # is invoked by path, so ./sg-migrate.sh completes too. cmd_completion() { - local shell="${1:-}" + local shell="${1:-$(current_shell)}" case "$shell" in bash) cat <<BASH # bash completion for sg-migrate.sh — generated by: sg-migrate.sh completion bash +# Sourced from zsh by mistake? Load the zsh version instead. +if [ -n "\${ZSH_VERSION:-}" ]; then + source <("$SG_REPO_ROOT/sg-migrate.sh" completion zsh) + return 0 +fi _sg_migrate() { local cur prev cmds opts w has_cmd=0 cur="\${COMP_WORDS[COMP_CWORD]}"; prev="\${COMP_WORDS[COMP_CWORD-1]}" @@ -735,6 +745,11 @@ BASH cat <<ZSH #compdef sg-migrate.sh migrate.sh # zsh completion for sg-migrate.sh — generated by: sg-migrate.sh completion zsh +# Sourced from bash by mistake? Load the bash version instead. +if [ -n "\${BASH_VERSION:-}" ]; then + source <("$SG_REPO_ROOT/sg-migrate.sh" completion bash) + return 0 +fi _sg_migrate() { local -a cmds cmds=( @@ -781,7 +796,7 @@ _sg_migrate() { compdef _sg_migrate sg-migrate.sh migrate.sh ZSH ;; - *) die "usage: $PROG completion <bash|zsh>" ;; + *) die "usage: $PROG completion [bash|zsh] (default: the shell you are running)" ;; esac } diff --git a/sg-migrate.sh b/sg-migrate.sh index 0d96281..8a6f500 100755 --- a/sg-migrate.sh +++ b/sg-migrate.sh @@ -40,6 +40,11 @@ done # Let migrate.sh print the name the user actually invoked in its help/hints. export SG_PROG="${0##*/}" case "$0" in */*) SG_PROG="./${0##*/}" ;; esac +# The shell the user is typing in (for the completion hint): the parent process, +# not $SHELL, which is only the login shell and is often wrong (bash vs zsh). +SG_SHELL="$(ps -p "$PPID" -o comm= 2>/dev/null | sed 's/^-//; s#.*/##')" +case "$SG_SHELL" in bash | zsh) ;; *) SG_SHELL="$(basename "${SHELL:-zsh}")" ;; esac +export SG_SHELL if [ "$NATIVE" = "1" ] || ! command -v docker >/dev/null 2>&1; then [ "$NATIVE" = "1" ] || sg_warn "docker not found; running natively" @@ -54,7 +59,7 @@ fi DOCKER_ARGS=(--rm -i -v "$SCRIPT_DIR:/app" -w /app -e SG_API_TOKEN -e SG_ORG -e SG_BASE_URL -e SG_CONCURRENCY -e SG_RETRIES -e SG_TF_PARALLELISM - -e TFE_TOKEN -e SG_PROG -e SG_NONINTERACTIVE -e SG_UI_URL) + -e TFE_TOKEN -e SG_PROG -e SG_SHELL -e SG_NONINTERACTIVE -e SG_UI_URL) # Interactive TTY only when attached to one (so the confirmation prompt works, # but CI/non-tty invocations still run — use -y there). From 7c2eb1f9ba8036491d3d54014fbdc4f7d9cb2a3d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adis=20Halilovi=C4=87?= <adis.halilovic@stackguardian.io> Date: Mon, 7 Sep 2026 09:45:42 +0200 Subject: [PATCH 24/71] feat: SG_TFVARS override for the tfvars path --- scripts/migrate.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/migrate.sh b/scripts/migrate.sh index 24ad72a..472f21b 100755 --- a/scripts/migrate.sh +++ b/scripts/migrate.sh @@ -34,7 +34,7 @@ source "$SCRIPT_DIR/lib/checklist.sh" # SCRIPT_DIR holds the sibling scripts; SG_REPO_ROOT (from tools.sh) is the repo # root used for all repo-relative paths. TRANSFORMER_DIR="$SG_REPO_ROOT/transformer/terraform-cloud" -TFVARS="$TRANSFORMER_DIR/terraform.tfvars" +TFVARS="${SG_TFVARS:-$TRANSFORMER_DIR/terraform.tfvars}" PROG="${SG_PROG:-$0}" EXPORT_DIR="${SG_EXPORT_DIR:-$SG_REPO_ROOT/export}" MAPPING="${SG_WFGROUP_MAP:-$SG_REPO_ROOT/.sg/workflow-groups.json}" From 8274d89db6e00e177dff43b0fd45308fba06969b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adis=20Halilovi=C4=87?= <adis.halilovic@stackguardian.io> Date: Mon, 7 Sep 2026 09:49:35 +0200 Subject: [PATCH 25/71] fix: import plan survives an unexpected listall shape; shorter init MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The per-workflow plan passed whatever sg_list_workflows returned to jq --argjson and crashed when the response was not a plain array. List helpers now always yield an array (also unwrapping data.Workflows) and the plan guards the value. The wizard no longer asks for the profile name (not required), the repo URL prefix, approvers or the fallback Terraform version — they take sensible defaults and are edited in terraform.tfvars. sg-migrate.sh says when it runs in Docker. --- scripts/lib/report.sh | 1 + scripts/lib/sg_api.sh | 7 +++---- scripts/lib/wizard.sh | 27 +++++++++++---------------- sg-migrate.sh | 1 + 4 files changed, 16 insertions(+), 20 deletions(-) diff --git a/scripts/lib/report.sh b/scripts/lib/report.sh index a670516..10b81db 100644 --- a/scripts/lib/report.sh +++ b/scripts/lib/report.sh @@ -61,6 +61,7 @@ show_import_plan() { seg="$(seg_of "$f")" grp="$(group_for "$seg")" existing="$(sg_list_workflows "$grp")" + printf '%s' "$existing" | "$JQ_BIN" -e 'type == "array"' >/dev/null 2>&1 || existing='[]' rows="$("$JQ_BIN" -r --argjson ex "$existing" --arg def "$SG_DEFAULT_TF_VERSION" --argjson ws "$(ws_filter_json)" \ --slurpfile sum "${summary:-/dev/null}" ' ($sum[0] // {}) as $S diff --git a/scripts/lib/sg_api.sh b/scripts/lib/sg_api.sh index 28f24d4..f165f6b 100644 --- a/scripts/lib/sg_api.sh +++ b/scripts/lib/sg_api.sh @@ -79,12 +79,11 @@ sg_workflow_exists() { [ "$(sg_http_code GET "$(wf_url "$1" "$2")")" = "200" ]; # sg_list_workflows <group> — ["wf-name", ...] in the group ([] on 404). sg_list_workflows() { - local body + local body out if body="$(sg_api_get "$(sg_org_url)/wfgrps/$1/wfs/listall/" 2>/dev/null)"; then - printf '%s' "$body" | "$(sg_resolve jq sg_ensure_jq)" -c '[(if (.msg | type) == "array" then .msg elif (.data | type) == "array" then .data elif type == "array" then . else [] end)[] | .ResourceName] | map(select(. != null))' - else - echo '[]' + out="$(printf '%s' "$body" | "$(sg_resolve jq sg_ensure_jq)" -c '[(if (.msg | type) == "array" then .msg elif (.data | type) == "array" then .data elif (.data.Workflows? | type) == "array" then .data.Workflows elif type == "array" then . else [] end)[] | (.ResourceName // .Id // empty)]' 2>/dev/null)" fi + printf '%s' "${out:-[]}" } # sg_patch_workflow <group> <wf> <json> — PATCH a workflow. diff --git a/scripts/lib/wizard.sh b/scripts/lib/wizard.sh index edbbffe..c6c20b2 100644 --- a/scripts/lib/wizard.sh +++ b/scripts/lib/wizard.sh @@ -140,7 +140,8 @@ wizard_sg() { GITHUB_COM | GITLAB_COM | BITBUCKET_ORG | AZURE_DEVOPS | GIT_OTHER) W_DEST_KIND="$kind" ;; *) W_DEST_KIND="$(sg_select "VCS provider kind" GITHUB_COM GITLAB_COM BITBUCKET_ORG AZURE_DEVOPS GIT_OTHER)" || return 1 ;; esac - W_REPO_PREFIX="$(sg_ask "Repository URL prefix" "$(_w_default .SGDefaultIACVCSRepoPrefix "$(_w_repo_prefix_for "$W_DEST_KIND")")")" || return 1 + # Repo URL prefix follows the connector kind; editable in terraform.tfvars. + W_REPO_PREFIX="$(_w_default .SGDefaultIACVCSRepoPrefix "$(_w_repo_prefix_for "$W_DEST_KIND")")" # Cloud connector -> DeploymentPlatformConfig. cloud="" @@ -154,12 +155,11 @@ wizard_sg() { kind="" fi if [ -z "$pick" ] || [ "$pick" = "skip" ]; then - W_DPC_JSON='[{"kind":"AWS_RBAC","config":{"integrationId":"/integrations/CHANGE_ME","profileName":"default"}}]' + W_DPC_JSON='[{"kind":"AWS_RBAC","config":{"integrationId":"/integrations/CHANGE_ME"}}]' W_DPC_PLACEHOLDER=1 else [ -n "$kind" ] || kind="$(sg_select "Connector kind" AWS_RBAC AWS_STATIC AWS_OIDC AZURE_STATIC AZURE_OIDC AZURE_MANAGED_ID_OIDC GCP_STATIC GCP_OIDC)" || return 1 - name="$(sg_ask "Profile name" "$(_w_default .SGDefaultDeploymentPlatformConfig[0].config.profileName default)")" || return 1 - W_DPC_JSON="$("$jqb" -nc --arg k "$kind" --arg i "/integrations/${pick#/integrations/}" --arg p "$name" '[{kind:$k, config:{integrationId:$i, profileName:$p}}]')" + W_DPC_JSON="$("$jqb" -nc --arg k "$kind" --arg i "/integrations/${pick#/integrations/}" '[{kind:$k, config:{integrationId:$i}}]')" W_DPC_PLACEHOLDER=0 fi @@ -187,19 +187,14 @@ wizard_sg() { # --- step 3: policy ------------------------------------------------------------ wizard_policy() { - local approvers sg_step "3/4 Workflow defaults" - approvers="$(sg_ask "Approver emails for plans (comma-separated, empty for none)" "$(tfvars_get_json .SGDefaultWfApprovers | "$(sg_resolve jq sg_ensure_jq)" -r 'if . == null then "" else join(", ") end')")" || return 1 - W_APPROVERS_JSON="$(_w_csv_json "$approvers")" + # Approvers, repo prefix and the fallback Terraform version are plain values + # with sensible defaults — edit them in terraform.tfvars if needed. + W_APPROVERS_JSON="$(tfvars_get_json .SGDefaultWfApprovers)" + [ "$W_APPROVERS_JSON" = "null" ] && W_APPROVERS_JSON='[]' + W_TF_VERSION="$(_w_default .SGDefaultTerraformVersion TERRAFORM-1.5.7)" if sg_confirm "Export Terraform state for each workspace?" "$([ "$(_w_default .exportStateFiles true)" = "false" ] && echo N || echo Y)"; then W_EXPORT_STATE=true; else W_EXPORT_STATE=false; fi if sg_confirm "Pre-configure VCS triggers (push / pull-request runs) from the TFC settings?" "$([ "$(_w_default .SGDefaultEnableVCSTriggers true)" = "false" ] && echo N || echo Y)"; then W_TRIGGERS=true; else W_TRIGGERS=false; fi - sg_dim "StackGuardian bundles managed Terraform only up to 1.5.7 (last MPL/FOSS release);" - sg_dim "workspaces pinned above it are imported with the fallback version below." - while :; do - W_TF_VERSION="$(sg_ask "Fallback Terraform version" "$(_w_default .SGDefaultTerraformVersion TERRAFORM-1.5.7)")" || return 1 - [[ "$W_TF_VERSION" =~ ^TERRAFORM-[0-9]+\.[0-9]+\.[0-9]+$ ]] && break - sg_warn "use the SG format, e.g. TERRAFORM-1.5.7" - done } # --- step 4: review + write ---------------------------------------------------- @@ -212,10 +207,10 @@ wizard_review() { row "VCS connector" "$W_VCS_INTEGRATION ($W_DEST_KIND, $W_REPO_PREFIX)" row "Cloud connector" "$W_DPC_JSON" row "Runners" "$W_RUNNER_JSON" - row "Approvers" "$W_APPROVERS_JSON" row "State export / triggers" "$W_EXPORT_STATE / $W_TRIGGERS" - row "Fallback Terraform" "$W_TF_VERSION${W_WS_ABOVE_CEILING:+ ($W_WS_ABOVE_CEILING workspace(s) above 1.5.7 will use it)}" + row "Fallback Terraform" "$W_TF_VERSION${W_WS_ABOVE_CEILING:+ ($W_WS_ABOVE_CEILING workspace(s) pinned above 1.5.7 (the last FOSS runtime SG bundles) will use it)}" [ "${W_DPC_PLACEHOLDER:-0}" -eq 1 ] && sg_warn "cloud connector left as a placeholder — edit SGDefaultDeploymentPlatformConfig in $(sg_rel "$TFVARS") before 'apply'" + sg_dim "approvers, repo URL prefix and the fallback version can be edited in $(sg_rel "$TFVARS")" sg_confirm "Write $(sg_rel "$TFVARS")?" Y } diff --git a/sg-migrate.sh b/sg-migrate.sh index 8a6f500..9374ceb 100755 --- a/sg-migrate.sh +++ b/sg-migrate.sh @@ -50,6 +50,7 @@ if [ "$NATIVE" = "1" ] || ! command -v docker >/dev/null 2>&1; then [ "$NATIVE" = "1" ] || sg_warn "docker not found; running natively" exec "$SCRIPT_DIR/scripts/migrate.sh" ${ARGS[@]+"${ARGS[@]}"} fi +sg_dim "running in Docker ($IMAGE); pass --native to run on this machine instead" if [ "$BUILD" = "1" ] || ! docker image inspect "$IMAGE" >/dev/null 2>&1; then sg_log "building image $IMAGE ..." From b7e8f4af19ae6af2f90055c0c7de2d16a5e687e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adis=20Halilovi=C4=87?= <adis.halilovic@stackguardian.io> Date: Mon, 7 Sep 2026 09:54:07 +0200 Subject: [PATCH 26/71] fix: 'all' continues after the init wizard instead of stopping When terraform.tfvars is missing, all runs the wizard and then asks whether to continue right away or edit the file first; only the non-interactive template copy still stops, since it holds placeholders. --- scripts/migrate.sh | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/scripts/migrate.sh b/scripts/migrate.sh index 472f21b..9ceb4bf 100755 --- a/scripts/migrate.sh +++ b/scripts/migrate.sh @@ -883,8 +883,16 @@ preflight) checklist) cmd_checklist ;; all) if [ ! -f "$TFVARS" ]; then - cmd_init - die "Edit $(sg_rel "$TFVARS"), then re-run '$PROG all'." + cmd_init || exit 1 + if ! sg_interactive; then + # Template copy: it still contains placeholders. + die "Edit $(sg_rel "$TFVARS"), then re-run '$PROG all'." + fi + echo >&2 + if ! sg_confirm "Continue with the migration now? (No = edit $(sg_rel "$TFVARS") first, e.g. workspaceOverrides, then re-run '$PROG all')" Y; then + sg_log "edit $(sg_rel "$TFVARS"), then re-run '$PROG all'" + exit 0 + fi fi # Fail fast on everything the whole pipeline needs, before the (long) apply. export SG_API_TOKEN SG_BASE_URL From 9c3cdde2eceaca8de94a0daba4c82ba0def6ac39 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adis=20Halilovi=C4=87?= <adis.halilovic@stackguardian.io> Date: Mon, 7 Sep 2026 09:56:12 +0200 Subject: [PATCH 27/71] fix: parse migrate.sh fully before running it Wrap argument parsing and dispatch in main so bash reads the whole script up front. The repo is bind-mounted into the container, and a file edited during a long run was read half-old, half-new, ending in "unexpected EOF while looking for matching quote" after the import had already succeeded. --- scripts/migrate.sh | 208 +++++++++++++++++++++++---------------------- 1 file changed, 108 insertions(+), 100 deletions(-) diff --git a/scripts/migrate.sh b/scripts/migrate.sh index 9ceb4bf..6fad061 100755 --- a/scripts/migrate.sh +++ b/scripts/migrate.sh @@ -800,109 +800,117 @@ ZSH esac } -CMD="" -while [ $# -gt 0 ]; do - case "$1" in - -y | --yes) ASSUME_YES=1 ;; - --org) - ORG="$2" - shift - ;; - --org=*) ORG="${1#*=}" ;; - --export-dir) - EXPORT_DIR="$2" - shift - ;; - --export-dir=*) EXPORT_DIR="${1#*=}" ;; - --mapping) - MAPPING="$2" - shift - ;; - --mapping=*) MAPPING="${1#*=}" ;; - --concurrency) - CONC="$2" - shift - ;; - --concurrency=*) CONC="${1#*=}" ;; - --no-create-groups) CREATE_GROUPS=0 ;; - --no-variable-sets) ENRICH_VARSETS=0 ;; - --no-vcs-triggers) VCS_TRIGGERS=0 ;; - --skip-preflight) export SKIP_PREFLIGHT=1 ;; - --fresh) FRESH=1 ;; - --dry-run) DRY_RUN=1 ;; - --no-secret-stubs) SECRET_STUBS=0 ;; - --project) - PROJECT_FILTER+=("$2") - shift - ;; - --project=*) PROJECT_FILTER+=("${1#*=}") ;; - --workspace) - WS_FILTER+=("$2") +# main — argument parsing and dispatch. Kept in a function so bash parses the +# whole file before running anything: the repo is bind-mounted into the +# container, and a script edited while it runs would otherwise be read +# half-old, half-new. +main() { + local CMD="" + while [ $# -gt 0 ]; do + case "$1" in + -y | --yes) ASSUME_YES=1 ;; + --org) + ORG="$2" + shift + ;; + --org=*) ORG="${1#*=}" ;; + --export-dir) + EXPORT_DIR="$2" + shift + ;; + --export-dir=*) EXPORT_DIR="${1#*=}" ;; + --mapping) + MAPPING="$2" + shift + ;; + --mapping=*) MAPPING="${1#*=}" ;; + --concurrency) + CONC="$2" + shift + ;; + --concurrency=*) CONC="${1#*=}" ;; + --no-create-groups) CREATE_GROUPS=0 ;; + --no-variable-sets) ENRICH_VARSETS=0 ;; + --no-vcs-triggers) VCS_TRIGGERS=0 ;; + --skip-preflight) export SKIP_PREFLIGHT=1 ;; + --fresh) FRESH=1 ;; + --dry-run) DRY_RUN=1 ;; + --no-secret-stubs) SECRET_STUBS=0 ;; + --project) + PROJECT_FILTER+=("$2") + shift + ;; + --project=*) PROJECT_FILTER+=("${1#*=}") ;; + --workspace) + WS_FILTER+=("$2") + shift + ;; + --workspace=*) WS_FILTER+=("${1#*=}") ;; + -v | --verbose) VERBOSE=1 ;; + --all) PURGE=1 ;; + -h | --help) + usage + exit 0 + ;; + init | apply | enrich | convert | validate | import | triggers | all | clean | preflight | checklist) CMD="$1" ;; + completion) + cmd_completion "${2:-}" + exit 0 + ;; + *) + echo "Unknown argument: $1" >&2 + usage + exit 1 + ;; + esac shift - ;; - --workspace=*) WS_FILTER+=("${1#*=}") ;; - -v | --verbose) VERBOSE=1 ;; - --all) PURGE=1 ;; - -h | --help) + done + # No command: show the help menu instead of running the whole pipeline. + if [ -z "$CMD" ]; then usage exit 0 + fi + export SG_VERBOSE="$VERBOSE" + + case "$CMD" in + init) cmd_init ;; + clean) cmd_clean ;; + apply) cmd_apply ;; + enrich) cmd_enrich ;; + convert) cmd_convert ;; + validate) cmd_validate ;; + import) cmd_import ;; + triggers) cmd_triggers ;; + preflight) + export SG_API_TOKEN SG_BASE_URL + cmd_preflight ;; - init | apply | enrich | convert | validate | import | triggers | all | clean | preflight | checklist) CMD="$1" ;; - completion) - cmd_completion "${2:-}" - exit 0 - ;; - *) - echo "Unknown argument: $1" >&2 - usage - exit 1 + checklist) cmd_checklist ;; + all) + if [ ! -f "$TFVARS" ]; then + cmd_init || exit 1 + if ! sg_interactive; then + # Template copy: it still contains placeholders. + die "Edit $(sg_rel "$TFVARS"), then re-run '$PROG all'." + fi + echo >&2 + if ! sg_confirm "Continue with the migration now? (No = edit $(sg_rel "$TFVARS") first, e.g. workspaceOverrides, then re-run '$PROG all')" Y; then + sg_log "edit $(sg_rel "$TFVARS"), then re-run '$PROG all'" + exit 0 + fi + fi + # Fail fast on everything the whole pipeline needs, before the (long) apply. + export SG_API_TOKEN SG_BASE_URL + preflight_run all + [ "$FRESH" -eq 1 ] && { state_reset; sg_log "--fresh: previous run state discarded"; } + JQ_BIN="$(sg_resolve jq sg_ensure_jq)" + run_phase apply "$(sg_sha "$(sg_sha_files "$TFVARS")|$(ws_filter_json)")" cmd_apply + if [ "$ENRICH_VARSETS" -eq 1 ]; then run_phase enrich "$(payload_sha)" cmd_enrich; fi + run_phase convert "$(payload_sha)" cmd_convert + run_phase validate "$(payload_sha)" cmd_validate + cmd_import ;; esac - shift -done -# No command: show the help menu instead of running the whole pipeline. -if [ -z "$CMD" ]; then - usage - exit 0 -fi -export SG_VERBOSE="$VERBOSE" - -case "$CMD" in -init) cmd_init ;; -clean) cmd_clean ;; -apply) cmd_apply ;; -enrich) cmd_enrich ;; -convert) cmd_convert ;; -validate) cmd_validate ;; -import) cmd_import ;; -triggers) cmd_triggers ;; -preflight) - export SG_API_TOKEN SG_BASE_URL - cmd_preflight - ;; -checklist) cmd_checklist ;; -all) - if [ ! -f "$TFVARS" ]; then - cmd_init || exit 1 - if ! sg_interactive; then - # Template copy: it still contains placeholders. - die "Edit $(sg_rel "$TFVARS"), then re-run '$PROG all'." - fi - echo >&2 - if ! sg_confirm "Continue with the migration now? (No = edit $(sg_rel "$TFVARS") first, e.g. workspaceOverrides, then re-run '$PROG all')" Y; then - sg_log "edit $(sg_rel "$TFVARS"), then re-run '$PROG all'" - exit 0 - fi - fi - # Fail fast on everything the whole pipeline needs, before the (long) apply. - export SG_API_TOKEN SG_BASE_URL - preflight_run all - [ "$FRESH" -eq 1 ] && { state_reset; sg_log "--fresh: previous run state discarded"; } - JQ_BIN="$(sg_resolve jq sg_ensure_jq)" - run_phase apply "$(sg_sha "$(sg_sha_files "$TFVARS")|$(ws_filter_json)")" cmd_apply - if [ "$ENRICH_VARSETS" -eq 1 ]; then run_phase enrich "$(payload_sha)" cmd_enrich; fi - run_phase convert "$(payload_sha)" cmd_convert - run_phase validate "$(payload_sha)" cmd_validate - cmd_import - ;; -esac +} + +main "$@" From 04b123025e8a041d3bbbce10573ec4171b3b5768 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adis=20Halilovi=C4=87?= <adis.halilovic@stackguardian.io> Date: Mon, 7 Sep 2026 10:07:55 +0200 Subject: [PATCH 28/71] feat: strip TFC-specific variables (ignoreVarPatterns) TFC_* / TFE_* variables (TFC_WORKSPACE_NAME, the TFC_AWS_* dynamic credential settings, ...) only mean something inside Terraform Cloud. A new ignoreVarPatterns input (default ["^TFC_", "^TFE_"]) drops matching terraform and env variables from the payloads, from workspaces and from variable sets, lists them in the migration summary and keeps them out of the sensitive-variable list so no secret stubs are created for them. The init wizard asks whether to strip them. Also fix the secrets link in the checklist (orgs/<org>?tab=secrets). --- README.md | 1 + scripts/enrich_variable_sets.sh | 8 ++++++-- scripts/lib/checklist.sh | 2 +- scripts/lib/report.sh | 2 ++ scripts/lib/tfvars.sh | 6 +++++- scripts/lib/wizard.sh | 7 +++++++ transformer/terraform-cloud/locals.tf | 20 +++++++++++++++---- transformer/terraform-cloud/summary.tmpl | 10 ++++++++++ .../terraform-cloud/terraform.tfvars.example | 4 ++++ transformer/terraform-cloud/variables.tf | 6 ++++++ 10 files changed, 58 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 2cd8050..c762143 100644 --- a/README.md +++ b/README.md @@ -164,6 +164,7 @@ To update workflows with different details, re-run the sg-cli command with the m - **Workflow groups.** Each TFC project imports into an SG workflow group `tfc-<project>`, created via the API if missing (disable with `--no-create-groups`). Override the target group per project in `.sg/workflow-groups.json`; override groups must already exist. - **Variable Sets are migrated** (the `enrich` phase) — global, project-, and workspace-scoped sets are resolved per workspace with TFC precedence (priority sets override workspace vars; otherwise workspace vars win). **Sensitive** set variables can't be read from the API, so they're skipped and reported — recreate them as StackGuardian secrets. +- **TFC-specific variables are stripped.** Variables whose name matches `ignoreVarPatterns` (default `^TFC_`, `^TFE_`, e.g. `TFC_WORKSPACE_NAME` or the `TFC_AWS_*` dynamic-credential settings) only mean something inside Terraform Cloud and are not migrated, from workspaces or variable sets. They are listed in the migration summary; set `ignoreVarPatterns = []` to keep them. - **Sensitive variables become placeholder secrets.** TFC never returns sensitive values via the API. The export omits them (listed in `migration-summary.md`); after import the orchestrator creates an SG secret `tfc-<workflow>-<VAR>` with the value `CHANGE_ME` for each, references it from the workflow (`${secret::<name>}`, as an environment variable or IaC input) and lists it in `export/post-import-checklist.md`. Set the real values in the SG UI. `--no-secret-stubs` leaves SG secrets untouched. - **Terraform version fallback (FOSS ceiling).** Workspaces set to `latest` or a version constraint use `SGDefaultTerraformVersion` at export time. Pinned versions are carried over as-is and tried first at import, so a custom runtime image or private runner that ships that binary keeps working. StackGuardian's _managed_ runtimes only go up to **1.5.7**, the last MPL-licensed (FOSS) Terraform release; newer versions are BSL-licensed and are not bundled. When the API rejects a workflow for that reason, the importer **automatically re-imports it with `SGDefaultTerraformVersion`**, patches the payload file to match, prints a notice, and records each case in `export/terraform-version-fallbacks.log`. Those workflows run a different Terraform than they did in TFC, so check compatibility before the first run. To keep a newer version, set `workspaceOverrides[<name>].terraformVersion` to a binary path mounted from a private runner (or use a custom runtime container template) and re-import. - **State export is idempotent.** Re-running `terraform apply` only pulls state for workspaces not yet exported. Set `forceStateRefresh = true` to re-pull everything. Workspaces not using `remote` execution may export incomplete state — see the summary. diff --git a/scripts/enrich_variable_sets.sh b/scripts/enrich_variable_sets.sh index 167b04a..88fd77d 100755 --- a/scripts/enrich_variable_sets.sh +++ b/scripts/enrich_variable_sets.sh @@ -106,15 +106,19 @@ sg_log "resolving $set_count variable set(s) across workspaces..." ) ' >"$WORK/effective.json" +# TFC-specific variables are stripped here too (same patterns as the transformer). +IGNORE_JSON="$(tfvars_get_json .ignoreVarPatterns)" +[ "$IGNORE_JSON" = "null" ] && IGNORE_JSON='["^TFC_","^TFE_"]' + # Merge the effective set vars into each payload, then report counts. for f in "$@"; do before_tf="$("$JQ_BIN" '[.[].VCSConfig.iacInputData.data | length] | add // 0' "$f")" out="$WORK/merged.json" - "$JQ_BIN" --slurpfile eff "$WORK/effective.json" ' + "$JQ_BIN" --slurpfile eff "$WORK/effective.json" --argjson ignore "$IGNORE_JSON" ' ($eff[0]) as $E | map( ((.CLIConfiguration.TfStateFilePath // "") | sub(".*/"; "") | sub("\\.tfstate$"; "")) as $wsName - | ($E[$wsName] // []) as $all + | ($E[$wsName] // [] | map(select(.key as $k | [$ignore[] | . as $p | select($k | test($p))] | length == 0))) as $all | ($all | map(select(.sensitive != true and .category == "terraform"))) as $tf | ($all | map(select(.sensitive != true and .category == "env"))) as $env | .VCSConfig.iacInputData.data = ( diff --git a/scripts/lib/checklist.sh b/scripts/lib/checklist.sh index 748683c..0bb3c61 100644 --- a/scripts/lib/checklist.sh +++ b/scripts/lib/checklist.sh @@ -7,7 +7,7 @@ SG_UI_URL="${SG_UI_URL:-https://app.stackguardian.io}" wf_ui_url() { printf '%s/orchestrator/orgs/%s/wfgrps/%s/wfs/%s' "$SG_UI_URL" "$ORG" "$1" "$2"; } -secrets_ui_url() { printf '%s/orchestrator/orgs/%s/settings?tab=secrets' "$SG_UI_URL" "$ORG"; } +secrets_ui_url() { printf '%s/orchestrator/orgs/%s?tab=secrets' "$SG_UI_URL" "$ORG"; } # secret_name_for <workflow> <var> — SG secret name for a stubbed variable. secret_name_for() { printf 'tfc-%s-%s' "$1" "$2" | tr -c 'A-Za-z0-9_-\n' '-' | cut -c1-100; } diff --git a/scripts/lib/report.sh b/scripts/lib/report.sh index 10b81db..36d39f5 100644 --- a/scripts/lib/report.sh +++ b/scripts/lib/report.sh @@ -16,6 +16,8 @@ show_migration_summary() { _summary_section "$f" '.skippedSensitiveVars' "Sensitive variables skipped (TFC never exposes them)" \ 'to_entries[] | "\(.key): \(.value | join(", "))"' "recreated as SG secrets after import" + _summary_section "$f" '.strippedVars' "TFC-specific variables stripped (ignoreVarPatterns)" \ + 'to_entries[] | "\(.key): \(.value | join(", "))"' "" _summary_section "$f" '.terraformVersionFallbacks' "Terraform version not pinned (SGDefaultTerraformVersion used)" \ 'to_entries[] | "\(.key): \"\(.value)\""' "" _summary_section "$f" '.nonRemoteExecutionModes' "Non-remote execution mode (state may not be in TFC)" \ diff --git a/scripts/lib/tfvars.sh b/scripts/lib/tfvars.sh index f37f8b6..6ec7412 100644 --- a/scripts/lib/tfvars.sh +++ b/scripts/lib/tfvars.sh @@ -52,7 +52,7 @@ _tfvars_hcl() { printf '%s' "$1" | "$(sg_resolve jq sg_ensure_jq)" --indent 2 '. # hand-editable afterwards. # W_TFORG W_TFHOST W_WSNAMES_JSON W_TAGS_JSON W_IGNORE_TAGS_JSON W_EXPORT_STATE # W_APPROVERS_JSON W_REPO_PREFIX W_VCS_INTEGRATION W_DPC_JSON W_RUNNER_JSON -# W_DEST_KIND W_TF_VERSION W_TRIGGERS +# W_DEST_KIND W_TF_VERSION W_TRIGGERS W_IGNORE_PATTERNS_JSON tfvars_write() { local dest="$1" host_line="" if [ "${W_TFHOST:-app.terraform.io}" != "app.terraform.io" ]; then @@ -80,6 +80,10 @@ tfWorkspaceIgnoreTags = $W_IGNORE_TAGS_JSON # Directory to export Terraform files to exportPath = "export" +# TFC/TFE-specific variables that are not migrated (regexes on the variable +# name), e.g. TFC_WORKSPACE_NAME or TFC_AWS_RUN_ROLE_ARN. [] keeps everything. +ignoreVarPatterns = $W_IGNORE_PATTERNS_JSON + # Emails of the users who must approve plans (approvalPreApply is set for # workspaces without auto-apply) SGDefaultWfApprovers = $W_APPROVERS_JSON diff --git a/scripts/lib/wizard.sh b/scripts/lib/wizard.sh index c6c20b2..ba4a7a2 100644 --- a/scripts/lib/wizard.sh +++ b/scripts/lib/wizard.sh @@ -195,6 +195,12 @@ wizard_policy() { W_TF_VERSION="$(_w_default .SGDefaultTerraformVersion TERRAFORM-1.5.7)" if sg_confirm "Export Terraform state for each workspace?" "$([ "$(_w_default .exportStateFiles true)" = "false" ] && echo N || echo Y)"; then W_EXPORT_STATE=true; else W_EXPORT_STATE=false; fi if sg_confirm "Pre-configure VCS triggers (push / pull-request runs) from the TFC settings?" "$([ "$(_w_default .SGDefaultEnableVCSTriggers true)" = "false" ] && echo N || echo Y)"; then W_TRIGGERS=true; else W_TRIGGERS=false; fi + sg_dim "TFC_* / TFE_* variables (e.g. TFC_WORKSPACE_NAME, TFC_AWS_RUN_ROLE_ARN) only mean something inside Terraform Cloud." + if sg_confirm "Strip TFC-specific variables (TFC_*, TFE_*) from the migrated workflows?" "$([ "$(tfvars_get_json .ignoreVarPatterns)" = "[]" ] && echo N || echo Y)"; then + W_IGNORE_PATTERNS_JSON='["^TFC_","^TFE_"]' + else + W_IGNORE_PATTERNS_JSON='[]' + fi } # --- step 4: review + write ---------------------------------------------------- @@ -208,6 +214,7 @@ wizard_review() { row "Cloud connector" "$W_DPC_JSON" row "Runners" "$W_RUNNER_JSON" row "State export / triggers" "$W_EXPORT_STATE / $W_TRIGGERS" + row "Strip variables matching" "$W_IGNORE_PATTERNS_JSON" row "Fallback Terraform" "$W_TF_VERSION${W_WS_ABOVE_CEILING:+ ($W_WS_ABOVE_CEILING workspace(s) pinned above 1.5.7 (the last FOSS runtime SG bundles) will use it)}" [ "${W_DPC_PLACEHOLDER:-0}" -eq 1 ] && sg_warn "cloud connector left as a placeholder — edit SGDefaultDeploymentPlatformConfig in $(sg_rel "$TFVARS") before 'apply'" sg_dim "approvers, repo URL prefix and the fallback version can be edited in $(sg_rel "$TFVARS")" diff --git a/transformer/terraform-cloud/locals.tf b/transformer/terraform-cloud/locals.tf index d2ba8f0..5692f95 100644 --- a/transformer/terraform-cloud/locals.tf +++ b/transformer/terraform-cloud/locals.tf @@ -24,11 +24,21 @@ locals { name => length(local.sanitizedGroups[san]) > 1 ? "${length(san) > 93 ? substr(san, 0, 93) : san}-${substr(md5(name), 0, 6)}" : san } + # TFC-specific variables (TFC_*, TFE_* by default) are meaningless in SG and + # are stripped; recorded per workspace so the summary can list them. + strippedVars = { + for name, id in data.tfe_workspace_ids.data.ids : + name => [for v in data.tfe_variables.data[id].variables : "${v.category}:${v.name}" + if anytrue([for p in var.ignoreVarPatterns : can(regex(p, v.name))])] + } + # TFC never returns values for sensitive variables, so they cannot be - # migrated. Record them per workspace so the summary can flag them. + # migrated. Record them per workspace so the summary can flag them + # (stripped variables excluded — nobody needs a secret stub for those). sensitiveVars = { for name, id in data.tfe_workspace_ids.data.ids : - name => [for v in data.tfe_variables.data[id].variables : "${v.category}:${v.name}" if v.sensitive] + name => [for v in data.tfe_variables.data[id].variables : "${v.category}:${v.name}" + if v.sensitive && !anytrue([for p in var.ignoreVarPatterns : can(regex(p, v.name))])] } # Workspaces whose terraform_version is not a pinned semver (e.g. "latest" or @@ -72,7 +82,7 @@ locals { EnvironmentVariables = concat( [for v in data.tfe_variables.data[wsId].variables : { "config" : { "textValue" : v.value, "varName" : v.name }, "kind" : "PLAIN_TEXT" } - if v.category == "env" && v.sensitive == false], + if v.category == "env" && v.sensitive == false && !anytrue([for p in var.ignoreVarPatterns : can(regex(p, v.name))])], try(var.workspaceOverrides[wsName].extraEnvironmentVariables, []) ) @@ -96,7 +106,7 @@ locals { }, "iacInputData" : { "schemaType" : "RAW_JSON", - "data" : { for v in data.tfe_variables.data[wsId].variables : v.name => try(jsondecode(v.value), v.value) if v.category == "terraform" && v.sensitive == false } + "data" : { for v in data.tfe_variables.data[wsId].variables : v.name => try(jsondecode(v.value), v.value) if v.category == "terraform" && v.sensitive == false && !anytrue([for p in var.ignoreVarPatterns : can(regex(p, v.name))]) } } } @@ -192,6 +202,8 @@ locals { workspaceCount = length(local.workflowNames) projectWorkspaceCounts = { for pid in local.projectsUsed : try(local.projectNames[pid], pid) => length(local.payloadByProject[pid]) } skippedSensitiveVars = { for name, vars in local.sensitiveVars : name => vars if length(vars) > 0 } + strippedVars = { for name, vars in local.strippedVars : name => vars if length(vars) > 0 } + ignoreVarPatterns = var.ignoreVarPatterns terraformVersionFallbacks = local.versionFallbacks nonRemoteExecutionModes = local.nonRemoteModes renamedWorkspaces = { for name in local.workflowNames : name => local.resourceNames[name] if local.resourceNames[name] != name } diff --git a/transformer/terraform-cloud/summary.tmpl b/transformer/terraform-cloud/summary.tmpl index 8dc1ba4..7e7f811 100644 --- a/transformer/terraform-cloud/summary.tmpl +++ b/transformer/terraform-cloud/summary.tmpl @@ -18,6 +18,16 @@ These are not migrated (TFC never returns sensitive values). Recreate them as SG %{ endfor ~} %{ endif ~} +## Stripped TFC-specific variables +Not migrated because they only mean something inside Terraform Cloud (patterns: ${join(", ", summary.ignoreVarPatterns)}; set ignoreVarPatterns = [] to keep them). +%{ if length(summary.strippedVars) == 0 ~} +- None. +%{ else ~} +%{ for ws, vars in summary.strippedVars ~} +- ${ws}: ${join(", ", vars)} +%{ endfor ~} +%{ endif ~} + ## Terraform version fallbacks Workspaces whose version was not a pinned semver; the configured SGDefaultTerraformVersion was used. %{ if length(summary.terraformVersionFallbacks) == 0 ~} diff --git a/transformer/terraform-cloud/terraform.tfvars.example b/transformer/terraform-cloud/terraform.tfvars.example index 5b7143f..f588f18 100644 --- a/transformer/terraform-cloud/terraform.tfvars.example +++ b/transformer/terraform-cloud/terraform.tfvars.example @@ -16,6 +16,10 @@ tfWorkspaceIgnoreTags = null # Directory to export Terraform files to exportPath = "export" +# TFC/TFE-specific variables that are not migrated (regexes on the variable +# name), e.g. TFC_WORKSPACE_NAME or TFC_AWS_RUN_ROLE_ARN. [] keeps everything. +ignoreVarPatterns = ["^TFC_", "^TFE_"] + # Add emails of the users who should approve the terraform plan, since approvalPreApply is set to true SGDefaultWfApprovers = [] diff --git a/transformer/terraform-cloud/variables.tf b/transformer/terraform-cloud/variables.tf index 26d8299..a8a103d 100644 --- a/transformer/terraform-cloud/variables.tf +++ b/transformer/terraform-cloud/variables.tf @@ -39,6 +39,12 @@ variable "exportPath" { type = string } +variable "ignoreVarPatterns" { + default = ["^TFC_", "^TFE_"] + description = "Regexes (matched against the variable name) for TFC/TFE-specific variables that have no meaning outside Terraform Cloud and are not migrated, e.g. TFC_WORKSPACE_NAME or the TFC_AWS_* dynamic-credential settings. Applies to terraform and env variables, including variable-set variables. Set to [] to keep everything; stripped variables are listed in migration-summary.md." + type = list(string) +} + variable "SGDefaultWfApprovers" { default = [] description = "Add emails of the users who should approve the terraform plan, since approvalPreApply is set to true" From a689dd848d647bdf225e0fb11ffcf5f23c5cb813 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adis=20Halilovi=C4=87?= <adis.halilovic@stackguardian.io> Date: Mon, 7 Sep 2026 12:18:34 +0200 Subject: [PATCH 29/71] fix: cloud connector picker offered VCS connectors The "^(AWS|AZURE|GCP)_" filter also matched AZURE_DEVOPS, so an Azure DevOps VCS connector could be chosen as the cloud connector and only schema validation caught it. Both pickers now use the exact kind lists (VCS also accepts AZURE_DEVOPS_SP and GITLAB_OAUTH_SSH and maps them to the source kind), are sorted by name, and preflight rejects a DeploymentPlatformConfig kind outside the schema enum. The validator no longer prints each failure twice. --- scripts/lib/preflight.sh | 7 +++++++ scripts/lib/wizard.sh | 9 ++++++--- scripts/validate_payload.sh | 2 +- 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/scripts/lib/preflight.sh b/scripts/lib/preflight.sh index 4e8725f..bed47c6 100644 --- a/scripts/lib/preflight.sh +++ b/scripts/lib/preflight.sh @@ -137,6 +137,13 @@ preflight_config() { GITHUB_COM | GITHUB_APP_CUSTOM | GIT_OTHER | INLINE | BITBUCKET_ORG | GITLAB_COM | AZURE_DEVOPS) pf_ok "SGDefaultSourceConfigDestKind = $v" ;; *) pf_fail "SGDefaultSourceConfigDestKind '$v' is not one of GITHUB_COM, GITLAB_COM, BITBUCKET_ORG, AZURE_DEVOPS, GIT_OTHER" ;; esac + while IFS= read -r v; do + [ -n "$v" ] || continue + case "$v" in + AWS_STATIC | AWS_RBAC | AWS_OIDC | AZURE_STATIC | AZURE_OIDC | AZURE_MANAGED_ID_OIDC | GCP_STATIC | GCP_OIDC) pf_ok "DeploymentPlatformConfig kind $v" ;; + *) pf_fail "DeploymentPlatformConfig kind '$v' is not a cloud connector kind (AWS_STATIC, AWS_RBAC, AWS_OIDC, AZURE_STATIC, AZURE_OIDC, AZURE_MANAGED_ID_OIDC, GCP_STATIC, GCP_OIDC) — a VCS connector was picked as the cloud connector?" ;; + esac + done < <(tfvars_json | "$jqb" -r '[ (.SGDefaultDeploymentPlatformConfig // [])[]?.kind, ((.workspaceOverrides // {}) | to_entries[]? | .value.DeploymentPlatformConfig // [] | .[]?.kind) ] | map(select(. != null)) | unique | .[]') v="$(tfvars_get .SGDefaultTerraformVersion TERRAFORM-1.5.7)" if [[ "$v" =~ ^TERRAFORM-([0-9]+)\.([0-9]+)\.([0-9]+)$ ]]; then if [ "$(printf '%03d%03d%03d' "${BASH_REMATCH[1]}" "${BASH_REMATCH[2]}" "${BASH_REMATCH[3]}")" -gt "001005007" ]; then diff --git a/scripts/lib/wizard.sh b/scripts/lib/wizard.sh index ba4a7a2..80bb2d8 100644 --- a/scripts/lib/wizard.sh +++ b/scripts/lib/wizard.sh @@ -124,7 +124,7 @@ wizard_sg() { # VCS connector -> integration id, source kind, repo prefix. vcs="" - [ "$W_SG_DISCOVERY" -eq 1 ] && vcs="$(printf '%s' "$ints" | "$jqb" -r '.[] | select((.type // "") | test("^(GITHUB_COM|GITHUB_APP_CUSTOM|GITLAB_COM|BITBUCKET_ORG|AZURE_DEVOPS|GIT_OTHER)$")) | "\(.name)|\(.type)"')" + [ "$W_SG_DISCOVERY" -eq 1 ] && vcs="$(printf '%s' "$ints" | "$jqb" -r '[.[] | select((.type // "") | IN("GITHUB_COM","GITHUB_APP_CUSTOM","GITLAB_COM","GITLAB_OAUTH_SSH","BITBUCKET_ORG","AZURE_DEVOPS","AZURE_DEVOPS_SP","GIT_OTHER"))] | sort_by(.name) | .[] | "\(.name)|\(.type)"')" if [ -n "$vcs" ]; then # shellcheck disable=SC2046 pick="$(SG_SELECT_OTHER=1 sg_select "Which VCS connector should clone the repositories?" $(printf '%s\n' "$vcs" | tr '\n' ' '))" || return 1 @@ -137,6 +137,8 @@ wizard_sg() { W_VCS_INTEGRATION="/integrations/${pick#/integrations/}" case "$kind" in GITHUB_APP_CUSTOM) W_DEST_KIND=GITHUB_COM ;; + GITLAB_OAUTH_SSH) W_DEST_KIND=GITLAB_COM ;; + AZURE_DEVOPS_SP) W_DEST_KIND=AZURE_DEVOPS ;; GITHUB_COM | GITLAB_COM | BITBUCKET_ORG | AZURE_DEVOPS | GIT_OTHER) W_DEST_KIND="$kind" ;; *) W_DEST_KIND="$(sg_select "VCS provider kind" GITHUB_COM GITLAB_COM BITBUCKET_ORG AZURE_DEVOPS GIT_OTHER)" || return 1 ;; esac @@ -145,7 +147,8 @@ wizard_sg() { # Cloud connector -> DeploymentPlatformConfig. cloud="" - [ "$W_SG_DISCOVERY" -eq 1 ] && cloud="$(printf '%s' "$ints" | "$jqb" -r '.[] | select((.type // "") | test("^(AWS|AZURE|GCP)_")) | "\(.name)|\(.type)"')" + # Only kinds DeploymentPlatformConfig accepts (AZURE_DEVOPS* are VCS connectors). + [ "$W_SG_DISCOVERY" -eq 1 ] && cloud="$(printf '%s' "$ints" | "$jqb" -r '[.[] | select((.type // "") | IN("AWS_STATIC","AWS_RBAC","AWS_OIDC","AZURE_STATIC","AZURE_OIDC","AZURE_MANAGED_ID_OIDC","GCP_STATIC","GCP_OIDC"))] | sort_by(.name) | .[] | "\(.name)|\(.type)"')" if [ -n "$cloud" ]; then # shellcheck disable=SC2046 pick="$(SG_SELECT_OTHER=1 sg_select "Which cloud connector should the workflows deploy with?" $(printf '%s\n' "$cloud" | tr '\n' ' ') "skip|decide later (leaves a placeholder to edit)")" || return 1 @@ -169,7 +172,7 @@ wizard_sg() { "private|a private runner group in your own network")" || return 1 if [ "$runner" = "private" ]; then groups="" - [ "$W_SG_DISCOVERY" -eq 1 ] && groups="$(sg_list_runnergroups 2>/dev/null | "$jqb" -r '.[]' 2>/dev/null || true)" + [ "$W_SG_DISCOVERY" -eq 1 ] && groups="$(sg_list_runnergroups 2>/dev/null | "$jqb" -r 'sort | .[]' 2>/dev/null || true)" if [ -n "$groups" ]; then # shellcheck disable=SC2046 name="$(SG_SELECT_OTHER=1 sg_select "Which runner group?" $(printf '%s\n' "$groups" | tr '\n' ' '))" || return 1 diff --git a/scripts/validate_payload.sh b/scripts/validate_payload.sh index 234638e..c5fbd4a 100755 --- a/scripts/validate_payload.sh +++ b/scripts/validate_payload.sh @@ -25,5 +25,5 @@ sg_log "validating $# file(s) against schema/sg-payload.schema.json" # Strip the repo-root prefix from its output for readable, relative paths # (pipefail off so the pipeline's status is sed's; yajsv's status via PIPESTATUS). set +o pipefail -"$YAJSV_BIN" -s "$SCHEMA" "$@" 2>&1 | sed "s#${SG_REPO_ROOT}/##g" +"$YAJSV_BIN" -s "$SCHEMA" "$@" 2>&1 | sed "s#${SG_REPO_ROOT}/##g" | awk '!seen[$0]++' exit "${PIPESTATUS[0]}" From 49712c1b69bc69db56d1c5610c47aa94116ea0f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adis=20Halilovi=C4=87?= <adis.halilovic@stackguardian.io> Date: Mon, 7 Sep 2026 14:37:02 +0200 Subject: [PATCH 30/71] feat: execution preset support; cleaner logs and flow Presets: SGTerraformVersionSource (carry|preset) plus nullable SGDefaultTerraformVersion and SGDefaultRunnerConstraints. Keys left out of the payload are filled from the org's execution preset by the SG API at import. init offers the preset with the current value shown, preflight reports it, the plan marks 'preset (...)' cells, the checklist explains what changed, and a pin rejected above the 1.5.7 ceiling falls back to the preset when the fallback is null. Flow: numbered phases with timings, live progress lines for terraform and the parallel phases, plain-language preflight with connector/kind/prefix consistency checks, readable wizard review, per-file convert/validate lines, a condensed post-import checklist with group links, and a neutral message when the import is declined. Fixes: init kept the previous repo URL prefix when the connector switched provider (the prefix now comes from the TFC repositories); enrich re-ran on every 'all' because convert changed its recorded hash; quotes in connector names broke the generated tfvars; a RunnerConstraints override with names failed HCL type unification. --- .gitignore | 2 + CLAUDE.md | 12 +- README.md | 7 +- scripts/convert_hcl_to_json.sh | 13 +- scripts/enrich_variable_sets.sh | 12 +- scripts/lib/checklist.sh | 140 +++++--- scripts/lib/preflight.sh | 191 ++++++++--- scripts/lib/prompt.sh | 7 +- scripts/lib/report.sh | 132 ++++++-- scripts/lib/sg_api.sh | 78 +++++ scripts/lib/tfc_api.sh | 61 +++- scripts/lib/tfvars.sh | 63 +++- scripts/lib/wizard.sh | 298 +++++++++++++++--- scripts/migrate.sh | 288 ++++++++++++----- scripts/tools.sh | 59 ++++ scripts/validate_payload.sh | 21 +- .../terraform-cloud/example_payload.jsonc | 4 +- transformer/terraform-cloud/locals.tf | 66 +++- transformer/terraform-cloud/summary.tmpl | 18 +- .../terraform-cloud/terraform.tfvars.example | 23 +- transformer/terraform-cloud/variables.tf | 22 +- 21 files changed, 1192 insertions(+), 325 deletions(-) diff --git a/.gitignore b/.gitignore index 4e5a8c2..7819528 100644 --- a/.gitignore +++ b/.gitignore @@ -148,6 +148,8 @@ crash.log # control as they are data points which are potentially sensitive and subject # to change depending on the environment. *.tfvars +# The init wizard keeps the previous file as terraform.tfvars.bak +*.tfvars.bak # Ignore override files as they are usually used to override resources locally and so # are not checked in diff --git a/CLAUDE.md b/CLAUDE.md index ed6fc39..1a3d3f1 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -15,8 +15,8 @@ The migration is a user-driven pipeline, not a single program: 1. **Extract + transform** — `transformer/terraform-cloud/` is a Terraform root module. `terraform apply` reads TFC/TFE workspaces and emits one `<exportPath>/sg-payload.<project>.json` per TFC project, a `migration-summary.md`/`.json` report, and per-workspace `.tfstate` files when `exportStateFiles` is true. 2. **Enrich (variable sets)** — `scripts/enrich_variable_sets.sh` merges TFC Variable Set variables into the payloads via the TFC API (the provider can't enumerate sets). Resolves global/project/workspace scope + TFC precedence per workspace; non-sensitive only. 3. **Tuning** — adjust per-workspace differences (integration IDs, VCS auth, runners, approvers, version) via the `workspaceOverrides` variable and re-apply; or hand-edit the payload files. `example_payload.jsonc` is the annotated field reference. -4. **HCL→JSON conversion** — `scripts/convert_hcl_to_json.sh` rewrites HCL-string variable values in each payload to real JSON. -5. **Validation** — `scripts/validate_payload.sh` checks each payload against `schema/sg-payload.schema.json` (downloads `yajsv`). +4. **HCL→JSON conversion** — `scripts/convert_hcl_to_json.sh` rewrites HCL-string variable values in each payload to real JSON (one result line per file; details with `-v`). +5. **Validation** — `scripts/validate_payload.sh` checks each payload against `schema/sg-payload.schema.json` (downloads `yajsv`; yajsv's lines are re-rendered as ✓/✗ per file, raw with `-v`). 6. **Import** — `sg-cli workflow create --bulk`, run once per project file, each into its own workflow group. ### Orchestration & tooling @@ -24,8 +24,8 @@ The migration is a user-driven pipeline, not a single program: The five phases are wrapped by an orchestrator so users don't run them by hand: - `sg-migrate.sh` (host entrypoint, repo root) — runs `scripts/migrate.sh` **inside the Docker image** (`Dockerfile`), bind-mounting the repo at `/app` and forwarding `SG_API_TOKEN`/`SG_ORG`/`SG_BASE_URL`/`TFE_TOKEN`/`TF_TOKEN_*`. For TFC auth it prefers a long-lived `TFE_TOKEN` (env); otherwise it mounts `~/.terraform.d/credentials.tfrc.json` read-only. Runs natively instead when `--native`/`--local` is passed, `SG_NATIVE=1` is set, the command is `clean`/`completion`/help (or no command is given), or Docker is absent. Exports `SG_PROG` so `migrate.sh` shows `./sg-migrate.sh` in its usage/hints. -- `scripts/migrate.sh` — the actual orchestrator (sources `scripts/lib/*.sh`, see below). Subcommands `init|preflight|apply|enrich|convert|validate|import|triggers|checklist|all|clean|completion` (no command prints the help menu; `enrich` runs in `all` unless `--no-variable-sets`). Flags beyond the basics: `--dry-run` (import: plan only), `--fresh` (ignore run state), `--project SEG` / `--workspace NAME` (repeatable filters; apply maps `--workspace` to `-var workspacenames=[...]`, import works on a jq-filtered temp copy), `--skip-preflight`, `--no-secret-stubs`. Resolves each tool via `sg_resolve` (PATH first — the image installs them — else `sg_ensure_*` cache). `apply`/`all` first run `require_tfc_auth`, which resolves the token the tfe provider will use (`TFE_TOKEN`, `TF_TOKEN_<host>`, or the `terraform login` file for `tfHostname`) and verifies it with `GET /api/v2/account/details`, failing fast on a missing or rejected (expired) credential. Runs `convert` and `import` in parallel (`--concurrency`, default 4), raises `terraform apply -parallelism`, and retries `sg-cli` imports with backoff (`sg_retry`, `SG_RETRIES`). Import resolves each project's workflow group, checks existence via the SG API, shows a plan (`exists`/`create`), prompts (skip with `-y`), **creates the missing `tfc-<project>` groups via the API**, then imports. sg-cli exits 0 even when individual workflows fail, so `do_import` parses its output: a workflow rejected as above SG's managed Terraform ceiling (1.5.7, last MPL/FOSS release) is re-imported with `SGDefaultTerraformVersion` (read from `terraform.tfvars`), the payload patched in place, and the case logged to `export/terraform-version-fallbacks.log` plus a printed notice; any other per-workflow failure fails the run. The trigger pass skips workflows that do not exist in SG, and `sg_api_post` returns 22 on 4xx so `sg_retry` (via `SG_NO_RETRY_RC`) does not retry definitive errors. `clean` removes local artifacts (`export/`, TF state, tool cache); `clean --all` also removes config. `completion bash|zsh` prints a completion script (`cmd_completion`; commands/options come from `SG_COMMANDS`/`SG_OPTIONS`, keep them in sync with the parser and the host flags); like `clean` it always runs natively. Output is concise by default — `apply` captures terraform's init/plan output and shows only progress + the final summary (or the full log on failure, `TF_IN_AUTOMATION=1`/`-no-color`); `-v`/`--verbose` (`SG_VERBOSE`) streams everything and un-gates the convert detail lines. Colored logging via `sg_step`/`sg_log`/`sg_success`/`sg_warn`/`sg_err`; paths shown relative via `sg_rel` (all auto-off when not a TTY / `NO_COLOR`). -- `scripts/lib/` — sourced libraries (a `lib/` gitignore rule is negated for this dir): `prompt.sh` (`sg_ask`/`sg_confirm`/`sg_select`; tty or `SG_ANSWERS_FILE` for tests; `SG_NONINTERACTIVE=1`/`-y`/no TTY → defaults), `tfvars.sh` (`tfvars_get` cached hcl2json reads, `tfvars_write` renders the wizard's `W_*` vars), `tfc_api.sh` (`tfc_http`/`tfc_get_all` paginated, `tfc_list_{orgs,projects,workspaces}`, `tfc_token` with provider precedence, `require_tfc_auth`), `sg_api.sh` (`_sg_api` 0/22/1 contract + `SG_HTTP_CODE`, `sg_list_integrations`, `sg_*_exists`, `sg_list_workflows`, `sg_create_secret`, `sg_patch_workflow`, `wfgroup_*`), `wizard.sh` (`wizard_run`: 4 steps TFC → SG → defaults → review), `preflight.sh` (`preflight_run <apply|import|all>`, once per process), `state.sh` (`.sg/state.json`: `phases.<name>.input_sha`, `import.<seg>.{imported,failed,tf_fallback}`, `triggers.<seg>`, `secrets.<name>`; `run_phase` in migrate.sh skips unchanged phases), `report.sh` (`show_migration_summary` after apply, `show_import_plan` per-workflow table), `errors.sh` (`explain_api_error` regex→hint table, `SG_API_HINTS`), `checklist.sh` (`create_secret_stubs` → SG secret `tfc-<wf>-<VAR>` = `CHANGE_ME`, referenced as `${secret::<name>}` via PATCH + payload patch; `write_checklist` → `export/post-import-checklist.md`). Parallel jobs (`run_parallel`) run in subshells, so `do_import`/`do_set_triggers` write `.import-result.<seg>.json` / `.triggers-result.<seg>.json` into the export dir and the caller merges them into state. +- `scripts/migrate.sh` — the actual orchestrator (sources `scripts/lib/*.sh`, see below). Subcommands `init|preflight|apply|enrich|convert|validate|import|triggers|checklist|all|clean|completion` (no command prints the help menu; `enrich` runs in `all` unless `--no-variable-sets`). Flags beyond the basics: `--dry-run` (import: plan only), `--fresh` (ignore run state), `--project SEG` / `--workspace NAME` (repeatable filters; apply maps `--workspace` to `-var workspacenames=[...]`, import works on a jq-filtered temp copy), `--skip-preflight`, `--no-secret-stubs`. Resolves each tool via `sg_resolve` (PATH first — the image installs them — else `sg_ensure_*` cache). `apply`/`all` first run `require_tfc_auth`, which resolves the token the tfe provider will use (`TFE_TOKEN`, `TF_TOKEN_<host>`, or the `terraform login` file for `tfHostname`) and verifies it with `GET /api/v2/account/details`, failing fast on a missing or rejected (expired) credential. Runs `convert` and `import` in parallel (`--concurrency`, default 4), raises `terraform apply -parallelism`, and retries `sg-cli` imports with backoff (`sg_retry`, `SG_RETRIES`). Import resolves each project's workflow group, checks existence via the SG API, shows a plan (`exists`/`create`), prompts (skip with `-y`), **creates the missing `tfc-<project>` groups via the API**, then imports. sg-cli exits 0 even when individual workflows fail, so `do_import` parses its output: a workflow rejected as above SG's managed Terraform ceiling (1.5.7, last MPL/FOSS release) is re-imported with `SGDefaultTerraformVersion` (read from `terraform.tfvars`; an explicit `null` there means the `terraformVersion` key is deleted instead, so the org's execution preset decides — `tfvars_is_null` tells an explicit null from a missing key), the payload patched in place, and the case logged to `export/terraform-version-fallbacks.log` plus a printed notice; any other per-workflow failure fails the run. `cmd_import` reads the org's execution preset (`sg_execution_preset`, `GET /orgs/{org}/` → `Settings.workflowDefaults`) and `preset_labels` (report.sh) turns it into the `preset (1.5.7)` / `preset (private:rg)` cells of the plan and the `SG_TF_FALLBACK_LABEL` used in messages. The trigger pass skips workflows that do not exist in SG, and `sg_api_post` returns 22 on 4xx so `sg_retry` (via `SG_NO_RETRY_RC`) does not retry definitive errors. `clean` removes local artifacts (`export/`, TF state, tool cache); `clean --all` also removes config. `completion bash|zsh` prints a completion script (`cmd_completion`; commands/options come from `SG_COMMANDS`/`SG_OPTIONS`, keep them in sync with the parser and the host flags); like `clean` it always runs natively. Output is concise by default — `apply` captures terraform's init/plan output and shows only progress + the final summary (or the full log on failure, `TF_IN_AUTOMATION=1`/`-no-color`); `-v`/`--verbose` (`SG_VERBOSE`) streams everything and un-gates the convert detail lines. Colored logging via `sg_step`/`sg_log`/`sg_success`/`sg_warn`/`sg_err`/`sg_row`; paths shown relative via `sg_rel` (all auto-off when not a TTY / `NO_COLOR`). Long steps show a live progress line (`sg_run_quiet` for terraform init/apply, `run_parallel <fn> <max> <label> <items>` for convert/import/triggers — animated only when `SG_ANIMATE=1`, i.e. colors are on; plain log lines otherwise) and report their duration (`sg_fmt_secs`). Phases start with `phase_begin` — inside `all` the header is numbered (`Phase 2/5: ...`, `PHASE_TOTAL` set in `main`) and `phase_took` gives the elapsed time; `run_parallel` prints a job's output as-is when it is a single line and adds the `── <file> ──` header only for longer or failed output (always with `-v`). The import confirmation goes through `sg_confirm` (so `SG_ANSWERS_FILE` works; a declined prompt is a neutral "cancelled" line, not an error) and `import`/`all` end with `finish_line` (outcome, total time, open checklist items from `CHECKLIST_OPEN`). A Ctrl-C is trapped to clear the progress line and print a resume hint. +- `scripts/lib/` — sourced libraries (a `lib/` gitignore rule is negated for this dir): `prompt.sh` (`sg_ask`/`sg_confirm`/`sg_select`; tty or `SG_ANSWERS_FILE` for tests; `SG_NONINTERACTIVE=1`/`-y`/no TTY → defaults), `tfvars.sh` (`tfvars_get` cached hcl2json reads, `tfvars_has`/`tfvars_is_null` to tell an explicit `null` from a missing key, `tfvars_write` renders the wizard's `W_*` vars — `W_TF_SOURCE`, and `W_TF_VERSION`/`W_RUNNER_JSON` may be the literal `null`; plain strings go through `_tfvars_str`, so quotes/backslashes in names stay valid HCL), `tfc_api.sh` (`tfc_http`/`tfc_get_all` paginated, `tfc_list_{orgs,projects,workspaces}` — workspaces include `vcs_provider`/`vcs_url`/`vcs_identifier` —, `tfc_select_workspaces` (the module's name-glob / include-tags(all) / exclude-tags filter, shared by wizard and preflight), `tfc_vcs_summary`/`tfc_vcs_kind_for`/`tfc_vcs_label_for`, `tfc_token` with provider precedence, `require_tfc_auth`), `sg_api.sh` (`_sg_api` 0/22/1 contract + `SG_HTTP_CODE`, `sg_list_integrations`, `sg_integration_type`, `sg_vcs_kind_of` (connector type → `sourceConfigDestKind`), `sg_execution_preset` + `sg_preset_{desc,runner_desc,version_desc,runner_provided}` (the org's execution preset and how to describe it), `sg_*_exists`, `sg_list_workflows`, `sg_create_secret`, `sg_patch_workflow`, `wfgroup_*`), `wizard.sh` (`wizard_run`: 4 steps TFC → SG → defaults → review; the TFC step keeps the workspace list and derives, via `tfc_vcs_summary`, which VCS provider the selected workspaces use and the repositories' URL prefix — the SG step lists matching connectors first and takes the repo URL prefix from TFC (never from a previous tfvars written for another provider kind), warns on a kind mismatch, reads the org's execution preset and offers it as the runner choice; the defaults step asks whether to carry TFC versions or use the preset (`SGTerraformVersionSource`) and, for carry, which fallback (`SGDefaultTerraformVersion` or the preset = null); the review shows human-readable rows plus the `tfc-<project>` groups; `_w_select_lines` feeds menus line-by-line so names with spaces/globs survive), `preflight.sh` (`preflight_run <apply|import|all>`, once per process; ✓ lines are plain language and name the connector's role/kind, ✗/! lines name the tfvars field to fix; besides existence it checks consistency — VCS kind vs. the connector's type (fail), repo URL prefix vs. where the TFC repositories live or the kind's well-known host (warn); it also reports the org's execution preset next to the version/runner policy and warns when `carry` meets a preset that mounts a runner-provided binary), `state.sh` (`.sg/state.json`: `phases.<name>.input_sha`, `import.<seg>.{imported,failed,tf_fallback}`, `triggers.<seg>`, `secrets.<name>`; `run_phase` in migrate.sh skips unchanged phases), `report.sh` (`show_migration_summary` after apply — incl. a state-files row —, `show_import_plan` per-workflow table with columns sized to their content), `errors.sh` (`explain_api_error` regex→hint table, `SG_API_HINTS`), `checklist.sh` (`create_secret_stubs` → SG secret `tfc-<wf>-<VAR>` = `CHANGE_ME`, referenced as `${secret::<name>}` via PATCH + payload patch; `write_checklist` → `export/post-import-checklist.md`, printed to the terminal as one ✓/! status line per section plus the imported groups with UI links — the file has the details; sets `CHECKLIST_OPEN`). Parallel jobs (`run_parallel`) run in subshells, so `do_import`/`do_set_triggers` write `.import-result.<seg>.json` / `.triggers-result.<seg>.json` into the export dir and the caller merges them into state. - `scripts/tools.sh` — sourced by the other scripts (sets `SG_REPO_ROOT` to its parent dir). Provides `sg_resolve` (PATH-or-cache), `sg_ensure_{jq,hcl2json,yajsv,sgcli}` (pinned downloads into `.sg/cached/`; `sg-cli` is the Go binary), `sg_retry`, the color vars + log helpers. The Docker image bundles these tools at build time (`yajsv` built from source so arm64 works), so in-container runs never download. - **Variable sets** — `scripts/enrich_variable_sets.sh <tfc-org> <payload...>` (run by `cmd_enrich`, which reads `tfOrg`/`tfHostname` from `terraform.tfvars` via `hcl2json` — variable sets live in the **TFC** org, not `SG_ORG`). Lists all sets + vars via the TFC API (Bearer token from the `terraform login` creds file or `TFE_TOKEN`), computes per-workspace effective vars (scope: global/project/workspace; precedence: priority sets > workspace vars > non-priority sets), and merges them into each workflow (matched by workspace name parsed from `TfStateFilePath`). Sensitive set vars and key conflicts are reported. Skipped by `--no-variable-sets`. - **Workflow groups** — each TFC project maps to an SG workflow group `tfc-<project-segment>`, created via `POST /api/v1/orgs/{org}/wfgrps/` (auth `Authorization: apikey <token>`) if missing. (Both the API calls and sg-cli honor the undocumented `SG_BASE_URL` for non-prod targets.) Groups are addressed by name, so no generated ID is tracked. `.sg/workflow-groups.json` (gitignored, optional) overrides the target group per segment (`{"<segment>": "<existing-group>"}`); override groups are not auto-created. `--no-create-groups` requires all groups to pre-exist. @@ -39,14 +39,14 @@ The whole transformation lives in `locals.tf` — there are no `outputs.tf`/`mai - TFC `terraform` (non-sensitive) variables → `VCSConfig.iacInputData.data` (kept as strings here; `try(jsondecode(...), v.value)` only decodes values that are already valid JSON). - TFC `env` (non-sensitive) variables → `EnvironmentVariables` as `PLAIN_TEXT`. - `auto_apply` → inverted into `approvalPreApply` / gated `Approvers`. - - `terraform_version` → `TERRAFORM-<version>` only for a pinned semver; otherwise `SGDefaultTerraformVersion`. + - `terraform_version` → with `SGTerraformVersionSource = "carry"` (default) `TERRAFORM-<version>` for a pinned semver, otherwise `SGDefaultTerraformVersion`; with `"preset"`, or when that default is `null`, the `terraformVersion` key is **left out** of `TerraformConfig` so the SG API fills it from the org's **execution preset** (`Settings.workflowDefaults`, UI: Settings → Runner groups → Execution presets; platform default: managed Terraform 1.5.7). The API only fills keys that are absent, never `null`/`""`, so the transformer filters nulls out (`local.tfVersion`). Likewise `RunnerConstraints` is omitted when `SGDefaultRunnerConstraints = null` (`local.runnerConstraints`, picked via a tuple because an override with `names` and a default without it are different object types). The summary records `terraformVersionSource`, `terraformVersionDefault`, `runnerConstraintsSource` and `tfcTerraformVersions` so later phases can explain what each workflow runs. - `project_id` → `CLIConfiguration.WorkflowGroup.name` = `tfc-<project-segment>` (matches the per-project filename and the group the importer creates/targets). - Sensitive variables (terraform + env) are skipped and recorded in the summary. - Per-workspace `var.workspaceOverrides[<name>]` fields take precedence over the `SGDefault*` values (resolved via `try(var.workspaceOverrides[name].<field>, null) != null ? ... : <default>`). - `local.resourceNames` sanitizes workspace names to a valid SG `ResourceName` (≤100 chars, `^[-a-zA-Z0-9_]+$`, collision-disambiguated). For normal TFC names this is a no-op; any actual rename is reported in the summary. This is the single place to adjust naming rules. - `resources.tf` — writes one `sg-payload.<project>.json` per project directly via `for_each` (no `mv`), plus `migration-summary.{md,json}`. When `exportStateFiles=true`, `null_resource.exportState` pulls each workspace's state **directly from the TFC/TFE API** (`GET /api/v2/workspaces/{id}/current-state-version` → `hosted-state-download-url`) via a `local-exec` `curl`/`jq` script — no `terraform init` or providers per workspace (avoids the plugin-cache concurrency bug and per-workspace provider downloads). The token is read at runtime from `~/.terraform.d/credentials.tfrc.json` (the `terraform login` file) or `TFE_TOKEN`, so it never enters TF state. Idempotent (keyed by workspace name/id; `forceStateRefresh` re-pulls), with per-workspace failures (no token / no state / download error) recorded in `state-export-failures.log` instead of aborting. `cmd_apply` ensures `jq`/`curl` are on PATH for the apply. - `summary.tmpl` — renders `local.summary` to `migration-summary.md`. -- `variables.tf` / `terraform.tfvars.example` — inputs. `SGDefault*` variables supply global defaults baked into every workflow (deployment platform, VCS auth, repo prefix, source kind, approvers, Terraform version, runner constraints — `SGDefaultRunnerConstraints`, validated to `shared` or `private`+`names`); `workspaceOverrides` overrides them per workspace; `forceStateRefresh` controls state re-pull. Requires Terraform `>= 1.3` (for `optional()` object attributes). +- `variables.tf` / `terraform.tfvars.example` — inputs. `SGDefault*` variables supply global defaults baked into every workflow (deployment platform, VCS auth, repo prefix, source kind, approvers, Terraform version, runner constraints — `SGDefaultRunnerConstraints`, validated to `shared` or `private`+`names`, or `null` to defer to the execution preset); `SGTerraformVersionSource` (`carry`|`preset`) and a nullable `SGDefaultTerraformVersion` choose between carrying TFC pins and leaving the version to the execution preset; `workspaceOverrides` overrides them per workspace; `forceStateRefresh` controls state re-pull. Requires Terraform `>= 1.3` (for `optional()` object attributes). ### `scripts/convert_hcl_to_json.sh` diff --git a/README.md b/README.md index c762143..8ada2c7 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,7 @@ export SG_ORG=<your SG org> ./sg-migrate.sh all # preflight -> apply -> enrich -> convert -> validate -> import (shows a plan, asks before importing) ``` -That's it — no IDs to look up and no workflow-group mapping to fill in. `init` lists what the tokens can see (TFC organisations and workspaces, SG VCS/cloud connectors and runner groups) and writes `terraform.tfvars` from your picks; `all` verifies every reference **before** running terraform (preflight), prints a migration summary after the export, shows a per-workflow import plan (create/update, Terraform version, runner, triggers, secrets), and ends with a **post-import checklist** of what still needs a human. Each TFC project is imported into an SG workflow group named `tfc-<project>`, **created automatically via the API** if it doesn't exist. +That's it — no IDs to look up and no workflow-group mapping to fill in. `init` lists what the tokens can see (TFC organisations and workspaces, SG VCS/cloud connectors and runner groups, the org's execution preset) and writes `terraform.tfvars` from your picks — it also reads which VCS provider your TFC workspaces are connected to, lists the matching connectors first and takes the repository URL prefix from TFC. `all` verifies every reference **before** running terraform (preflight — including that the VCS connector, the VCS kind and the repo URL prefix agree with each other and with the TFC repositories), prints a migration summary after the export, shows a per-workflow import plan (create/update, Terraform version, runner, triggers, secrets), and ends with a **post-import checklist** of what still needs a human. Phases are numbered, long steps show a live progress line, and every phase reports how long it took. Each TFC project is imported into an SG workflow group named `tfc-<project>`, **created automatically via the API** if it doesn't exist. - Single phase: `./sg-migrate.sh preflight|apply|enrich|convert|validate|import|triggers|checklist`. Running `./sg-migrate.sh` with no command prints the help menu. - **Resume.** `all` remembers what it completed (`.sg/state.json`) and skips phases whose inputs have not changed, so after a failure you just re-run it; files already imported in full are skipped and files with failures are retried. `--fresh` redoes everything. @@ -166,8 +166,9 @@ To update workflows with different details, re-run the sg-cli command with the m - **Variable Sets are migrated** (the `enrich` phase) — global, project-, and workspace-scoped sets are resolved per workspace with TFC precedence (priority sets override workspace vars; otherwise workspace vars win). **Sensitive** set variables can't be read from the API, so they're skipped and reported — recreate them as StackGuardian secrets. - **TFC-specific variables are stripped.** Variables whose name matches `ignoreVarPatterns` (default `^TFC_`, `^TFE_`, e.g. `TFC_WORKSPACE_NAME` or the `TFC_AWS_*` dynamic-credential settings) only mean something inside Terraform Cloud and are not migrated, from workspaces or variable sets. They are listed in the migration summary; set `ignoreVarPatterns = []` to keep them. - **Sensitive variables become placeholder secrets.** TFC never returns sensitive values via the API. The export omits them (listed in `migration-summary.md`); after import the orchestrator creates an SG secret `tfc-<workflow>-<VAR>` with the value `CHANGE_ME` for each, references it from the workflow (`${secret::<name>}`, as an environment variable or IaC input) and lists it in `export/post-import-checklist.md`. Set the real values in the SG UI. `--no-secret-stubs` leaves SG secrets untouched. -- **Terraform version fallback (FOSS ceiling).** Workspaces set to `latest` or a version constraint use `SGDefaultTerraformVersion` at export time. Pinned versions are carried over as-is and tried first at import, so a custom runtime image or private runner that ships that binary keeps working. StackGuardian's _managed_ runtimes only go up to **1.5.7**, the last MPL-licensed (FOSS) Terraform release; newer versions are BSL-licensed and are not bundled. When the API rejects a workflow for that reason, the importer **automatically re-imports it with `SGDefaultTerraformVersion`**, patches the payload file to match, prints a notice, and records each case in `export/terraform-version-fallbacks.log`. Those workflows run a different Terraform than they did in TFC, so check compatibility before the first run. To keep a newer version, set `workspaceOverrides[<name>].terraformVersion` to a binary path mounted from a private runner (or use a custom runtime container template) and re-import. +- **Terraform version fallback (FOSS ceiling).** With `SGTerraformVersionSource = "carry"` (the default) pinned versions are carried over as-is and tried first at import, so a custom runtime image or private runner that ships that binary keeps working; workspaces set to `latest` or a version constraint use `SGDefaultTerraformVersion` at export time. StackGuardian's _managed_ runtimes only go up to **1.5.7**, the last MPL-licensed (FOSS) Terraform release; newer versions are BSL-licensed and are not bundled. When the API rejects a workflow for that reason, the importer **automatically re-imports it with `SGDefaultTerraformVersion`** (or, when that is `null`, without any version so the execution preset decides — see the next bullet), patches the payload file to match, prints a notice, and records each case in `export/terraform-version-fallbacks.log`. Those workflows run a different Terraform than they did in TFC, so check compatibility before the first run. To keep a newer version, set `workspaceOverrides[<name>].terraformVersion` to a binary path mounted from a private runner (or use a custom runtime container template) and re-import. +- **Execution presets.** StackGuardian org admins can define an execution preset (Settings → Runner groups → Execution presets): default runner constraints plus a Terraform and an OpenTofu configuration that the API applies to any new workflow whose payload does not carry those fields. To lean on it, set `SGTerraformVersionSource = "preset"` (no version is sent at all) and/or `SGDefaultRunnerConstraints = null` (no runner constraints are sent); `SGDefaultTerraformVersion = null` keeps carrying TFC pins but hands the unpinned and rejected ones to the preset. `init` offers these choices with the org's current preset shown inline, preflight prints what the preset would supply, and the import plan marks such cells as `preset (…)`. The preset's custom runtime image or runner-provided binary is inherited in every mode, since the migrator never sets those keys. - **State export is idempotent.** Re-running `terraform apply` only pulls state for workspaces not yet exported. Set `forceStateRefresh = true` to re-pull everything. Workspaces not using `remote` execution may export incomplete state — see the summary. - **Workflow naming.** `ResourceName` mirrors the TFC workspace name, sanitized to StackGuardian's rules (1-100 chars, `[-a-zA-Z0-9_]`); any rename is listed in the summary and the checklist. - **Preflight.** `apply`, `import` and `all` first verify the TFC and SG tokens, the TFC org and workspace selection, and that every connector, secret and runner group referenced in `terraform.tfvars` exists. Fix what it reports (or re-run `init`); `--skip-preflight` bypasses it. -- **Post-import checklist.** `export/post-import-checklist.md` (also printed) collects the secrets to fill in, failed imports, Terraform version fallbacks, failed VCS triggers, missing state exports and renames, with links into the SG UI. +- **Post-import checklist.** `export/post-import-checklist.md` collects the secrets to fill in, failed imports, Terraform version fallbacks, failed VCS triggers, missing state exports and renames, with links into the SG UI. The terminal shows one status line per section (plus links to the imported workflow groups); the file has the details. diff --git a/scripts/convert_hcl_to_json.sh b/scripts/convert_hcl_to_json.sh index cb778f5..db51ae4 100755 --- a/scripts/convert_hcl_to_json.sh +++ b/scripts/convert_hcl_to_json.sh @@ -41,8 +41,11 @@ tmpfile="$WORKDIR/updated.ndjson" : >"$tmpfile" JSON_PATH=".VCSConfig.iacInputData.data" +converted=0 +touched=0 for ((i = 0; i < length; i++)); do + wf_converted=0 # Extract ith object obj=$($JQ_BIN ".[$i]" <<<"$json_data") @@ -86,6 +89,8 @@ for ((i = 0; i < length; i++)); do if [[ -n "$parsed" && "$parsed" != "null" ]]; then log " workflow $((i + 1)): converted '$key' from HCL to JSON" new_val=$($JQ_BIN --arg k "$key" --argjson v "$parsed" '. + {($k): $v}' <<<"$new_val") + converted=$((converted + 1)) + wf_converted=1 else sg_warn "$(sg_rel "$INPUT_FILE_JSON") workflow $((i + 1)): could not parse '$key' as HCL; keeping original value" fi @@ -93,6 +98,7 @@ for ((i = 0; i < length; i++)); do # Assign the converted data back at JSON_PATH updated_obj=$($JQ_BIN --argjson nv "$new_val" "$JSON_PATH = \$nv" <<<"$obj") + touched=$((touched + wf_converted)) echo "$updated_obj" >>"$tmpfile" done @@ -102,4 +108,9 @@ done outfile="$WORKDIR/output.json" $JQ_BIN -s '.' "$tmpfile" >"$outfile" mv "$outfile" "$INPUT_FILE_JSON" -log "Done. Updated $(sg_rel "$INPUT_FILE_JSON") in place." +# One result line per file (the orchestrator shows it as-is). +if [ "$converted" -gt 0 ]; then + sg_log "$(basename "$INPUT_FILE_JSON"): $converted HCL value(s) converted to JSON in $touched of $length workflow(s)" +else + sg_log "$(basename "$INPUT_FILE_JSON"): nothing to convert ($length workflow(s), values already JSON)" +fi diff --git a/scripts/enrich_variable_sets.sh b/scripts/enrich_variable_sets.sh index 88fd77d..8a653a2 100755 --- a/scripts/enrich_variable_sets.sh +++ b/scripts/enrich_variable_sets.sh @@ -78,7 +78,9 @@ if [ "$set_count" -eq 0 ]; then sg_log "no variable sets found; nothing to enrich" exit 0 fi -sg_log "resolving $set_count variable set(s) across workspaces..." +sg_log "resolving $set_count variable set(s) across workspaces:" +# One line per set: name, scope, size — so it is clear where merged vars come from. +"$JQ_BIN" -r '.[] | " - \(.name): \(if .global then "global" else ([(if (.projids | length) > 0 then "\(.projids | length) project(s)" else empty end), (if (.wsids | length) > 0 then "\(.wsids | length) workspace(s)" else empty end)] | if length == 0 then "unassigned" else join(", ") end) end), \(.vars | length) var(s)\(if .priority then ", priority" else "" end)"' "$WORK/sets.json" >&2 # Per workspace -> list of winning vars (set-vs-set precedence resolved; tagged # with priority + sensitive + conflict). rank: non-priority global/proj/ws = 1/2/3, @@ -113,6 +115,7 @@ IGNORE_JSON="$(tfvars_get_json .ignoreVarPatterns)" # Merge the effective set vars into each payload, then report counts. for f in "$@"; do before_tf="$("$JQ_BIN" '[.[].VCSConfig.iacInputData.data | length] | add // 0' "$f")" + before_env="$("$JQ_BIN" '[.[].EnvironmentVariables | length] | add // 0' "$f")" out="$WORK/merged.json" "$JQ_BIN" --slurpfile eff "$WORK/effective.json" --argjson ignore "$IGNORE_JSON" ' ($eff[0]) as $E @@ -136,7 +139,12 @@ for f in "$@"; do ) ' "$f" >"$out" && mv "$out" "$f" after_tf="$("$JQ_BIN" '[.[].VCSConfig.iacInputData.data | length] | add // 0' "$f")" - sg_log "$(basename "$f"): +$((after_tf - before_tf)) terraform var(s) from variable sets" + after_env="$("$JQ_BIN" '[.[].EnvironmentVariables | length] | add // 0' "$f")" + if [ "$((after_tf - before_tf + after_env - before_env))" -eq 0 ]; then + sg_log "$(basename "$f"): no new variables (sets only override or add nothing here)" + else + sg_log "$(basename "$f"): +$((after_tf - before_tf)) terraform, +$((after_env - before_env)) env var(s) from variable sets" + fi done # Report sensitive set vars (cannot be migrated) and key conflicts. diff --git a/scripts/lib/checklist.sh b/scripts/lib/checklist.sh index 0bb3c61..2bfa9e1 100644 --- a/scripts/lib/checklist.sh +++ b/scripts/lib/checklist.sh @@ -7,7 +7,12 @@ SG_UI_URL="${SG_UI_URL:-https://app.stackguardian.io}" wf_ui_url() { printf '%s/orchestrator/orgs/%s/wfgrps/%s/wfs/%s' "$SG_UI_URL" "$ORG" "$1" "$2"; } +wfgrp_ui_url() { printf '%s/orchestrator/orgs/%s/wfgrps/%s' "$SG_UI_URL" "$ORG" "$1"; } secrets_ui_url() { printf '%s/orchestrator/orgs/%s?tab=secrets' "$SG_UI_URL" "$ORG"; } +# Set by write_checklist: how many checklist items still need a human (read by +# finish_line in migrate.sh). +# shellcheck disable=SC2034 +CHECKLIST_OPEN=0 # secret_name_for <workflow> <var> — SG secret name for a stubbed variable. secret_name_for() { printf 'tfc-%s-%s' "$1" "$2" | tr -c 'A-Za-z0-9_-\n' '-' | cut -c1-100; } @@ -84,73 +89,116 @@ create_secret_stubs() { return 0 } -# write_checklist — render export/post-import-checklist.md and print it. +# _cl_count <text> — number of non-empty lines in a block of items. +_cl_count() { + if [ -z "$1" ]; then printf '0'; else printf '%s\n' "$1" | grep -c . || true; fi +} + +# _cl_status <count> <ok-text> <attention-text> — one terminal line per section. +_cl_status() { + if [ "${1:-0}" -gt 0 ]; then printf ' %s!%s %s\n' "$C_YELLOW" "$C_RESET" "$3" >&2 + else printf ' %s✓%s %s\n' "$C_GREEN" "$C_RESET" "$2" >&2; fi +} + +# _cl_block <items> — the markdown list for a section, or "- None." +_cl_block() { if [ -n "$1" ]; then printf '%s\n\n' "$1"; else printf -- '- None.\n\n'; fi; } + +# write_checklist — render export/post-import-checklist.md, then show what +# landed and which sections still need a human (the file has the details and +# links; the terminal gets one status line per section). Sets CHECKLIST_OPEN. write_checklist() { - local out="$EXPORT_DIR/post-import-checklist.md" summary="$EXPORT_DIR/migration-summary.json" st items + local out="$EXPORT_DIR/post-import-checklist.md" summary="$EXPORT_DIR/migration-summary.json" st + local i_secrets i_unstubbed="" i_failed i_fallback="" i_unpinned="" i_preset="" i_trig i_state="" i_nonremote="" i_renamed="" + local n_secrets n_unstubbed n_failed n_fallback n_unpinned n_preset n_trig n_state n_nonremote n_renamed + local f seg grp n total=0 gw preset_desc="" + local -a glines=() st="$(state_read)" + [ -n "${SG_PRESET_JSON:-}" ] && preset_desc="$(sg_preset_desc "$SG_PRESET_JSON")" + + # --- collect the items once (markdown lines) -------------------------------- + i_secrets="$(printf '%s' "$st" | "$JQ_BIN" -r --arg ui "$SG_UI_URL" --arg org "$ORG" '.secrets // {} | to_entries[] | "- [ ] `\(.key)` — \(.value.category) var `\(.value.var)` of [\(.value.group)/\(.value.workflow)](\($ui)/orchestrator/orgs/\($org)/wfgrps/\(.value.group)/wfs/\(.value.workflow))"')" + if [ -f "$summary" ]; then + i_unstubbed="$("$JQ_BIN" -r --argjson stubbed "$(printf '%s' "$st" | "$JQ_BIN" -c '[.secrets // {} | .[] | "\(.workspace)|\(.category):\(.var)"]')" \ + '.skippedSensitiveVars | to_entries[] | .key as $ws | .value[] | select(($ws + "|" + .) as $k | $stubbed | index($k) == null) | "- [ ] `\(.)` of workspace `\($ws)` — no stub created (workflow missing or --no-secret-stubs); create the secret and reference it by hand"' "$summary")" + i_unpinned="$("$JQ_BIN" -r --arg fb "${SG_TF_FALLBACK_LABEL:-the fallback version}" '.terraformVersionFallbacks | to_entries[] | "- [ ] `\(.key)` was not pinned in TFC (\"\(.value)\"); it runs \($fb) — confirm it is compatible"' "$summary")" + # Preset mode: no version was sent for anyone, so every workflow may run + # something other than its TFC version — list them all with what TFC had. + if [ "$("$JQ_BIN" -r '.terraformVersionSource // "carry"' "$summary")" = "preset" ]; then + i_preset="$("$JQ_BIN" -r --arg p "${preset_desc:-the execution preset of the org}" '.tfcTerraformVersions // {} | to_entries[] | "- [ ] `\(.key)` ran Terraform \(.value) in TFC; now \($p) applies — run a plan before relying on it"' "$summary")" + fi + i_nonremote="$("$JQ_BIN" -r '.nonRemoteExecutionModes | to_entries[] | "- [ ] `\(.key)` used `\(.value)` execution in TFC — its state may live outside TFC; verify the exported state is current"' "$summary")" + i_renamed="$("$JQ_BIN" -r '.renamedWorkspaces | to_entries[] | "- `\(.key)` → `\(.value)`"' "$summary")" + fi + i_failed="$(printf '%s' "$st" | "$JQ_BIN" -r '.import // {} | to_entries[] | .value.group as $g | .value.failed[]? | "- [ ] `\($g)/\(.)` — see the import output for the API error; fix terraform.tfvars (or workspaceOverrides) and re-run `./sg-migrate.sh import`"')" + [ -s "$EXPORT_DIR/terraform-version-fallbacks.log" ] && i_fallback="$(sed 's/^/- [ ] /' "$EXPORT_DIR/terraform-version-fallbacks.log")" + i_trig="$(printf '%s' "$st" | "$JQ_BIN" -r '.triggers // {} | to_entries[] | .value.group as $g | (.value.failed[]? | "- [ ] `\($g)/\(.)` — trigger registration failed; check the connector has admin/webhook rights on the repository, then re-run `./sg-migrate.sh triggers`"), (.value.missing[]? | "- [ ] `\($g)/\(.)` — workflow was not imported, so no trigger was registered")')" + [ -s "$EXPORT_DIR/state-export-failures.log" ] && i_state="$(sed 's/^/- [ ] /' "$EXPORT_DIR/state-export-failures.log")" + n_secrets="$(_cl_count "$i_secrets")"; n_unstubbed="$(_cl_count "$i_unstubbed")"; n_failed="$(_cl_count "$i_failed")" + n_fallback="$(_cl_count "$i_fallback")"; n_unpinned="$(_cl_count "$i_unpinned")"; n_preset="$(_cl_count "$i_preset")"; n_trig="$(_cl_count "$i_trig")" + n_state="$(_cl_count "$i_state")"; n_nonremote="$(_cl_count "$i_nonremote")"; n_renamed="$(_cl_count "$i_renamed")" + + # --- the file ----------------------------------------------------------------- { printf '# Post-import checklist — StackGuardian org %s\n\n' "$ORG" printf 'Generated %s by stackguardian-migrator. Tick items as you complete them.\n\n' "$(state_now)" - - # 1. secrets printf '## 1. Set the real values of the placeholder secrets\n\n' printf 'TFC never exposes sensitive variable values, so each one was recreated as an SG secret with the value `CHANGE_ME` and referenced from its workflow as `${secret::<name>}`. Set the real values under [Org settings → Secrets](%s).\n\n' "$(secrets_ui_url)" - items="$(printf '%s' "$st" | "$JQ_BIN" -r --arg ui "$SG_UI_URL" --arg org "$ORG" '.secrets // {} | to_entries[] | "- [ ] `\(.key)` — \(.value.category) var `\(.value.var)` of [\(.value.group)/\(.value.workflow)](\($ui)/orchestrator/orgs/\($org)/wfgrps/\(.value.group)/wfs/\(.value.workflow))"')" - if [ -n "$items" ]; then printf '%s\n\n' "$items"; else printf -- '- None.\n\n'; fi - if [ -f "$summary" ]; then - items="$("$JQ_BIN" -r --argjson stubbed "$(printf '%s' "$st" | "$JQ_BIN" -c '[.secrets // {} | .[] | "\(.workspace)|\(.category):\(.var)"]')" \ - '.skippedSensitiveVars | to_entries[] | .key as $ws | .value[] | select(($ws + "|" + .) as $k | $stubbed | index($k) == null) | "- [ ] `\(.)` of workspace `\($ws)` — no stub created (workflow missing or --no-secret-stubs); create the secret and reference it by hand"' "$summary")" - [ -n "$items" ] && printf '%s\n\n' "$items" - fi - - # 2. failed imports + _cl_block "$i_secrets" + [ -n "$i_unstubbed" ] && printf '%s\n\n' "$i_unstubbed" printf '## 2. Workflows that failed to import\n\n' - items="$(printf '%s' "$st" | "$JQ_BIN" -r '.import // {} | to_entries[] | .value.group as $g | .value.failed[]? | "- [ ] `\($g)/\(.)` — see the import output for the API error; fix terraform.tfvars (or workspaceOverrides) and re-run `./sg-migrate.sh import`"')" - if [ -n "$items" ]; then printf '%s\n\n' "$items"; else printf -- '- None.\n\n'; fi - - # 3. terraform version fallbacks + _cl_block "$i_failed" printf '## 3. Verify workflows moved to a different Terraform version\n\n' - printf 'StackGuardian bundles managed Terraform only up to 1.5.7 (the last MPL/FOSS release). These workflows were pinned higher in TFC and now run the fallback version; run a plan and check for incompatibilities. To keep the newer version, point `workspaceOverrides[<ws>].terraformVersion` at a binary on a private runner and re-import.\n\n' - items="" - [ -s "$EXPORT_DIR/terraform-version-fallbacks.log" ] && items="$(sed 's/^/- [ ] /' "$EXPORT_DIR/terraform-version-fallbacks.log")" - if [ -f "$summary" ]; then - items="$items$(printf '%s' "$items" | grep -q . && echo)$("$JQ_BIN" -r '.terraformVersionFallbacks | to_entries[] | "- [ ] `\(.key)` was not pinned in TFC (\"\(.value)\"); SGDefaultTerraformVersion was used — confirm it is compatible"' "$summary")" - fi - if [ -n "$items" ]; then printf '%s\n\n' "$items"; else printf -- '- None.\n\n'; fi - - # 4. VCS triggers + printf 'StackGuardian bundles managed Terraform only up to 1.5.7 (the last MPL/FOSS release). Workflows listed here run a different version than they did in TFC: the fallback `SGDefaultTerraformVersion`, or the version from the execution preset of the org (Settings → Runner groups → Execution presets) when no version was sent. Run a plan and check for incompatibilities. To keep a newer version, point `workspaceOverrides[<ws>].terraformVersion` at a binary on a private runner, or give the execution preset a custom runtime image that ships it, and re-import.\n\n' + _cl_block "$(printf '%s\n%s\n%s' "$i_fallback" "$i_unpinned" "$i_preset" | grep . || true)" printf '## 4. VCS triggers\n\n' - items="$(printf '%s' "$st" | "$JQ_BIN" -r '.triggers // {} | to_entries[] | .value.group as $g | (.value.failed[]? | "- [ ] `\($g)/\(.)` — trigger registration failed; check the connector has admin/webhook rights on the repository, then re-run `./sg-migrate.sh triggers`"), (.value.missing[]? | "- [ ] `\($g)/\(.)` — workflow was not imported, so no trigger was registered")')" - if [ -n "$items" ]; then printf '%s\n\n' "$items"; else printf -- '- None failed. Push to a tracked branch or open a pull request to confirm the webhooks fire.\n\n'; fi - - # 5. state + if [ -n "$i_trig" ]; then printf '%s\n\n' "$i_trig"; else printf -- '- None failed. Push to a tracked branch or open a pull request to confirm the webhooks fire.\n\n'; fi printf '## 5. Terraform state\n\n' - if [ -s "$EXPORT_DIR/state-export-failures.log" ]; then - printf 'State could not be pulled from TFC for these workspaces; upload it manually (Workflow → Settings → State) or run an import in the new workflow.\n\n' - sed 's/^/- [ ] /' "$EXPORT_DIR/state-export-failures.log"; echo + if [ -n "$i_state" ]; then + printf 'State could not be pulled from TFC for these workspaces; upload it manually (Workflow → Settings → State) or run an import in the new workflow.\n\n%s\n\n' "$i_state" else printf -- '- All selected workspaces had their state exported.\n\n' fi - if [ -f "$summary" ]; then - items="$("$JQ_BIN" -r '.nonRemoteExecutionModes | to_entries[] | "- [ ] `\(.key)` used `\(.value)` execution in TFC — its state may live outside TFC; verify the exported state is current"' "$summary")" - [ -n "$items" ] && printf '%s\n\n' "$items" - fi - - # 6. renames - if [ -f "$summary" ] && [ "$("$JQ_BIN" '.renamedWorkspaces | length' "$summary")" -gt 0 ]; then - printf '## 6. Renamed workflows\n\nThese TFC workspace names were not valid StackGuardian workflow names and were adjusted:\n\n' - "$JQ_BIN" -r '.renamedWorkspaces | to_entries[] | "- `\(.key)` → `\(.value)`"' "$summary"; echo + [ -n "$i_nonremote" ] && printf '%s\n\n' "$i_nonremote" + if [ -n "$i_renamed" ]; then + printf '## 6. Renamed workflows\n\nThese TFC workspace names were not valid StackGuardian workflow names and were adjusted:\n\n%s\n\n' "$i_renamed" fi - printf '## Finally\n\n- [ ] Run a plan on one workflow per project and compare with the last TFC run.\n- [ ] Disable auto-apply / triggers on the TFC workspaces once StackGuardian owns the deployments.\n' } >"$out" + + # --- the terminal view -------------------------------------------------------- sg_step "Post-import checklist" - sed 's/^/ /' "$out" >&2 - sg_dim "saved to $(sg_rel "$out")" + for f in "${PF[@]}"; do + seg="$(seg_of "$f")" + grp="$(group_for "$seg")" + n="$(printf '%s' "$st" | "$JQ_BIN" -r --arg s "$seg" '.import[$s].imported // [] | length')" + total=$((total + n)) + glines+=("$grp ($n)|$(wfgrp_ui_url "$grp")") + done + if [ "$total" -gt 0 ]; then + printf ' %s✓%s %s\n' "$C_GREEN" "$C_RESET" "$total workflow(s) are in StackGuardian org $ORG:" >&2 + gw="$(sg_maxlen 10 "${glines[@]%%|*}")" + for n in "${glines[@]}"; do printf " %-${gw}s %s%s%s\n" "${n%%|*}" "$C_DIM" "${n#*|}" "$C_RESET" >&2; done + fi + _cl_status "$((n_secrets + n_unstubbed))" "secrets: none to fill in" \ + "secrets: set the real value of $n_secrets placeholder secret(s) (value CHANGE_ME)${i_unstubbed:+; $n_unstubbed sensitive var(s) have no stub}" + _cl_status "$n_failed" "imports: none failed" "imports: $n_failed workflow(s) failed — fix and re-run '$PROG import'" + if [ "$n_preset" -gt 0 ]; then + _cl_status "$n_preset" "" "Terraform version: all $n_preset workflow(s) take the execution preset's version (${preset_desc:-see the SG org settings}) instead of their TFC version — run a plan before relying on them" + else + _cl_status "$((n_fallback + n_unpinned))" "Terraform version: every workflow keeps its TFC version" \ + "Terraform version: $((n_fallback + n_unpinned)) workflow(s) run ${SG_TF_FALLBACK_LABEL:-the fallback version} instead of what TFC used — run a plan before relying on them" + fi + _cl_status "$n_trig" "VCS triggers: registered for every workflow that had them" "VCS triggers: $n_trig workflow(s) without triggers — see the checklist, then '$PROG triggers'" + _cl_status "$((n_state + n_nonremote))" "state: exported for every selected workspace" \ + "state: $n_state workspace(s) without exported state${i_nonremote:+, $n_nonremote with non-remote execution} — upload by hand" + [ "$n_renamed" -gt 0 ] && _cl_status "$n_renamed" "" "names: $n_renamed workflow(s) were renamed to valid SG names" + # shellcheck disable=SC2034 + CHECKLIST_OPEN=$((n_secrets + n_unstubbed + n_failed + n_fallback + n_unpinned + n_preset + n_trig + n_state + n_nonremote)) + sg_dim "full checklist with links: $(sg_rel "$out")" } cmd_checklist() { - sg_step "Phase: checklist" + phase_begin "checklist" [ -n "${SG_API_TOKEN:-}" ] || die "SG_API_TOKEN is not set." [ -n "$ORG" ] || die "StackGuardian org not set (use --org or SG_ORG)." export SG_API_TOKEN SG_BASE_URL diff --git a/scripts/lib/preflight.sh b/scripts/lib/preflight.sh index bed47c6..ecf29b2 100644 --- a/scripts/lib/preflight.sh +++ b/scripts/lib/preflight.sh @@ -3,18 +3,21 @@ # # Verifies, before the long terraform apply or a bulk import, that the tokens # work and that everything terraform.tfvars refers to actually exists in TFC -# and StackGuardian. Prints one line per check (✓ ok, ! warning, ✗ failure) and -# fails the run when any check fails. Skipped with --skip-preflight. +# and StackGuardian — and that the pieces agree with each other (VCS connector +# kind vs. sourceConfigDestKind, repo URL prefix vs. where the TFC repositories +# live). Prints one line per check (✓ ok, ! warning, ✗ failure) and fails the +# run when any check fails. Skipped with --skip-preflight. +PF_OK=0 PF_FAIL=0 PF_WARN=0 -pf_ok() { printf ' %s✓%s %s\n' "$C_GREEN" "$C_RESET" "$*" >&2; } +pf_ok() { printf ' %s✓%s %s\n' "$C_GREEN" "$C_RESET" "$*" >&2; PF_OK=$((PF_OK + 1)); } pf_warn() { printf ' %s!%s %s\n' "$C_YELLOW" "$C_RESET" "$*" >&2; PF_WARN=$((PF_WARN + 1)); } pf_fail() { printf ' %s✗%s %s\n' "$C_RED$C_BOLD" "$C_RESET" "$*" >&2; PF_FAIL=$((PF_FAIL + 1)); } # --- Terraform Cloud ----------------------------------------------------------- preflight_tfc() { - local host token org body n names_json tags_json ignore_json jqb + local host token org body n sel jqb jqb="$(sg_resolve jq sg_ensure_jq)" host="$(tfc_hostname)" org="$(tfvars_get .tfOrg)" @@ -44,22 +47,15 @@ preflight_tfc() { fi # Workspace selection: mirror the module's filters (names glob, include/exclude tags). if body="$(tfc_list_workspaces "$org" 2>/dev/null)"; then - names_json="$(tfvars_get_json .workspacenames)" - tags_json="$(tfvars_get_json .tfWorkspaceTags)" - ignore_json="$(tfvars_get_json .tfWorkspaceIgnoreTags)" - n="$(printf '%s' "$body" | "$jqb" --argjson names "${names_json:-null}" --argjson tags "${tags_json:-null}" --argjson ignore "${ignore_json:-null}" ' - def glob($p): ("^" + ($p | gsub("\\*"; ".*")) + "$"); - [ .[] - | select(($names == null) or ($names == ["*"]) or ([.name] | inside([]) | not) and ([$names[] as $p | (.name | test(glob($p)))] | any)) - | select(($tags == null) or (($tags | length) == 0) or ([.tags[]?] | inside($tags) | not) or (([.tags[]?] | map(select(. as $t | $tags | index($t) != null)) | length) > 0)) - | select(($ignore == null) or (($ignore | length) == 0) or (([.tags[]?] | map(select(. as $t | $ignore | index($t) != null)) | length) == 0)) - ] | length')" + sel="$(tfc_select_workspaces "$body" "$(tfvars_get_json .workspacenames)" "$(tfvars_get_json .tfWorkspaceTags)" "$(tfvars_get_json .tfWorkspaceIgnoreTags)")" + n="$(printf '%s' "$sel" | "$jqb" 'length')" if [ "$n" -gt 0 ]; then pf_ok "$n workspace(s) match the selection (of $(printf '%s' "$body" | "$jqb" 'length') in the org)" else pf_warn "no workspace matches workspacenames/tfWorkspaceTags/tfWorkspaceIgnoreTags — apply would export nothing" fi PF_TFC_WORKSPACES="$body" + PF_TFC_SELECTED="$sel" else pf_warn "could not list workspaces for '$org' (HTTP $TFC_HTTP_CODE)" fi @@ -69,7 +65,7 @@ preflight_tfc() { # --- StackGuardian ------------------------------------------------------------ # preflight_sg <required:0|1> preflight_sg() { - local required="$1" ints names id n jqb rg + local required="$1" ints names label id kind n jqb rg jqb="$(sg_resolve jq sg_ensure_jq)" if [ -z "${SG_API_TOKEN:-}" ] || [ -z "$ORG" ]; then if [ "$required" -eq 1 ]; then @@ -90,30 +86,38 @@ preflight_sg() { return 0 fi pf_ok "StackGuardian credentials valid (org '$ORG', $SG_BASE_URL)" + PF_SG_INTS="$ints" + # The execution preset (org workflow defaults) decides whatever tfvars leaves + # to it; preflight_config reports it next to the version/runner settings. + if PF_PRESET="$(sg_execution_preset)"; then PF_PRESET_READ=1; else PF_PRESET_READ=0; fi - # Every integration id referenced anywhere in tfvars must exist. + # Every integration id referenced anywhere in tfvars must exist. Each line is + # "<role>\t<id>" so the report can say what the connector is for. names="$(printf '%s' "$ints" | "$jqb" -c '[.[].name]')" - while IFS= read -r id; do + while IFS=$'\t' read -r label id; do [ -n "$id" ] || continue case "$id" in - *CHANGE_ME* | *INTEGRATION_ID*) pf_fail "placeholder connector id '$id' in $(sg_rel "$TFVARS") — run '$PROG init' or edit the file" ;; + *CHANGE_ME* | *INTEGRATION_ID*) pf_fail "placeholder $label connector id '$id' in $(sg_rel "$TFVARS") — run '$PROG init' or edit the file" ;; /integrations/*) if printf '%s' "$names" | "$jqb" -e --arg n "${id#/integrations/}" 'index($n) != null' >/dev/null; then - pf_ok "connector $id exists" + kind="$(sg_integration_type "$ints" "$id")" + pf_ok "$label connector ${id#/integrations/} exists${kind:+ ($kind)}" else - pf_fail "connector $id not found in org '$ORG' (available: $(printf '%s' "$names" | "$jqb" -r 'join(", ")'))" + pf_fail "$label connector $id not found in org '$ORG' (available: $(printf '%s' "$names" | "$jqb" -r 'join(", ")'))" fi ;; /secrets/*) if sg_secret_exists "${id#/secrets/}"; then pf_ok "secret $id exists"; else pf_fail "secret $id not found in org '$ORG'"; fi ;; - *) pf_warn "unrecognised integration id '$id' (expected /integrations/<name> or /secrets/<name>)" ;; + *) pf_warn "unrecognised $label integration id '$id' (expected /integrations/<name> or /secrets/<name>)" ;; esac done < <(tfvars_json | "$jqb" -r ' - [ .SGDefaultVCSAuthIntegrationID, - (.SGDefaultDeploymentPlatformConfig // [])[]?.config.integrationId, - ((.workspaceOverrides // {}) | to_entries[]? | .value | (.vcsAuthIntegrationID, ((.DeploymentPlatformConfig // [])[]?.config.integrationId))) ] - | map(select(. != null and . != "")) | unique | .[]') + [ (.SGDefaultVCSAuthIntegrationID | select(. != null and . != "") | ["VCS", .]), + ((.SGDefaultDeploymentPlatformConfig // [])[]?.config.integrationId | select(. != null and . != "") | ["cloud", .]), + ((.workspaceOverrides // {}) | to_entries[]? | .value + | ((.vcsAuthIntegrationID | select(. != null and . != "") | ["override VCS", .]), + ((.DeploymentPlatformConfig // [])[]?.config.integrationId | select(. != null and . != "") | ["override cloud", .]))) ] + | unique_by(.[1]) | .[] | @tsv') # Every private runner group must exist. while IFS= read -r rg; do @@ -123,41 +127,137 @@ preflight_sg() { [ (.SGDefaultRunnerConstraints // {} | select(.type == "private") | .names[]?), ((.workspaceOverrides // {}) | to_entries[]? | .value.RunnerConstraints // {} | select(.type == "private") | .names[]?) ] | unique | .[]') - n="$(tfvars_json | "$jqb" -r '(.SGDefaultRunnerConstraints // {}).type // "shared"')" - if [ "$n" = "shared" ]; then pf_ok "runners: StackGuardian shared runners"; fi + if tfvars_is_null SGDefaultRunnerConstraints; then + if [ "${PF_PRESET_READ:-0}" -eq 1 ]; then + pf_ok "runners: from the org's execution preset — $(sg_preset_runner_desc "$PF_PRESET")" + else + pf_ok "runners: from the org's execution preset (platform default: shared runners)" + fi + else + n="$(tfvars_json | "$jqb" -r '(.SGDefaultRunnerConstraints // {}).type // "shared"')" + if [ "$n" = "shared" ]; then pf_ok "runners: StackGuardian shared runners"; fi + fi return 0 } -# --- static config -------------------------------------------------------------- +# --- static config + consistency --------------------------------------------------- +# _pf_host <url> — host part of a URL. +_pf_host() { local h="${1#*://}"; printf '%s' "${h%%/*}"; } + +# _pf_kind_of_host <host> — the provider kind a well-known VCS host implies. +_pf_kind_of_host() { + case "$1" in + github.com | www.github.com) printf 'GITHUB_COM' ;; + gitlab.com | www.gitlab.com) printf 'GITLAB_COM' ;; + bitbucket.org | www.bitbucket.org) printf 'BITBUCKET_ORG' ;; + dev.azure.com | *.visualstudio.com) printf 'AZURE_DEVOPS' ;; + *) printf '' ;; + esac +} + preflight_config() { - local v ceiling="1.5.7" export_path want jqb + local v kind ceiling="1.5.7" export_path want jqb prefix host conn conn_kind line prov tfc_prefix tfc_kind jqb="$(sg_resolve jq sg_ensure_jq)" + + # VCS kind, and does it agree with the connector / the repositories? v="$(tfvars_get .SGDefaultSourceConfigDestKind GIT_OTHER)" case "$v" in - GITHUB_COM | GITHUB_APP_CUSTOM | GIT_OTHER | INLINE | BITBUCKET_ORG | GITLAB_COM | AZURE_DEVOPS) pf_ok "SGDefaultSourceConfigDestKind = $v" ;; + GITHUB_COM | GITHUB_APP_CUSTOM | GIT_OTHER | INLINE | BITBUCKET_ORG | GITLAB_COM | AZURE_DEVOPS) ;; *) pf_fail "SGDefaultSourceConfigDestKind '$v' is not one of GITHUB_COM, GITLAB_COM, BITBUCKET_ORG, AZURE_DEVOPS, GIT_OTHER" ;; esac + conn="$(tfvars_get .SGDefaultVCSAuthIntegrationID)" + conn_kind="" + [ -n "${PF_SG_INTS:-}" ] && [ -n "$conn" ] && conn_kind="$(sg_vcs_kind_of "$(sg_integration_type "$PF_SG_INTS" "$conn")")" + if [ -n "$conn_kind" ] && [ "$conn_kind" != "$v" ]; then + pf_fail "VCS kind $v does not match connector ${conn#/integrations/}, which is a $conn_kind connector — set SGDefaultSourceConfigDestKind = \"$conn_kind\" (or pick another connector)" + elif [ -n "$conn_kind" ]; then + pf_ok "VCS kind $v matches connector ${conn#/integrations/}" + else + pf_ok "VCS kind $v" + fi + + # Repo URL prefix: compare with where the TFC workspaces' repositories live + # (the transformer emits <prefix>/<tfc repo identifier>); otherwise with the + # kind's well-known host. + prefix="$(tfvars_get .SGDefaultIACVCSRepoPrefix)" + host="$(_pf_host "$prefix")" + tfc_prefix="" + tfc_kind="" + if [ -n "${PF_TFC_SELECTED:-}" ]; then + while IFS=$'\t' read -r prov _ line; do + [ -n "$prov" ] || continue + tfc_kind="$(tfc_vcs_kind_for "$prov")" + [ "$line" != "-" ] && tfc_prefix="$line" + break # most common provider only + done <<<"$(tfc_vcs_summary "$PF_TFC_SELECTED")" + fi + if [ -z "$prefix" ] || [ "$prefix" = "https://VCS_PROVIDER_DOMAIN" ]; then + pf_fail "SGDefaultIACVCSRepoPrefix is not set — the repositories' base URL, e.g. https://github.com" + elif [ -n "$tfc_prefix" ]; then + case "$host" in + *"$(_pf_host "$tfc_prefix")") pf_ok "repo URL prefix $prefix matches the TFC repositories" ;; + *) pf_warn "repo URL prefix $prefix does not match where the TFC repositories live ($tfc_prefix) — every workflow would clone from the wrong place unless the repositories moved; check SGDefaultIACVCSRepoPrefix" ;; + esac + else + kind="$(_pf_kind_of_host "$host")" + if [ -n "$kind" ] && [ "$kind" != "$v" ] && [ "$v" != "GIT_OTHER" ]; then + pf_warn "repo URL prefix $prefix looks like $kind but the VCS kind is $v — check SGDefaultIACVCSRepoPrefix / SGDefaultSourceConfigDestKind" + else + pf_ok "repo URL prefix $prefix" + fi + fi + if [ -n "$tfc_kind" ] && [ "$tfc_kind" != "$v" ] && [ "$v" != "GIT_OTHER" ]; then + pf_warn "the TFC workspaces are connected to $(tfc_vcs_label_for "$prov") but the VCS kind is $v" + fi + while IFS= read -r v; do [ -n "$v" ] || continue case "$v" in - AWS_STATIC | AWS_RBAC | AWS_OIDC | AZURE_STATIC | AZURE_OIDC | AZURE_MANAGED_ID_OIDC | GCP_STATIC | GCP_OIDC) pf_ok "DeploymentPlatformConfig kind $v" ;; - *) pf_fail "DeploymentPlatformConfig kind '$v' is not a cloud connector kind (AWS_STATIC, AWS_RBAC, AWS_OIDC, AZURE_STATIC, AZURE_OIDC, AZURE_MANAGED_ID_OIDC, GCP_STATIC, GCP_OIDC) — a VCS connector was picked as the cloud connector?" ;; + AWS_STATIC | AWS_RBAC | AWS_OIDC | AZURE_STATIC | AZURE_OIDC | AZURE_MANAGED_ID_OIDC | GCP_STATIC | GCP_OIDC) pf_ok "cloud connector kind $v" ;; + *) pf_fail "cloud connector kind '$v' (DeploymentPlatformConfig) is not one of AWS_STATIC, AWS_RBAC, AWS_OIDC, AZURE_STATIC, AZURE_OIDC, AZURE_MANAGED_ID_OIDC, GCP_STATIC, GCP_OIDC — a VCS connector was picked as the cloud connector?" ;; esac done < <(tfvars_json | "$jqb" -r '[ (.SGDefaultDeploymentPlatformConfig // [])[]?.kind, ((.workspaceOverrides // {}) | to_entries[]? | .value.DeploymentPlatformConfig // [] | .[]?.kind) ] | map(select(. != null)) | unique | .[]') - v="$(tfvars_get .SGDefaultTerraformVersion TERRAFORM-1.5.7)" - if [[ "$v" =~ ^TERRAFORM-([0-9]+)\.([0-9]+)\.([0-9]+)$ ]]; then - if [ "$(printf '%03d%03d%03d' "${BASH_REMATCH[1]}" "${BASH_REMATCH[2]}" "${BASH_REMATCH[3]}")" -gt "001005007" ]; then - pf_warn "SGDefaultTerraformVersion $v is above SG's managed ceiling ($ceiling); fallbacks would fail too unless a private runner ships that binary" + + # Terraform version policy, and what the execution preset would supply where + # tfvars leaves the decision to it. + local src preset_note="" + src="$(tfvars_get .SGTerraformVersionSource carry)" + case "$src" in + carry | preset) ;; + *) + pf_fail "SGTerraformVersionSource '$src' must be \"carry\" or \"preset\"" + src="carry" + ;; + esac + [ "${PF_PRESET_READ:-0}" -eq 1 ] && preset_note="$(sg_preset_desc "$PF_PRESET")" + if tfvars_is_null SGDefaultTerraformVersion; then v=""; else v="$(tfvars_get .SGDefaultTerraformVersion TERRAFORM-1.5.7)"; fi + if [ "$src" = "preset" ]; then + if [ -n "$preset_note" ]; then + pf_ok "Terraform version: from the org's execution preset — $preset_note" else - pf_ok "SGDefaultTerraformVersion = $v" + pf_warn "Terraform version: from the org's execution preset, which could not be read here (platform default if none is configured: managed Terraform 1.5.7 on shared runners)" fi else - pf_fail "SGDefaultTerraformVersion '$v' must look like TERRAFORM-1.5.7" + if [ -z "$v" ]; then + pf_ok "Terraform version: pinned TFC versions are carried over; unpinned or rejected pins go to the execution preset${preset_note:+ — $preset_note}" + elif [[ "$v" =~ ^TERRAFORM-([0-9]+)\.([0-9]+)\.([0-9]+)$ ]]; then + if [ "$(printf '%03d%03d%03d' "${BASH_REMATCH[1]}" "${BASH_REMATCH[2]}" "${BASH_REMATCH[3]}")" -gt "001005007" ]; then + pf_warn "fallback Terraform ${v#TERRAFORM-} (SGDefaultTerraformVersion) is above SG's managed ceiling ($ceiling); fallbacks would fail too unless a private runner ships that binary" + else + pf_ok "Terraform version: pinned TFC versions are carried over; fallback ${v#TERRAFORM-} is within SG's managed ceiling ($ceiling)" + fi + else + pf_fail "SGDefaultTerraformVersion '$v' must look like TERRAFORM-1.5.7 (or be null to defer to the execution preset)" + fi + if [ "${PF_PRESET_READ:-0}" -eq 1 ] && sg_preset_runner_provided "$PF_PRESET"; then + pf_warn "the execution preset mounts a runner-provided Terraform binary; with SGTerraformVersionSource = \"carry\" the carried pins and that binary would be sent together — consider \"preset\"" + fi fi + export_path="$(tfvars_get .exportPath export)" case "$export_path" in /*) want="$export_path" ;; *) want="$SG_REPO_ROOT/$export_path" ;; esac if [ "$(cd "$(dirname "$want")" 2>/dev/null && pwd)/$(basename "$want")" = "$(cd "$(dirname "$EXPORT_DIR")" 2>/dev/null && pwd)/$(basename "$EXPORT_DIR")" ]; then - pf_ok "export directory: $(sg_rel "$EXPORT_DIR")" + pf_ok "export directory $(sg_rel "$EXPORT_DIR")/" else pf_warn "exportPath in tfvars ($export_path) differs from the orchestrator's export dir ($(sg_rel "$EXPORT_DIR")) — payloads would be written where later phases don't look" fi @@ -178,7 +278,7 @@ preflight_config() { preflight_import_inputs() { payload_files if [ "${#PF[@]}" -gt 0 ]; then - pf_ok "${#PF[@]} payload file(s) in $(sg_rel "$EXPORT_DIR")" + pf_ok "${#PF[@]} payload file(s) in $(sg_rel "$EXPORT_DIR")/" else pf_fail "no payload files in $(sg_rel "$EXPORT_DIR") — run '$PROG apply' first" fi @@ -192,11 +292,16 @@ preflight_run() { local ctx="$1" [ "${SKIP_PREFLIGHT:-0}" -eq 1 ] && { sg_warn "preflight skipped (--skip-preflight)"; return 0; } [ "${PREFLIGHT_DONE:-0}" -eq 1 ] && return 0 - sg_step "Preflight ($ctx)" + sg_step "Preflight" [ -f "$TFVARS" ] || die "Missing $(sg_rel "$TFVARS"). Run: $PROG init" + PF_OK=0 PF_FAIL=0 PF_WARN=0 PF_TFC_WORKSPACES="" + PF_TFC_SELECTED="" + PF_SG_INTS="" + PF_PRESET="{}" + PF_PRESET_READ=0 local parse_err if ! parse_err="$(tfvars_valid)"; then pf_fail "$(sg_rel "$TFVARS") is not valid HCL: ${parse_err:-parse error}" @@ -225,9 +330,9 @@ preflight_run() { die "preflight found $PF_FAIL problem(s); fix them (or re-run '$PROG init') and try again. Use --skip-preflight to bypass." fi if [ "$PF_WARN" -gt 0 ]; then - sg_warn "preflight passed with $PF_WARN warning(s)" + sg_warn "preflight passed with $PF_WARN warning(s) — read them before continuing" else - sg_success "preflight passed" + sg_success "preflight passed ($PF_OK checks)" fi } diff --git a/scripts/lib/prompt.sh b/scripts/lib/prompt.sh index 7e754e5..fec08bc 100644 --- a/scripts/lib/prompt.sh +++ b/scripts/lib/prompt.sh @@ -93,9 +93,14 @@ sg_select() { return 0 fi printf '%s? %s%s\n' "$C_BOLD" "$q" "$C_RESET" >&2 + # Descriptions line up in one column sized to the longest value (capped so a + # single very long name cannot push everything off-screen). + local w + w="$(sg_maxlen 10 "${vals[@]}")" + [ "$w" -gt 40 ] && w=40 for ((i = 0; i < n; i++)); do if [ -n "${descs[i]}" ]; then - printf ' %s%2d)%s %-10s %s%s%s\n' "$C_CYAN" "$((i + 1))" "$C_RESET" "${vals[i]}" "$C_DIM" "— ${descs[i]}" "$C_RESET" >&2 + printf " %s%2d)%s %-${w}s %s%s%s\n" "$C_CYAN" "$((i + 1))" "$C_RESET" "${vals[i]}" "$C_DIM" "— ${descs[i]}" "$C_RESET" >&2 else printf ' %s%2d)%s %s\n' "$C_CYAN" "$((i + 1))" "$C_RESET" "${vals[i]}" >&2 fi diff --git a/scripts/lib/report.sh b/scripts/lib/report.sh index 36d39f5..edadbf5 100644 --- a/scripts/lib/report.sh +++ b/scripts/lib/report.sh @@ -4,42 +4,65 @@ # confirmation prompt (and by 'import --dry-run'). # show_migration_summary — condensed view of export/migration-summary.json plus -# state-export-failures.log, so nobody has to know to open the files. +# the state export result, so nobody has to know to open the files. show_migration_summary() { - local f="$EXPORT_DIR/migration-summary.json" jqb n + local f="$EXPORT_DIR/migration-summary.json" jqb n fails=0 states [ -f "$f" ] || return 0 jqb="$(sg_resolve jq sg_ensure_jq)" sg_step "Migration summary" - printf ' %s%-30s%s %s\n' "$C_BOLD" "TFC organisation" "$C_RESET" "$("$jqb" -r '.organization' "$f")" >&2 - printf ' %s%-30s%s %s\n' "$C_BOLD" "Workspaces exported" "$C_RESET" "$("$jqb" -r '.workspaceCount' "$f")" >&2 + sg_row "TFC organisation" "$("$jqb" -r '.organization' "$f")" + sg_row "Workspaces exported" "$("$jqb" -r '.workspaceCount' "$f")" "$jqb" -r '.projectWorkspaceCounts | to_entries[] | " tfc-\(.key | ascii_downcase | gsub("[^a-z0-9-]+"; "-")): \(.value) workflow(s)"' "$f" >&2 + if [ "$(tfvars_get .exportStateFiles true)" != "false" ]; then + states=0 + for n in "$EXPORT_DIR"/states/*.tfstate; do [ -f "$n" ] && states=$((states + 1)); done + [ -s "$EXPORT_DIR/state-export-failures.log" ] && fails="$(wc -l <"$EXPORT_DIR/state-export-failures.log" | tr -d ' ')" + if [ "$fails" -gt 0 ]; then + sg_row "State files" "$states exported, ${C_YELLOW}$fails failed${C_RESET} (see below)" + else + sg_row "State files" "$states exported to $(sg_rel "$EXPORT_DIR")/states/" + fi + fi + + # Version / runner policy (what the transformer sent, or left to the preset). + local src def + src="$("$jqb" -r '.terraformVersionSource // "carry"' "$f")" + def="$("$jqb" -r '.terraformVersionDefault // empty' "$f")" + if [ "$src" = "preset" ]; then + sg_row "Terraform version" "none sent; the org's execution preset applies at import" + elif [ -z "$def" ]; then + sg_row "Terraform version" "carried from TFC; unpinned or rejected pins go to the execution preset" + else + sg_row "Terraform version" "carried from TFC; fallback ${def#TERRAFORM-} for unpinned or rejected pins" + fi + [ "$("$jqb" -r '.runnerConstraintsSource // "config"' "$f")" = "preset" ] && sg_row "Runners" "none sent; the org's execution preset applies at import" - _summary_section "$f" '.skippedSensitiveVars' "Sensitive variables skipped (TFC never exposes them)" \ - 'to_entries[] | "\(.key): \(.value | join(", "))"' "recreated as SG secrets after import" - _summary_section "$f" '.strippedVars' "TFC-specific variables stripped (ignoreVarPatterns)" \ - 'to_entries[] | "\(.key): \(.value | join(", "))"' "" - _summary_section "$f" '.terraformVersionFallbacks' "Terraform version not pinned (SGDefaultTerraformVersion used)" \ - 'to_entries[] | "\(.key): \"\(.value)\""' "" - _summary_section "$f" '.nonRemoteExecutionModes' "Non-remote execution mode (state may not be in TFC)" \ - 'to_entries[] | "\(.key): \(.value)"' "" + _summary_section "$f" '.skippedSensitiveVars' "Sensitive variables skipped" \ + 'to_entries[] | "\(.key): \(.value | join(", "))"' "TFC never exposes their values; they become placeholder SG secrets after import" + _summary_section "$f" '.strippedVars' "TFC-specific variables stripped" \ + 'to_entries[] | "\(.key): \(.value | join(", "))"' "they only mean something inside Terraform Cloud (ignoreVarPatterns)" + _summary_section "$f" '.terraformVersionFallbacks' "Terraform version not pinned" \ + 'to_entries[] | "\(.key): \"\(.value)\""' "$([ -z "$def" ] && echo "left to the execution preset" || echo "the fallback ${def#TERRAFORM-} is used")" + _summary_section "$f" '.nonRemoteExecutionModes' "Non-remote execution mode" \ + 'to_entries[] | "\(.key): \(.value)"' "their state may not live in TFC" _summary_section "$f" '.renamedWorkspaces' "Renamed to a valid SG workflow name" \ 'to_entries[] | "\(.key) -> \(.value)"' "" - if [ -s "$EXPORT_DIR/state-export-failures.log" ]; then - n="$(wc -l <"$EXPORT_DIR/state-export-failures.log" | tr -d ' ')" - printf ' %s!%s %s\n' "$C_YELLOW" "$C_RESET" "state could not be exported for $n workspace(s):" >&2 + if [ "$fails" -gt 0 ]; then + printf ' %s!%s %s\n' "$C_YELLOW" "$C_RESET" "state could not be exported for $fails workspace(s):" >&2 sed 's/^/ /' "$EXPORT_DIR/state-export-failures.log" | head -10 >&2 - [ "$n" -gt 10 ] && sg_dim " ... see $(sg_rel "$EXPORT_DIR")/state-export-failures.log" + [ "$fails" -gt 10 ] && sg_dim " ... see $(sg_rel "$EXPORT_DIR")/state-export-failures.log" fi sg_dim "full report: $(sg_rel "$EXPORT_DIR")/migration-summary.md" } -# _summary_section <file> <jq-path> <title> <jq-line-filter> <hint> +# _summary_section <file> <jq-path> <title> <jq-line-filter> <hint> — prints +# "! <title> in <n> workspace(s) — <hint>" plus up to 10 detail lines. _summary_section() { local f="$1" path="$2" title="$3" lines="$4" hint="$5" jqb n jqb="$(sg_resolve jq sg_ensure_jq)" n="$("$jqb" -r "$path | length" "$f")" [ "$n" -gt 0 ] || return 0 - printf ' %s!%s %s (%s)%s\n' "$C_YELLOW" "$C_RESET" "$title" "$n" "${hint:+ — $hint}" >&2 + printf ' %s!%s %s in %s workspace(s)%s\n' "$C_YELLOW" "$C_RESET" "$title" "$n" "${hint:+ — $hint}" >&2 "$jqb" -r "$path | $lines" "$f" | head -10 | sed 's/^/ /' >&2 [ "$n" -gt 10 ] && sg_dim " ... $((n - 10)) more in migration-summary.md" return 0 @@ -51,42 +74,85 @@ tf_version_above_ceiling() { [ "$(printf '%03d%03d%03d' "${BASH_REMATCH[1]}" "${BASH_REMATCH[2]}" "${BASH_REMATCH[3]}")" -gt "001005007" ] } +# preset_labels — short labels for the plan and the fallback messages, from +# SG_PRESET_JSON (the org's execution preset, "" when it could not be read) and +# SG_DEFAULT_TF_VERSION ("" = drop the version so the preset decides): +# SG_PRESET_TFV "1.5.7" / "bin:/opt/tf" / "platform default" +# SG_PRESET_RUNNER_SHORT "shared" / "private:rg" +# SG_TF_FALLBACK_SHORT "1.5.7" / "preset" +# SG_TF_FALLBACK_LABEL "TERRAFORM-1.5.7" / "no pinned version (the org's execution preset decides: ...)" +# shellcheck disable=SC2034 # consumed by migrate.sh (do_import, tf_fallback_notice) and checklist.sh +preset_labels() { + SG_PRESET_TFV="" + SG_PRESET_RUNNER_SHORT="" + if [ -n "${SG_PRESET_JSON:-}" ]; then + SG_PRESET_TFV="$(printf '%s' "$SG_PRESET_JSON" | "$JQ_BIN" -r ' + ((.terraformDefaults // {}).TerraformConfig // {}) as $c + | if (($c.terraformBinPath // []) | length) > 0 then "bin:\($c.terraformBinPath[0].source // "")" + elif ($c.terraformVersion // "") == "" then "platform default" + else ($c.terraformVersion | ltrimstr("TERRAFORM-")) end')" + SG_PRESET_RUNNER_SHORT="$(printf '%s' "$SG_PRESET_JSON" | "$JQ_BIN" -r '(.RunnerConstraints // {}) | if (.type // "shared") == "private" then "private:\((.names // []) | join(","))" else "shared" end')" + fi + if [ -n "${SG_DEFAULT_TF_VERSION:-}" ]; then + SG_TF_FALLBACK_SHORT="${SG_DEFAULT_TF_VERSION#TERRAFORM-}" + SG_TF_FALLBACK_LABEL="$SG_DEFAULT_TF_VERSION" + else + SG_TF_FALLBACK_SHORT="preset" + SG_TF_FALLBACK_LABEL="the version from the org's execution preset ($(sg_preset_desc "${SG_PRESET_JSON:-}"))" + fi +} + # show_import_plan <payload>... — per-workflow table: what will be created or # updated, with the Terraform version (and fallback), runner, triggers and the -# number of variables / skipped secrets. Needs JQ_BIN, ORG, SG_API_TOKEN. +# number of variables / skipped secrets. "preset" marks a value the payload +# leaves to the org's execution preset. Columns are sized to their content. +# Needs JQ_BIN, ORG, SG_API_TOKEN, preset_labels. show_import_plan() { - local f seg grp existing summary rows + local f seg grp existing summary rows all_rows="" name action tfv runner trig vars secrets wn gn rw + local -a names=() groups=() summary="$EXPORT_DIR/migration-summary.json" [ -f "$summary" ] || summary="" - printf '\n %s%-28s %-24s %-7s %-28s %-8s %-8s %-5s %s%s\n' "$C_BOLD" "WORKFLOW" "GROUP" "ACTION" "TERRAFORM" "RUNNER" "TRIGGERS" "VARS" "SECRETS" "$C_RESET" >&2 for f in "$@"; do seg="$(seg_of "$f")" grp="$(group_for "$seg")" existing="$(sg_list_workflows "$grp")" printf '%s' "$existing" | "$JQ_BIN" -e 'type == "array"' >/dev/null 2>&1 || existing='[]' - rows="$("$JQ_BIN" -r --argjson ex "$existing" --arg def "$SG_DEFAULT_TF_VERSION" --argjson ws "$(ws_filter_json)" \ + rows="$("$JQ_BIN" -r --argjson ex "$existing" --arg grp "$grp" --argjson ws "$(ws_filter_json)" \ --slurpfile sum "${summary:-/dev/null}" ' ($sum[0] // {}) as $S | .[] | select(($ws | length) == 0 or (.ResourceName as $n | $ws | index($n) != null)) | ((.CLIConfiguration.TfStateFilePath // "") | sub(".*/"; "") | sub("\\.tfstate$"; "")) as $wsName | .ResourceName as $n - | [ $n, + | [ $n, $grp, (if ($ex | index($n)) != null then "update" else "create" end), - (.TerraformConfig.terraformVersion // "-"), - ((.RunnerConstraints // {}) | if .type == "private" then "private" else "shared" end), + (.TerraformConfig.terraformVersion // "preset"), + ((.RunnerConstraints // null) | if . == null then "preset" elif .type == "private" then "private" else "shared" end), (if (.VCSTriggers // null) != null then "yes" else "no" end), ((.VCSConfig.iacInputData.data // {}) | length), (($S.skippedSensitiveVars // {})[$wsName] // [] | length) ] | @tsv' "$f")" - while IFS=$'\t' read -r name action tfv runner trig vars secrets; do - [ -n "$name" ] || continue - if tf_version_above_ceiling "$tfv"; then tfv="${tfv#TERRAFORM-} -> ${SG_DEFAULT_TF_VERSION#TERRAFORM-} (fallback)"; else tfv="${tfv#TERRAFORM-}"; fi - [ "$secrets" = "0" ] && secrets="-" - case "$action" in create) action="${C_GREEN}create ${C_RESET}" ;; update) action="${C_YELLOW}update ${C_RESET}" ;; esac - printf ' %-28s %-24s %s %-28s %-8s %-8s %-5s %s\n' "$name" "$grp" "$action" "$tfv" "$runner" "$trig" "$vars" "$secrets" >&2 - done <<<"$rows" + [ -n "$rows" ] && all_rows="$all_rows${all_rows:+$'\n'}$rows" + names+=("$grp") done + while IFS=$'\t' read -r name grp _; do [ -n "$name" ] && names+=("$name"); done <<<"$all_rows" + wn="$(sg_maxlen 8 ${names[@]+"${names[@]}"})" + while IFS=$'\t' read -r _ grp _; do [ -n "$grp" ] && groups+=("$grp"); done <<<"$all_rows" + gn="$(sg_maxlen 5 ${groups[@]+"${groups[@]}"})" + # The RUNNER column only grows when a row defers to the preset ("preset (private:rg)"). + rw=8 + case "$all_rows" in *$'\t'preset$'\t'*) rw="$(sg_maxlen 8 "preset (${SG_PRESET_RUNNER_SHORT:-})")" ;; esac + printf "\n %s%-${wn}s %-${gn}s %-7s %-28s %-${rw}s %-8s %-5s %s%s\n" "$C_BOLD" "WORKFLOW" "GROUP" "ACTION" "TERRAFORM" "RUNNER" "TRIGGERS" "VARS" "SECRETS" "$C_RESET" >&2 + while IFS=$'\t' read -r name grp action tfv runner trig vars secrets; do + [ -n "$name" ] || continue + if [ "$tfv" = "preset" ]; then tfv="preset${SG_PRESET_TFV:+ ($SG_PRESET_TFV)}" + elif tf_version_above_ceiling "$tfv"; then tfv="${tfv#TERRAFORM-} -> ${SG_TF_FALLBACK_SHORT:-${SG_DEFAULT_TF_VERSION#TERRAFORM-}} (fallback)" + else tfv="${tfv#TERRAFORM-}"; fi + [ "$runner" = "preset" ] && runner="preset${SG_PRESET_RUNNER_SHORT:+ ($SG_PRESET_RUNNER_SHORT)}" + [ "$secrets" = "0" ] && secrets="-" + case "$action" in create) action="${C_GREEN}create ${C_RESET}" ;; update) action="${C_YELLOW}update ${C_RESET}" ;; esac + printf " %-${wn}s %-${gn}s %s %-28s %-${rw}s %-8s %-5s %s\n" "$name" "$grp" "$action" "$tfv" "$runner" "$trig" "$vars" "$secrets" >&2 + done <<<"$all_rows" echo >&2 - sg_dim "TERRAFORM '-> fallback' = pinned above SG's managed ceiling (1.5.7, last FOSS release); SECRETS = sensitive vars to recreate" + sg_dim "TERRAFORM '-> fallback' = pinned above SG's managed ceiling (1.5.7, last FOSS release); 'preset' = left to the org's execution preset at import; SECRETS = sensitive vars recreated as placeholder secrets" } diff --git a/scripts/lib/sg_api.sh b/scripts/lib/sg_api.sh index f165f6b..e5856b6 100644 --- a/scripts/lib/sg_api.sh +++ b/scripts/lib/sg_api.sh @@ -89,6 +89,65 @@ sg_list_workflows() { # sg_patch_workflow <group> <wf> <json> — PATCH a workflow. sg_patch_workflow() { sg_api_patch "$(wf_url "$1" "$2")" "$3"; } +# --- execution preset (org workflow defaults) ------------------------------ +# Settings -> Runner groups -> Execution presets is stored as the org's +# Settings.workflowDefaults: RunnerConstraints plus a TerraformConfig each for +# TERRAFORM and OPENTOFU workflows (terraformVersion, optional terraformBinPath / +# wfStepTemplateRevisionId). At workflow creation the API fills those keys from +# it when the payload does not carry them. + +# sg_execution_preset — the preset as compact JSON; "{}" when the org has none. +# Exit 1 when the org could not be read (the caller decides how loud to be). +sg_execution_preset() { + local body out + body="$(sg_api_get "$(sg_org_url)/" 2>/dev/null)" || { printf '{}'; return 1; } + out="$(printf '%s' "$body" | "$(sg_resolve jq sg_ensure_jq)" -c '(.msg // .data // {}) | (.Settings // {}).workflowDefaults // {}' 2>/dev/null)" + [ -n "$out" ] || out='{}' + printf '%s' "$out" +} + +# sg_preset_runner_desc <preset-json> — "shared runners" / "private runner group X". +sg_preset_runner_desc() { + local p="${1:-}" + [ -n "$p" ] || p='{}' + printf '%s' "$p" | "$(sg_resolve jq sg_ensure_jq)" -r ' + (.RunnerConstraints // {}) as $r + | if ($r.type // "shared") == "private" then "private runner group \(($r.names // []) | join(", "))" else "shared runners" end' 2>/dev/null +} + +# sg_preset_version_desc <preset-json> [TERRAFORM|OPENTOFU] — e.g. "Terraform 1.5.7", +# "Terraform 1.5.7 with runtime image /org/img:3", "runner-provided binary /usr/bin/terraform", +# or "managed Terraform 1.5.7 (platform default)" when the preset has no version. +sg_preset_version_desc() { + local p="${1:-}" + [ -n "$p" ] || p='{}' + printf '%s' "$p" | "$(sg_resolve jq sg_ensure_jq)" -r --arg t "${2:-TERRAFORM}" ' + (if $t == "OPENTOFU" then "OpenTofu" else "Terraform" end) as $tool + | ((if $t == "OPENTOFU" then .openTofuDefaults else .terraformDefaults end // {}).TerraformConfig // {}) as $c + | (if (($c.terraformBinPath // []) | length) > 0 then "runner-provided binary \($c.terraformBinPath[0].source // "")" + elif ($c.terraformVersion // "") == "" then (if $t == "OPENTOFU" then "managed OpenTofu (platform default)" else "managed Terraform 1.5.7 (platform default)" end) + else "\($tool) \($c.terraformVersion | ltrimstr("TERRAFORM-") | ltrimstr("OPENTOFU-"))" end) + + (if ($c.wfStepTemplateRevisionId // "") != "" then " with runtime image \($c.wfStepTemplateRevisionId)" else "" end)' 2>/dev/null +} + +# sg_preset_desc <preset-json> — one line: "Terraform 1.5.7 on shared runners"; +# "none configured (platform defaults: managed Terraform 1.5.7 on shared runners)" for {}. +sg_preset_desc() { + if [ -z "$1" ] || [ "$1" = "{}" ] || [ "$1" = "null" ]; then + printf 'none configured (platform defaults: managed Terraform 1.5.7 on shared runners)' + else + printf '%s on %s' "$(sg_preset_version_desc "$1")" "$(sg_preset_runner_desc "$1")" + fi +} + +# sg_preset_runner_provided <preset-json> — exit 0 when the Terraform defaults +# mount a runner-provided binary (terraformBinPath). +sg_preset_runner_provided() { + local p="${1:-}" + [ -n "$p" ] || p='{}' + printf '%s' "$p" | "$(sg_resolve jq sg_ensure_jq)" -e '((.terraformDefaults // {}).TerraformConfig.terraformBinPath // []) | length > 0' >/dev/null 2>&1 +} + # --- integrations (connectors) -------------------------------------------- # sg_list_integrations — [{name, type}, ...] for the org (fails on error). @@ -99,6 +158,25 @@ sg_list_integrations() { printf '%s' "$body" | "$(sg_resolve jq sg_ensure_jq)" -c '[(if (.msg | type) == "array" then .msg elif (.data | type) == "array" then .data elif type == "array" then . else [] end)[] | {name: (.ResourceName // .Id // ""), type: (.Settings.kind // .kind // .ResourceType // "")}] | map(select(.name != ""))' } +# sg_vcs_kind_of <connector-type> — the sourceConfigDestKind a VCS connector +# type implies (GITHUB_APP_CUSTOM -> GITHUB_COM, AZURE_DEVOPS_SP -> AZURE_DEVOPS, +# ...); empty for cloud connectors and unknown types. +sg_vcs_kind_of() { + case "$1" in + GITHUB_COM | GITHUB_APP_CUSTOM) printf 'GITHUB_COM' ;; + GITLAB_COM | GITLAB_OAUTH_SSH) printf 'GITLAB_COM' ;; + BITBUCKET_ORG) printf 'BITBUCKET_ORG' ;; + AZURE_DEVOPS | AZURE_DEVOPS_SP) printf 'AZURE_DEVOPS' ;; + GIT_OTHER) printf 'GIT_OTHER' ;; + *) printf '' ;; + esac +} + +# sg_integration_type <integrations-json> <name> — the connector's kind, or empty. +sg_integration_type() { + printf '%s' "$1" | "$(sg_resolve jq sg_ensure_jq)" -r --arg n "${2#/integrations/}" '[.[] | select(.name == $n) | .type][0] // empty' +} + # sg_integration_exists <name-or-/integrations/name> — exit 0 when it exists. sg_integration_exists() { local n="${1#/integrations/}" diff --git a/scripts/lib/tfc_api.sh b/scripts/lib/tfc_api.sh index 5a9c74e..ba0bc3c 100644 --- a/scripts/lib/tfc_api.sh +++ b/scripts/lib/tfc_api.sh @@ -78,10 +78,67 @@ tfc_list_orgs() { tfc_get_all "organizations" | "$(sg_resolve jq sg_ensure_jq)" # tfc_list_projects <org> — [{id, name}, ...] tfc_list_projects() { tfc_get_all "organizations/$1/projects" | "$(sg_resolve jq sg_ensure_jq)" -c '[.[] | {id: .id, name: .attributes.name}]'; } -# tfc_list_workspaces <org> — [{name, id, project, tags, terraform_version, execution_mode}, ...] +# tfc_list_workspaces <org> — [{name, id, project, tags, terraform_version, +# execution_mode, vcs_provider, vcs_url, vcs_identifier}, ...]. The vcs_* fields +# are empty for CLI-driven workspaces (no VCS connection). tfc_list_workspaces() { tfc_get_all "organizations/$1/workspaces" | "$(sg_resolve jq sg_ensure_jq)" -c \ - '[.[] | {name: .attributes.name, id: .id, project: (.relationships.project.data.id // ""), tags: (.attributes."tag-names" // []), terraform_version: .attributes."terraform-version", execution_mode: .attributes."execution-mode"}]' + '[.[] | {name: .attributes.name, id: .id, project: (.relationships.project.data.id // ""), tags: (.attributes."tag-names" // []), terraform_version: .attributes."terraform-version", execution_mode: .attributes."execution-mode", + vcs_provider: (.attributes."vcs-repo"."service-provider" // ""), vcs_url: (.attributes."vcs-repo"."repository-http-url" // ""), vcs_identifier: (.attributes."vcs-repo".identifier // "")}]' +} + +# tfc_select_workspaces <workspaces-json> <names-json> <tags-json> <ignore-json> +# — the subset the transformer exports, mirroring tfe_workspace_ids: name globs +# (["*"] = all), include tags (a workspace must carry all of them), exclude +# tags (any of them drops the workspace). null/[] disables a filter. +tfc_select_workspaces() { + printf '%s' "$1" | "$(sg_resolve jq sg_ensure_jq)" -c --argjson names "${2:-null}" --argjson tags "${3:-null}" --argjson ignore "${4:-null}" ' + def glob($p): ("^" + ($p | gsub("\\*"; ".*")) + "$"); + [ .[] + | select(($names == null) or ($names == ["*"]) or ([$names[] as $p | (.name | test(glob($p)))] | any)) + | select(($tags == null) or (($tags | length) == 0) or ([$tags[] as $t | ([.tags[]?] | index($t) != null)] | all)) + | select(($ignore == null) or (($ignore | length) == 0) or (([.tags[]?] | map(select(. as $t | $ignore | index($t) != null)) | length) == 0)) + ]' +} + +# tfc_vcs_kind_for <tfc-service-provider> — the SG sourceConfigDestKind that +# matches a TFC VCS provider (github, github_app, gitlab_hosted, ado_services, +# ...); empty when unknown. +tfc_vcs_kind_for() { + case "$1" in + github | github_app | github_enterprise) printf 'GITHUB_COM' ;; + gitlab_hosted | gitlab_community_edition | gitlab_enterprise_edition) printf 'GITLAB_COM' ;; + bitbucket_hosted | bitbucket_server | bitbucket_data_center) printf 'BITBUCKET_ORG' ;; + ado_services | ado_server) printf 'AZURE_DEVOPS' ;; + *) printf '' ;; + esac +} + +# tfc_vcs_label_for <tfc-service-provider> — human name (GitHub, GitLab, ...). +tfc_vcs_label_for() { + case "$1" in + github*) printf 'GitHub' ;; + gitlab*) printf 'GitLab' ;; + bitbucket*) printf 'Bitbucket' ;; + ado*) printf 'Azure DevOps' ;; + *) printf '%s' "$1" ;; + esac +} + +# tfc_vcs_summary <workspaces-json> — one line per provider in use: +# "<provider>\t<count>\t<repo-url-prefix or ->", most common first. The prefix +# is repository-http-url with the repo identifier stripped (https://github.com, +# https://gitlab.example.com, https://dev.azure.com, ...); "-" when the +# workspaces of that provider disagree. +tfc_vcs_summary() { + printf '%s' "$1" | "$(sg_resolve jq sg_ensure_jq)" -r ' + [ .[] | select(.vcs_provider != "") | . as $o + | { provider: .vcs_provider, + prefix: (if ($o.vcs_url | length) > 0 and ($o.vcs_identifier | length) > 0 and ($o.vcs_url | endswith("/" + $o.vcs_identifier)) + then ($o.vcs_url | rtrimstr("/" + $o.vcs_identifier)) + else (($o.vcs_url | capture("^(?<h>https?://[^/]+)").h) // "") end) } ] + | group_by(.provider) | sort_by(-length) + | .[] | "\(.[0].provider)\t\(length)\t\(([.[].prefix] | unique | map(select(. != ""))) as $p | if ($p | length) == 1 then $p[0] else "-" end)"' } # require_tfc_auth — fail fast when the tfe provider would not be able to diff --git a/scripts/lib/tfvars.sh b/scripts/lib/tfvars.sh index 6ec7412..a0f52ea 100644 --- a/scripts/lib/tfvars.sh +++ b/scripts/lib/tfvars.sh @@ -32,6 +32,18 @@ tfvars_get_json() { tfvars_json | "$(sg_resolve jq sg_ensure_jq)" -c "$1 // null" 2>/dev/null || echo null } +# tfvars_has <key> — exit 0 when the top-level key is present in the file (an +# explicit `key = null` counts as present; a missing key means the variable's +# default applies). +tfvars_has() { + tfvars_json | "$(sg_resolve jq sg_ensure_jq)" -e --arg k "$1" 'has($k)' >/dev/null 2>&1 +} + +# tfvars_is_null <key> — exit 0 when the file sets the key to null explicitly. +tfvars_is_null() { + tfvars_has "$1" && [ "$(tfvars_get_json ".$1")" = "null" ] +} + # tfvars_invalidate — forget the cached conversion (after writing the file). tfvars_invalidate() { _TFVARS_JSON=""; } @@ -46,24 +58,38 @@ tfvars_valid() { # _tfvars_hcl <json> — pretty-print a JSON value so it reads like HCL in the file. _tfvars_hcl() { printf '%s' "$1" | "$(sg_resolve jq sg_ensure_jq)" --indent 2 '.' 2>/dev/null || printf '%s' "$1"; } +# _tfvars_str <text> — a quoted HCL string literal (backslashes, quotes and +# template sequences escaped, so any connector or org name round-trips). +_tfvars_str() { + local s="$1" + s="${s//\\/\\\\}" + s="${s//\"/\\\"}" + s="${s//\$\{/\$\$\{}" + s="${s//%\{/%%\{}" + printf '"%s"' "$s" +} + # tfvars_write <dest> — render terraform.tfvars from W_* variables set by the # wizard (lists/objects are passed as compact JSON, which HCL accepts). Keeps # the same order and comments as terraform.tfvars.example so the file stays # hand-editable afterwards. # W_TFORG W_TFHOST W_WSNAMES_JSON W_TAGS_JSON W_IGNORE_TAGS_JSON W_EXPORT_STATE # W_APPROVERS_JSON W_REPO_PREFIX W_VCS_INTEGRATION W_DPC_JSON W_RUNNER_JSON -# W_DEST_KIND W_TF_VERSION W_TRIGGERS W_IGNORE_PATTERNS_JSON +# W_DEST_KIND W_TF_SOURCE W_TF_VERSION W_TRIGGERS W_IGNORE_PATTERNS_JSON +# W_RUNNER_JSON and W_TF_VERSION may be the literal "null" (defer to the org's +# execution preset). tfvars_write() { - local dest="$1" host_line="" + local dest="$1" host_line="" tf_version_hcl + if [ "${W_TF_VERSION:-null}" = "null" ]; then tf_version_hcl="null"; else tf_version_hcl="$(_tfvars_str "$W_TF_VERSION")"; fi if [ "${W_TFHOST:-app.terraform.io}" != "app.terraform.io" ]; then - host_line="$(printf '\n# Terraform Enterprise hostname (omit for Terraform Cloud)\ntfHostname = "%s"\n' "$W_TFHOST")" + host_line="$(printf '\n# Terraform Enterprise hostname (omit for Terraform Cloud)\ntfHostname = %s\n' "$(_tfvars_str "$W_TFHOST")")" fi cat >"$dest" <<TFVARS # Generated by 'sg-migrate.sh init' on $(date -u +%Y-%m-%dT%H:%M:%SZ). Safe to edit by hand; # re-run 'sg-migrate.sh init' to go through the wizard again. # Terraform Cloud/Enterprise organization name -tfOrg = "$W_TFORG" +tfOrg = $(_tfvars_str "$W_TFORG") $host_line # List of workspace names to export. Use wildcards (e.g., ["*"]) for all workspaces workspacenames = $W_WSNAMES_JSON @@ -89,25 +115,32 @@ ignoreVarPatterns = $W_IGNORE_PATTERNS_JSON SGDefaultWfApprovers = $W_APPROVERS_JSON # Prefix for your repo URL -SGDefaultIACVCSRepoPrefix = "$W_REPO_PREFIX" +SGDefaultIACVCSRepoPrefix = $(_tfvars_str "$W_REPO_PREFIX") # VCS connector used to clone the repositories (/integrations/<name>) -SGDefaultVCSAuthIntegrationID = "$W_VCS_INTEGRATION" +SGDefaultVCSAuthIntegrationID = $(_tfvars_str "$W_VCS_INTEGRATION") # Cloud connector the workflows deploy with SGDefaultDeploymentPlatformConfig = $(_tfvars_hcl "$W_DPC_JSON") -# Runners for every workflow: { type = "shared" } for SG-hosted runners, or -# { type = "private", names = ["<runner-group>"] } for a private runner group. -SGDefaultRunnerConstraints = $(_tfvars_hcl "$W_RUNNER_JSON") +# Runners for every workflow: { type = "shared" } for SG-hosted runners, +# { type = "private", names = ["<runner-group>"] } for a private runner group, or +# null to let the org's execution preset decide (Settings -> Runner groups). +SGDefaultRunnerConstraints = $(_tfvars_hcl "${W_RUNNER_JSON:-null}") # Choose from: GITHUB_COM, BITBUCKET_ORG, GITLAB_COM, AZURE_DEVOPS, GIT_OTHER -SGDefaultSourceConfigDestKind = "$W_DEST_KIND" - -# SG Terraform version used when a workspace's version is not a pinned semver, or -# when the SG API rejects a pinned version as above the managed ceiling (1.5.7, -# the last MPL/FOSS release; newer versions are BSL and not bundled). -SGDefaultTerraformVersion = "$W_TF_VERSION" +SGDefaultSourceConfigDestKind = $(_tfvars_str "$W_DEST_KIND") + +# Where the workflows get their Terraform version from: "carry" keeps each +# workspace's pinned TFC version (the fallback below covers the rest); "preset" +# sends no version, so the org's execution preset applies. +SGTerraformVersionSource = $(_tfvars_str "${W_TF_SOURCE:-carry}") + +# Fallback for "carry": SG Terraform version used when a workspace's version is +# not a pinned semver, or when the SG API rejects a pinned version as above the +# managed ceiling (1.5.7, the last MPL/FOSS release; newer versions are BSL and +# not bundled). null = leave those workflows to the org's execution preset. +SGDefaultTerraformVersion = $tf_version_hcl # Pre-configure VCS triggers on each workflow from the workspace's TFC settings SGDefaultEnableVCSTriggers = $W_TRIGGERS diff --git a/scripts/lib/wizard.sh b/scripts/lib/wizard.sh index 80bb2d8..2398678 100644 --- a/scripts/lib/wizard.sh +++ b/scripts/lib/wizard.sh @@ -1,9 +1,16 @@ #!/bin/bash # Interactive 'init' wizard (sourced; needs tools.sh, prompt.sh, tfvars.sh, -# tfc_api.sh, sg_api.sh). Discovers TFC orgs/workspaces and SG integrations / -# runner groups with the tokens already in the environment and writes -# terraform.tfvars. Every step degrades to free-text entry when a token is +# tfc_api.sh, sg_api.sh, state.sh). Discovers TFC orgs/workspaces and SG +# integrations / runner groups with the tokens already in the environment and +# writes terraform.tfvars. Every step degrades to free-text entry when a token is # missing or an API call fails, so the wizard always completes. +# +# What TFC knows about the workspaces steers the StackGuardian step: the VCS +# provider each workspace is connected to ranks the matching connectors first, +# and the repositories' URL gives the repo URL prefix (so a rerun that switches +# connector never keeps a prefix from the previous provider). +# +# shellcheck disable=SC2034 # the W_* results are consumed by tfvars_write (lib/tfvars.sh) # _w_default <jq-expr> <fallback> — current tfvars value (re-run) or fallback. _w_default() { local v; v="$(tfvars_get "$1")"; printf '%s' "${v:-$2}"; } @@ -17,7 +24,11 @@ _w_csv_json() { printf '%s' "${out:-[]}" } -# _w_repo_prefix_for <sourceConfigDestKind> — proposed repo URL prefix. +# _w_csv <json-array> — ["a","b"] -> "a, b" (for the review). +_w_csv() { printf '%s' "$1" | "$(sg_resolve jq sg_ensure_jq)" -r 'if type == "array" then join(", ") else tostring end' 2>/dev/null; } + +# _w_repo_prefix_for <sourceConfigDestKind> — the provider's well-known host, +# used only when TFC does not tell us where the repositories live. _w_repo_prefix_for() { case "$1" in GITHUB_COM) printf 'https://github.com' ;; @@ -28,9 +39,40 @@ _w_repo_prefix_for() { esac } +# _w_host <url> — "https://www.github.com/x" -> "www.github.com". +_w_host() { local h="${1#*://}"; printf '%s' "${h%%/*}"; } + +# _w_select_lines <question> <lines> [extra-items...] — sg_select (with an +# "Other" entry) over newline-separated "value|desc" items; names may contain +# spaces or glob characters, so no word splitting. +_w_select_lines() { + local q="$1" lines="$2" line + shift 2 + local -a items=() + while IFS= read -r line; do [ -n "$line" ] && items+=("$line"); done <<<"$lines" + SG_SELECT_OTHER=1 sg_select "$q" "${items[@]}" "$@" +} + +# _w_project_counts <workspaces-json> <projects-json> <groups:0|1> — +# "Default Project (2), Team A (1)" or, as SG groups, "tfc-default-project (2), ...". +_w_project_counts() { + printf '%s' "$1" | "$(sg_resolve jq sg_ensure_jq)" -r --argjson pr "$2" --argjson g "$3" ' + ($pr | map({key: .id, value: .name}) | from_entries) as $names + | group_by(.project) | sort_by(-length) + | map((($names[.[0].project] // .[0].project) as $n + | if $g == 1 then "tfc-" + ($n | ascii_downcase | gsub("[^a-z0-9-]+"; "-")) else $n end) + " (\(length))") + | join(", ")' 2>/dev/null +} + +# _w_tfc_kind_matches <sg-kind> — exit 0 when the TFC workspaces use that provider. +_w_tfc_kind_matches() { + [ -n "$1" ] || return 1 + case " ${W_TFC_VCS_KINDS:-} " in *" $1 "*) return 0 ;; *) return 1 ;; esac +} + # --- step 1: Terraform Cloud ------------------------------------------------ wizard_tfc() { - local host token orgs n ws projects wsn prn scope tags alltags + local host token orgs n ws projects wsn prn scope tags alltags sel prov cnt prefix kind local jqb jqb="$(sg_resolve jq sg_ensure_jq)" sg_step "1/4 Terraform Cloud / Enterprise" @@ -54,8 +96,7 @@ wizard_tfc() { W_TFORG="$(printf '%s' "$orgs" | "$jqb" -r '.[0]')" sg_log "organisation: $W_TFORG (the only one this token can see)" else - # shellcheck disable=SC2046 - W_TFORG="$(SG_SELECT_OTHER=1 sg_select "Which TFC organisation do you want to migrate?" $(printf '%s' "$orgs" | "$jqb" -r '.[]'))" || return 1 + W_TFORG="$(_w_select_lines "Which TFC organisation do you want to migrate?" "$(printf '%s' "$orgs" | "$jqb" -r '.[]')")" || return 1 fi else W_TFORG="$(sg_ask "TFC organisation name" "$(_w_default .tfOrg '')")" || return 1 @@ -65,13 +106,23 @@ wizard_tfc() { W_WSNAMES_JSON='["*"]' W_TAGS_JSON=null W_IGNORE_TAGS_JSON=null + W_TFC_WS_JSON="" + W_WS_COUNT="" + W_WS_TOTAL="" + W_WS_ABOVE_CEILING="" + W_GROUPS="" + W_TFC_VCS="" + W_TFC_VCS_KINDS="" + W_TFC_REPO_PREFIX="" + W_TFC_VCS_OTHER=0 + projects='[]' if [ "$W_TFC_DISCOVERY" -eq 1 ] && ws="$(tfc_list_workspaces "$W_TFORG" 2>/dev/null)"; then + W_TFC_WS_JSON="$ws" wsn="$(printf '%s' "$ws" | "$jqb" 'length')" projects="$(tfc_list_projects "$W_TFORG" 2>/dev/null || echo '[]')" prn="$(printf '%s' "$projects" | "$jqb" 'length')" sg_log "found $wsn workspace(s) in $prn project(s); each project becomes SG workflow group tfc-<project>" - W_WS_COUNT="$wsn" - W_WS_ABOVE_CEILING="$(printf '%s' "$ws" | "$jqb" '[.[] | select((.terraform_version // "") | test("^[0-9]+\\.[0-9]+\\.[0-9]+$")) | select(((.terraform_version | split(".") | map(tonumber)) as $v | ($v[0] > 1) or ($v[0] == 1 and $v[1] > 5) or ($v[0] == 1 and $v[1] == 5 and $v[2] > 7)))] | length')" + [ "$prn" -le 8 ] && [ "$wsn" -gt 0 ] && sg_dim "$(_w_project_counts "$ws" "$projects" 0)" alltags="$(printf '%s' "$ws" | "$jqb" -r '[.[].tags[]?] | unique | join(", ")')" else alltags="" @@ -81,6 +132,7 @@ wizard_tfc() { "tags|only workspaces carrying certain tags" \ "exclude|all workspaces except those carrying certain tags" \ "names|specific workspace names")" || return 1 + W_SCOPE="$scope" case "$scope" in tags) [ -n "$alltags" ] && sg_dim "tags in use: $alltags" @@ -97,11 +149,41 @@ wizard_tfc() { W_WSNAMES_JSON="$(_w_csv_json "$tags")" ;; esac + + # What the selection looks like (drives the review and the SG step's hints). + if [ -n "$W_TFC_WS_JSON" ]; then + sel="$(tfc_select_workspaces "$W_TFC_WS_JSON" "$W_WSNAMES_JSON" "$W_TAGS_JSON" "$W_IGNORE_TAGS_JSON")" + W_WS_TOTAL="$wsn" + W_WS_COUNT="$(printf '%s' "$sel" | "$jqb" 'length')" + W_WS_ABOVE_CEILING="$(printf '%s' "$sel" | "$jqb" '[.[] | select((.terraform_version // "") | test("^[0-9]+\\.[0-9]+\\.[0-9]+$")) | select(((.terraform_version | split(".") | map(tonumber)) as $v | ($v[0] > 1) or ($v[0] == 1 and $v[1] > 5) or ($v[0] == 1 and $v[1] == 5 and $v[2] > 7)))] | length')" + W_GROUPS="$(_w_project_counts "$sel" "$projects" 1)" + if [ "$scope" != "all" ]; then + if [ "$W_WS_COUNT" -eq 0 ]; then + sg_warn "no workspace matches that selection — 'apply' would export nothing" + else + sg_log "$W_WS_COUNT of $wsn workspace(s) match" + fi + fi + # VCS providers in use, most common first; the first one's repo URL prefix + # becomes the default for every workflow (others need workspaceOverrides). + W_TFC_VCS="$(tfc_vcs_summary "$sel")" + while IFS=$'\t' read -r prov cnt prefix; do + [ -n "$prov" ] || continue + kind="$(tfc_vcs_kind_for "$prov")" + [ -n "$kind" ] && W_TFC_VCS_KINDS="$W_TFC_VCS_KINDS $kind" + if [ -z "$W_TFC_REPO_PREFIX" ]; then + [ "$prefix" != "-" ] && W_TFC_REPO_PREFIX="$prefix" + else + W_TFC_VCS_OTHER=$((W_TFC_VCS_OTHER + cnt)) + fi + done <<<"$W_TFC_VCS" + fi + return 0 } # --- step 2: StackGuardian --------------------------------------------------- wizard_sg() { - local ints vcs cloud pick kind name runner groups jqb + local ints vcs cloud pick kind name runner groups jqb hint prov cnt prefix prev_kind prev_prefix line k jqb="$(sg_resolve jq sg_ensure_jq)" sg_step "2/4 StackGuardian" if [ -z "$ORG" ]; then @@ -122,37 +204,71 @@ wizard_sg() { W_SG_DISCOVERY=1 fi - # VCS connector -> integration id, source kind, repo prefix. + # VCS connector -> integration id, source kind, repo prefix. Connectors of the + # provider the TFC workspaces are connected to come first (and are the default). + if [ -n "${W_TFC_VCS:-}" ]; then + hint="" + while IFS=$'\t' read -r prov cnt prefix; do + [ -n "$prov" ] || continue + hint="$hint${hint:+, }$(tfc_vcs_label_for "$prov") ($cnt)" + done <<<"$W_TFC_VCS" + sg_dim "the selected TFC workspaces are connected to: $hint — matching connectors are listed first" + elif [ -n "${W_TFC_WS_JSON:-}" ] && [ "${W_WS_COUNT:-0}" -gt 0 ]; then + sg_dim "none of the selected TFC workspaces is VCS-connected (CLI-driven); pick the connector for the repositories anyway" + fi vcs="" - [ "$W_SG_DISCOVERY" -eq 1 ] && vcs="$(printf '%s' "$ints" | "$jqb" -r '[.[] | select((.type // "") | IN("GITHUB_COM","GITHUB_APP_CUSTOM","GITLAB_COM","GITLAB_OAUTH_SSH","BITBUCKET_ORG","AZURE_DEVOPS","AZURE_DEVOPS_SP","GIT_OTHER"))] | sort_by(.name) | .[] | "\(.name)|\(.type)"')" + if [ "$W_SG_DISCOVERY" -eq 1 ]; then + local -a first=() rest=() + while IFS='|' read -r name kind; do + [ -n "$name" ] || continue + if _w_tfc_kind_matches "$(sg_vcs_kind_of "$kind")"; then first+=("$name|$kind"); else rest+=("$name|$kind"); fi + done <<<"$(printf '%s' "$ints" | "$jqb" -r '[.[] | select((.type // "") | IN("GITHUB_COM","GITHUB_APP_CUSTOM","GITLAB_COM","GITLAB_OAUTH_SSH","BITBUCKET_ORG","AZURE_DEVOPS","AZURE_DEVOPS_SP","GIT_OTHER"))] | sort_by(.name) | .[] | "\(.name)|\(.type)"')" + vcs="$(printf '%s\n' ${first[@]+"${first[@]}"} ${rest[@]+"${rest[@]}"})" + fi if [ -n "$vcs" ]; then - # shellcheck disable=SC2046 - pick="$(SG_SELECT_OTHER=1 sg_select "Which VCS connector should clone the repositories?" $(printf '%s\n' "$vcs" | tr '\n' ' '))" || return 1 - kind="$(printf '%s' "$ints" | "$jqb" -r --arg n "$pick" '.[] | select(.name == $n) | .type' | head -1)" + pick="$(_w_select_lines "Which VCS connector should clone the repositories?" "$vcs")" || return 1 + kind="$(sg_integration_type "$ints" "$pick")" else pick="$(sg_ask "VCS connector name (as in StackGuardian, e.g. github_com)" "$(_w_default .SGDefaultVCSAuthIntegrationID '' | sed 's#^/integrations/##')")" || return 1 [ -n "$pick" ] || pick="$(sg_ask_required "VCS connector name")" || return 1 kind="" fi W_VCS_INTEGRATION="/integrations/${pick#/integrations/}" - case "$kind" in - GITHUB_APP_CUSTOM) W_DEST_KIND=GITHUB_COM ;; - GITLAB_OAUTH_SSH) W_DEST_KIND=GITLAB_COM ;; - AZURE_DEVOPS_SP) W_DEST_KIND=AZURE_DEVOPS ;; - GITHUB_COM | GITLAB_COM | BITBUCKET_ORG | AZURE_DEVOPS | GIT_OTHER) W_DEST_KIND="$kind" ;; - *) W_DEST_KIND="$(sg_select "VCS provider kind" GITHUB_COM GITLAB_COM BITBUCKET_ORG AZURE_DEVOPS GIT_OTHER)" || return 1 ;; - esac - # Repo URL prefix follows the connector kind; editable in terraform.tfvars. - W_REPO_PREFIX="$(_w_default .SGDefaultIACVCSRepoPrefix "$(_w_repo_prefix_for "$W_DEST_KIND")")" + W_DEST_KIND="$(sg_vcs_kind_of "$kind")" + if [ -z "$W_DEST_KIND" ]; then + # Unknown connector type (or no discovery): ask, defaulting to what TFC uses. + local -a kinds=(GITHUB_COM GITLAB_COM BITBUCKET_ORG AZURE_DEVOPS GIT_OTHER) + for k in ${W_TFC_VCS_KINDS:-}; do kinds=("$k" "${kinds[@]}"); break; done + W_DEST_KIND="$(sg_select "VCS provider kind" "${kinds[@]}")" || return 1 + fi + if [ -n "${W_TFC_VCS_KINDS:-}" ] && ! _w_tfc_kind_matches "$W_DEST_KIND"; then + sg_warn "the TFC workspaces are connected to $(tfc_vcs_label_for "${W_TFC_VCS%% *}") but '$pick' is a $W_DEST_KIND connector — the workflows will not be able to clone unless the repositories moved" + fi + # Repo URL prefix: where TFC says the repositories live; otherwise the previous + # value (only if the provider kind did not change — this is what used to leave + # a stale prefix behind); otherwise the provider's well-known host. + prev_kind="$(tfvars_get .SGDefaultSourceConfigDestKind)" + prev_prefix="$(tfvars_get .SGDefaultIACVCSRepoPrefix)" + [ "$prev_kind" = "$W_DEST_KIND" ] || prev_prefix="" + if [ -n "${W_TFC_REPO_PREFIX:-}" ]; then + # Keep a hand-tuned variant of the same host (e.g. www.github.com). + case "$(_w_host "$prev_prefix")" in + *"$(_w_host "$W_TFC_REPO_PREFIX")") [ -n "$prev_prefix" ] && W_REPO_PREFIX="$prev_prefix" || W_REPO_PREFIX="$W_TFC_REPO_PREFIX" ;; + *) W_REPO_PREFIX="$W_TFC_REPO_PREFIX" ;; + esac + elif [ -n "$prev_prefix" ]; then + W_REPO_PREFIX="$prev_prefix" + else + W_REPO_PREFIX="$(_w_repo_prefix_for "$W_DEST_KIND")" + fi # Cloud connector -> DeploymentPlatformConfig. cloud="" # Only kinds DeploymentPlatformConfig accepts (AZURE_DEVOPS* are VCS connectors). - [ "$W_SG_DISCOVERY" -eq 1 ] && cloud="$(printf '%s' "$ints" | "$jqb" -r '[.[] | select((.type // "") | IN("AWS_STATIC","AWS_RBAC","AWS_OIDC","AZURE_STATIC","AZURE_OIDC","AZURE_MANAGED_ID_OIDC","GCP_STATIC","GCP_OIDC"))] | sort_by(.name) | .[] | "\(.name)|\(.type)"')" + [ "$W_SG_DISCOVERY" -eq 1 ] && cloud="$(printf '%s' "$ints" | "$jqb" -r '[.[] | select((.type // "") | IN("AWS_STATIC","AWS_RBAC","AWS_OIDC","AZURE_STATIC","AZURE_OIDC","AZURE_MANAGED_ID_OIDC","GCP_STATIC","GCP_OIDC"))] | sort_by(.type, .name) | .[] | "\(.name)|\(.type)"')" if [ -n "$cloud" ]; then - # shellcheck disable=SC2046 - pick="$(SG_SELECT_OTHER=1 sg_select "Which cloud connector should the workflows deploy with?" $(printf '%s\n' "$cloud" | tr '\n' ' ') "skip|decide later (leaves a placeholder to edit)")" || return 1 - kind="$(printf '%s' "$ints" | "$jqb" -r --arg n "$pick" '.[] | select(.name == $n) | .type' | head -1)" + pick="$(_w_select_lines "Which cloud connector should the workflows deploy with?" "$cloud" "skip|decide later (leaves a placeholder to edit)")" || return 1 + kind="$(sg_integration_type "$ints" "$pick")" else pick="$(sg_ask "Cloud connector name (as in StackGuardian; empty to decide later)" "$(_w_default .SGDefaultDeploymentPlatformConfig[0].config.integrationId '' | sed 's#^/integrations/##')")" || return 1 kind="" @@ -160,22 +276,53 @@ wizard_sg() { if [ -z "$pick" ] || [ "$pick" = "skip" ]; then W_DPC_JSON='[{"kind":"AWS_RBAC","config":{"integrationId":"/integrations/CHANGE_ME"}}]' W_DPC_PLACEHOLDER=1 + W_CLOUD_DESC="none yet — a placeholder is written; edit it before 'apply'" else [ -n "$kind" ] || kind="$(sg_select "Connector kind" AWS_RBAC AWS_STATIC AWS_OIDC AZURE_STATIC AZURE_OIDC AZURE_MANAGED_ID_OIDC GCP_STATIC GCP_OIDC)" || return 1 W_DPC_JSON="$("$jqb" -nc --arg k "$kind" --arg i "/integrations/${pick#/integrations/}" '[{kind:$k, config:{integrationId:$i}}]')" W_DPC_PLACEHOLDER=0 + W_CLOUD_DESC="${pick#/integrations/} ($kind)" + fi + + # The org's execution preset (Settings -> Runner groups -> Execution presets): + # what StackGuardian fills in for runners / Terraform version when the payload + # carries none. Read once here; steps 2 and 3 offer it as a choice. + W_PRESET_JSON='{}' + W_PRESET_READ=0 + W_PRESET_RUNNER="" + W_PRESET_TFVER="" + if [ "$W_SG_DISCOVERY" -eq 1 ]; then + if W_PRESET_JSON="$(sg_execution_preset)"; then + W_PRESET_READ=1 + W_PRESET_RUNNER="$(sg_preset_runner_desc "$W_PRESET_JSON")" + W_PRESET_TFVER="$(sg_preset_version_desc "$W_PRESET_JSON")" + if [ "$W_PRESET_JSON" = "{}" ]; then + sg_dim "execution preset: none configured in $ORG — StackGuardian's platform defaults apply ($W_PRESET_TFVER on $W_PRESET_RUNNER)" + else + sg_dim "execution preset of $ORG: $W_PRESET_TFVER on $W_PRESET_RUNNER" + fi + fi fi - # Runner constraints. - runner="$(sg_select "Where should the workflows run?" \ - "shared|StackGuardian-hosted shared runners" \ - "private|a private runner group in your own network")" || return 1 - if [ "$runner" = "private" ]; then + # Runner constraints: the org's execution preset (first when we could read it + # and nothing explicit was configured before), SG shared runners, or a private + # runner group. + local -a runner_items=("shared|StackGuardian-hosted shared runners" "private|a private runner group in your own network") + local preset_item="preset|whatever the org's execution preset says${W_PRESET_RUNNER:+ (now: $W_PRESET_RUNNER)}" + if tfvars_is_null SGDefaultRunnerConstraints || { [ "$W_PRESET_READ" -eq 1 ] && ! tfvars_has SGDefaultRunnerConstraints; }; then + runner_items=("$preset_item" "${runner_items[@]}") + else + runner_items+=("$preset_item") + fi + runner="$(sg_select "Where should the workflows run?" "${runner_items[@]}")" || return 1 + if [ "$runner" = "preset" ]; then + W_RUNNER_JSON="null" + W_RUNNER_DESC="from the org's execution preset${W_PRESET_RUNNER:+ (now: $W_PRESET_RUNNER)}" + elif [ "$runner" = "private" ]; then groups="" [ "$W_SG_DISCOVERY" -eq 1 ] && groups="$(sg_list_runnergroups 2>/dev/null | "$jqb" -r 'sort | .[]' 2>/dev/null || true)" if [ -n "$groups" ]; then - # shellcheck disable=SC2046 - name="$(SG_SELECT_OTHER=1 sg_select "Which runner group?" $(printf '%s\n' "$groups" | tr '\n' ' '))" || return 1 + name="$(_w_select_lines "Which runner group?" "$groups")" || return 1 else name="$(sg_ask_required "Runner group name")" || return 1 fi @@ -183,8 +330,10 @@ wizard_sg() { sg_warn "runner group '$name' was not found in org '$ORG' — preflight will fail until it exists" fi W_RUNNER_JSON="$("$jqb" -nc --arg n "$name" '{type:"private", names:[$n]}')" + W_RUNNER_DESC="private runner group '$name'" else W_RUNNER_JSON='{"type":"shared"}' + W_RUNNER_DESC="StackGuardian shared runners" fi } @@ -195,7 +344,32 @@ wizard_policy() { # with sensible defaults — edit them in terraform.tfvars if needed. W_APPROVERS_JSON="$(tfvars_get_json .SGDefaultWfApprovers)" [ "$W_APPROVERS_JSON" = "null" ] && W_APPROVERS_JSON='[]' - W_TF_VERSION="$(_w_default .SGDefaultTerraformVersion TERRAFORM-1.5.7)" + + # Terraform version: carry the TFC pins (plus a fallback for the rest) or send + # none and let the org's execution preset decide. Previous choices come first. + local src fb prev_src prev_fb fixed above_hint="" + local -a src_items fb_items + [ "${W_WS_ABOVE_CEILING:-0}" -gt 0 ] && above_hint=" ($W_WS_ABOVE_CEILING pinned above SG's 1.5.7 ceiling would use the fallback)" + src_items=("carry|keep each workspace's pinned TFC version$above_hint" + "preset|no version at all: the org's execution preset applies${W_PRESET_TFVER:+ (now: $W_PRESET_TFVER)}") + prev_src="$(_w_default .SGTerraformVersionSource carry)" + [ "$prev_src" = "preset" ] && src_items=("${src_items[1]}" "${src_items[0]}") + src="$(sg_select "Which Terraform version should the migrated workflows run?" "${src_items[@]}")" || return 1 + W_TF_SOURCE="$src" + if [ "$src" = "preset" ]; then + W_TF_VERSION="null" + else + if tfvars_is_null SGDefaultTerraformVersion; then prev_fb="null"; else prev_fb="$(_w_default .SGDefaultTerraformVersion TERRAFORM-1.5.7)"; fi + [ "$prev_fb" = "null" ] && fixed="TERRAFORM-1.5.7" || fixed="TERRAFORM-${prev_fb#TERRAFORM-}" + fb_items=("${fixed#TERRAFORM-}|a fixed version ($fixed; StackGuardian bundles Terraform up to 1.5.7)" + "preset|the org's execution preset${W_PRESET_TFVER:+ (now: $W_PRESET_TFVER)}") + [ "$prev_fb" = "null" ] && fb_items=("${fb_items[1]}" "${fb_items[0]}") + fb="$(sg_select "Fallback for workspaces without a pinned version, and for pins StackGuardian rejects (above 1.5.7)?" "${fb_items[@]}")" || return 1 + case "$fb" in + preset) W_TF_VERSION="null" ;; + *) W_TF_VERSION="TERRAFORM-${fb#TERRAFORM-}" ;; + esac + fi if sg_confirm "Export Terraform state for each workspace?" "$([ "$(_w_default .exportStateFiles true)" = "false" ] && echo N || echo Y)"; then W_EXPORT_STATE=true; else W_EXPORT_STATE=false; fi if sg_confirm "Pre-configure VCS triggers (push / pull-request runs) from the TFC settings?" "$([ "$(_w_default .SGDefaultEnableVCSTriggers true)" = "false" ] && echo N || echo Y)"; then W_TRIGGERS=true; else W_TRIGGERS=false; fi sg_dim "TFC_* / TFE_* variables (e.g. TFC_WORKSPACE_NAME, TFC_AWS_RUN_ROLE_ARN) only mean something inside Terraform Cloud." @@ -208,29 +382,55 @@ wizard_policy() { # --- step 4: review + write ---------------------------------------------------- wizard_review() { + local scope tf yn_state yn_trig strip sg_step "4/4 Review" - row() { printf ' %s%-28s%s %s\n' "$C_BOLD" "$1" "$C_RESET" "$2" >&2; } - row "TFC host / org" "$W_TFHOST / $W_TFORG" - row "Workspaces" "$W_WSNAMES_JSON tags=$W_TAGS_JSON ignore=$W_IGNORE_TAGS_JSON${W_WS_COUNT:+ ($W_WS_COUNT found)}" - row "SG org" "$ORG" - row "VCS connector" "$W_VCS_INTEGRATION ($W_DEST_KIND, $W_REPO_PREFIX)" - row "Cloud connector" "$W_DPC_JSON" - row "Runners" "$W_RUNNER_JSON" - row "State export / triggers" "$W_EXPORT_STATE / $W_TRIGGERS" - row "Strip variables matching" "$W_IGNORE_PATTERNS_JSON" - row "Fallback Terraform" "$W_TF_VERSION${W_WS_ABOVE_CEILING:+ ($W_WS_ABOVE_CEILING workspace(s) pinned above 1.5.7 (the last FOSS runtime SG bundles) will use it)}" + sg_row "TFC" "$W_TFHOST / $W_TFORG" + case "${W_SCOPE:-all}" in + tags) scope="workspaces tagged $(_w_csv "$W_TAGS_JSON")" ;; + exclude) scope="all workspaces except those tagged $(_w_csv "$W_IGNORE_TAGS_JSON")" ;; + names) scope="workspaces named $(_w_csv "$W_WSNAMES_JSON")" ;; + *) scope="all workspaces" ;; + esac + if [ -n "${W_WS_COUNT:-}" ]; then + if [ "${W_SCOPE:-all}" = "all" ]; then scope="$scope ($W_WS_COUNT)"; else scope="$scope — $W_WS_COUNT of $W_WS_TOTAL match"; fi + fi + sg_row "Workspaces" "$scope" + [ -n "${W_GROUPS:-}" ] && sg_row "Workflow groups" "$W_GROUPS" + sg_row "StackGuardian org" "$ORG" + sg_row "VCS connector" "${W_VCS_INTEGRATION#/integrations/} ($W_DEST_KIND) — repositories under $W_REPO_PREFIX" + sg_row "Cloud connector" "$W_CLOUD_DESC" + sg_row "Runners" "$W_RUNNER_DESC" + [ "$W_EXPORT_STATE" = "true" ] && yn_state=yes || yn_state=no + [ "$W_TRIGGERS" = "true" ] && yn_trig="yes, from each workspace's TFC settings" || yn_trig=no + sg_row "State export" "$yn_state" + sg_row "VCS triggers" "$yn_trig" + [ "$W_IGNORE_PATTERNS_JSON" = "[]" ] && strip="none" || strip="TFC_*, TFE_*" + sg_row "Strip variables" "$strip" + if [ "${W_TF_SOURCE:-carry}" = "preset" ]; then + tf="from the org's execution preset${W_PRESET_TFVER:+ (now: $W_PRESET_TFVER)}" + else + if [ "${W_TF_VERSION:-null}" = "null" ]; then + tf="carried from TFC; unpinned workspaces and pins above 1.5.7 go to the execution preset${W_PRESET_TFVER:+ (now: $W_PRESET_TFVER)}" + else + tf="carried from TFC; fallback ${W_TF_VERSION#TERRAFORM-} for unpinned workspaces and pins above 1.5.7" + fi + [ "${W_WS_ABOVE_CEILING:-0}" -gt 0 ] && tf="$tf — $W_WS_ABOVE_CEILING workspace(s) affected" + fi + sg_row "Terraform version" "$tf" [ "${W_DPC_PLACEHOLDER:-0}" -eq 1 ] && sg_warn "cloud connector left as a placeholder — edit SGDefaultDeploymentPlatformConfig in $(sg_rel "$TFVARS") before 'apply'" - sg_dim "approvers, repo URL prefix and the fallback version can be edited in $(sg_rel "$TFVARS")" + [ "${W_TFC_VCS_OTHER:-0}" -gt 0 ] && sg_warn "$W_TFC_VCS_OTHER workspace(s) use a different VCS provider than the default above — give them their own connector/prefix via workspaceOverrides in $(sg_rel "$TFVARS")" + sg_dim "approvers and the repo URL prefix can be edited in $(sg_rel "$TFVARS")" sg_confirm "Write $(sg_rel "$TFVARS")?" Y } # wizard_run — the whole flow; returns non-zero when aborted. wizard_run() { + local kept="" wizard_tfc && wizard_sg && wizard_policy || { sg_err "init aborted"; return 1; } wizard_review || { sg_log "nothing written"; return 1; } if [ -f "$TFVARS" ]; then cp "$TFVARS" "$TFVARS.bak" - sg_log "previous file kept as $(sg_rel "$TFVARS.bak")" + kept=" (previous version kept as $(basename "$TFVARS").bak)" fi tfvars_write "$TFVARS" if ! tfvars_valid; then @@ -240,5 +440,5 @@ wizard_run() { # Remember the SG org (and API host) for later phases, so users don't have to # export SG_ORG again in a new shell. Tokens are never stored. state_update '.config = ((.config // {}) + {sg_org: $o, sg_base_url: $u})' --arg o "$ORG" --arg u "$SG_BASE_URL" - sg_success "wrote $(sg_rel "$TFVARS")" + sg_success "wrote $(sg_rel "$TFVARS")$kept" } diff --git a/scripts/migrate.sh b/scripts/migrate.sh index 6fad061..b9e8c9b 100755 --- a/scripts/migrate.sh +++ b/scripts/migrate.sh @@ -61,6 +61,12 @@ TF_PARALLELISM="${SG_TF_PARALLELISM:-20}" RETRIES="${SG_RETRIES:-4}" RETRY_BASE="${SG_RETRY_BASE:-2}" PF=() +# Phase bookkeeping: 'all' numbers its phases ("Phase 2/5: ...") and every +# phase reports how long it took. +PHASE_TOTAL=0 +PHASE_N=0 +PHASE_T0=$SECONDS +RUN_T0=$SECONDS # The init wizard remembers the SG org / API host in .sg/state.json so a new # shell without SG_ORG still works; flags and env always win. if [ -z "$ORG" ] && [ -f "$STATE_FILE" ]; then @@ -130,31 +136,73 @@ die() { exit 1 } -throttle() { while [ "$(jobs -rp | wc -l | tr -d ' ')" -ge "$1" ]; do sleep 0.2; done; } +# phase_begin <title> — "==> Phase 2/5: title" inside 'all', "==> Phase: title" +# for a single command; starts the phase timer. +phase_begin() { + PHASE_T0=$SECONDS + if [ "$PHASE_TOTAL" -gt 0 ]; then + PHASE_N=$((PHASE_N + 1)) + sg_step "Phase $PHASE_N/$PHASE_TOTAL: $1" + else + sg_step "Phase: $1" + fi +} +# phase_took — "12s" / "1m 04s" since phase_begin. +phase_took() { sg_fmt_secs $((SECONDS - PHASE_T0)); } + +# Ctrl-C: end the live progress line cleanly and say what happens next, instead +# of a bare "^C" (a background terraform/sg-cli gets the same SIGINT from the +# terminal, so nothing keeps running). +trap 'sg_spin_clear; printf "\n" >&2; sg_warn "interrupted — re-run the same command to pick up where this left off"; exit 130' INT -# run_parallel <fn> <max> <items...> — runs fn over items, up to max at a time. -# Each job's stdout+stderr is buffered to its own file (so concurrent tools see -# a non-TTY and don't scatter spinner output across the terminal), then flushed -# as a clean labeled block in submission order. Returns non-zero if any failed. +# rp_progress <statusdir> <label> <total> <t0> — one frame of the live +# "<label> — done/total (elapsed)" line while run_parallel waits. +rp_progress() { + local finished=0 f + for f in "$1"/*.rc; do [ -e "$f" ] && finished=$((finished + 1)); done + sg_spin_frame "$2 ${C_DIM}— $finished/$3 done ($(sg_fmt_secs $((SECONDS - $4))))${C_RESET}" +} + +# run_parallel <fn> <max> <label> <items...> — runs fn over items, up to max at +# a time, with a live progress line meanwhile. Each job's stdout+stderr is +# buffered to its own file (so concurrent tools see a non-TTY and don't scatter +# spinner output across the terminal), then flushed in submission order: a job +# that printed a single line is shown as-is, longer or failed output gets a +# "── <item> ──" header (always with -v). Returns non-zero if any job failed. run_parallel() { - local fn="$1" max="$2" - shift 2 - local statusdir i=0 rc=0 item + local fn="$1" max="$2" label="$3" + shift 3 + local statusdir i=0 rc=0 item total=$# t0=$SECONDS statusdir="$(mktemp -d)" + [ "$SG_ANIMATE" = "1" ] || sg_log "$label, up to $max in parallel..." for item in "$@"; do - throttle "$max" + while [ "$(jobs -rp | wc -l | tr -d ' ')" -ge "$max" ]; do + rp_progress "$statusdir" "$label" "$total" "$t0" + sleep 0.2 + done ( "$fn" "$item" >"$statusdir/$i.out" 2>&1 echo "$?" >"$statusdir/$i.rc" ) & i=$((i + 1)) done + while [ "$(jobs -rp | wc -l | tr -d ' ')" -gt 0 ]; do + rp_progress "$statusdir" "$label" "$total" "$t0" + sleep 0.2 + done wait + sg_spin_clear i=0 for item in "$@"; do - printf '%s── %s ──%s\n' "$C_CYAN" "$(basename "$item")" "$C_RESET" >&2 + if [ "$(cat "$statusdir/$i.rc" 2>/dev/null)" = "0" ]; then + if [ "$VERBOSE" -eq 1 ] || [ "$(wc -l <"$statusdir/$i.out" | tr -d ' ')" -gt 1 ]; then + printf '%s── %s ──%s\n' "$C_CYAN" "$(basename "$item")" "$C_RESET" >&2 + fi + else + printf '%s── %s ──%s\n' "$C_CYAN" "$(basename "$item")" "$C_RESET" >&2 + rc=1 + fi [ -s "$statusdir/$i.out" ] && cat "$statusdir/$i.out" >&2 - [ "$(cat "$statusdir/$i.rc" 2>/dev/null)" = "0" ] || rc=1 i=$((i + 1)) done rm -rf "$statusdir" @@ -169,17 +217,27 @@ payload_sha() { # run_phase <name> <input-sha> <fn> — in 'all', skip a phase that already ran # with identical inputs; otherwise run it and record the inputs it ran with. -# Phases that rewrite the payloads (enrich, convert) record the post-run hash, -# so an unchanged export is recognised on the next run. +# apply and enrich are keyed by the apply inputs (tfvars + workspace filter): +# enrich only depends on what apply produced, and the later convert phase +# rewrites the payloads, so a payload hash could never match again. convert and +# validate record the post-run payload hash, so an unchanged export is +# recognised on the next run. run_phase() { local name="$1" sha="$2" fn="$3" at if at="$(state_phase_done "$name" "$sha")" && [ -n "$at" ]; then - sg_log "skipping $name — unchanged since $at (--fresh to redo)" + at="${at%:*}" + at="${at/T/ } UTC" + if [ "$PHASE_TOTAL" -gt 0 ]; then + PHASE_N=$((PHASE_N + 1)) + sg_log "skipping phase $PHASE_N/$PHASE_TOTAL ($name) — inputs unchanged since $at (--fresh to redo)" + else + sg_log "skipping $name — inputs unchanged since $at (--fresh to redo)" + fi return 0 fi "$fn" || return $? case "$name" in - apply) state_mark_phase "$name" "$sha" ;; + apply | enrich) state_mark_phase "$name" "$sha" ;; *) state_mark_phase "$name" "$(payload_sha)" ;; esac } @@ -240,18 +298,21 @@ cmd_init() { wizard_run || return 1 fi fi - sg_log "workflow groups are created automatically as tfc-<project>; no mapping needed" - sg_log "next: ./sg-migrate.sh all" - completion_hint - sg_success "init complete" + # A standalone 'init' ends with what to do next; 'all' just carries on. + [ "${1:-}" = "standalone" ] && show_next_steps + return 0 } -# completion_hint — tell the user how to enable tab completion for their shell. -# A child process cannot register completions in the parent shell, so this only -# prints the one-liner (nothing is written to the user's rc files). -completion_hint() { - sg_log "tab completion for this shell session: source <(./sg-migrate.sh completion)" +# show_next_steps — the commands that make sense after init. (A child process +# cannot register completions in the parent shell, so the completion line is a +# hint only; nothing is written to the user's rc files.) +show_next_steps() { + sg_step "Next steps" + next_row "$PROG all" "run the migration — the import plan is shown and confirmed before anything is created" + next_row "$PROG apply" "export only: review the payloads in $(sg_rel "$EXPORT_DIR")/ first, then '$PROG import'" + next_row "source <($PROG completion)" "tab completion for this shell session" } +next_row() { printf ' %s%-38s%s %s%s%s\n' "$C_BOLD" "$1" "$C_RESET" "$C_DIM" "$2" "$C_RESET" >&2; } # current_shell — bash|zsh: the shell the user is typing in (detected by # sg-migrate.sh from its parent process), else the login shell. @@ -326,9 +387,8 @@ set_triggers_pass() { sg_log "no workflows carry VCS triggers — nothing to register" return 0 fi - sg_log "registering VCS triggers for $total workflow(s), up to $CONC in parallel (retries: $RETRIES)" local rc=0 f seg - run_parallel do_set_triggers "$CONC" "${PF[@]}" || rc=1 + run_parallel do_set_triggers "$CONC" "registering VCS triggers for $total workflow(s)" "${PF[@]}" || rc=1 for f in "${PF[@]}"; do seg="$(seg_of "$f")" if [ -f "$EXPORT_DIR/.triggers-result.$seg.json" ]; then @@ -337,7 +397,7 @@ set_triggers_pass() { fi done if [ "$rc" -eq 0 ]; then - sg_success "vcs triggers registered" + sg_success "VCS triggers registered for $total workflow(s)" else sg_err "one or more VCS trigger registrations failed (re-run: $PROG triggers)" return 1 @@ -345,7 +405,7 @@ set_triggers_pass() { } cmd_triggers() { - sg_step "Phase: vcs triggers" + phase_begin "VCS triggers" [ -n "${SG_API_TOKEN:-}" ] || die "SG_API_TOKEN is not set." [ -n "$ORG" ] || die "StackGuardian org not set (use --org or SG_ORG)." command -v curl >/dev/null 2>&1 || die "curl is required for VCS trigger registration." @@ -357,7 +417,7 @@ cmd_triggers() { } cmd_clean() { - sg_step "Phase: clean" + phase_begin "clean" sg_log "removing local working artifacts..." rm -rf "$EXPORT_DIR" rm -rf "$TRANSFORMER_DIR/.terraform" "$TRANSFORMER_DIR/.terraform.lock.hcl" \ @@ -372,17 +432,24 @@ cmd_clean() { sg_success "clean complete" } +# tf_init / tf_apply — terraform in the transformer dir, with jq on PATH for the +# state export's local-exec. Always run in a subshell (sg_run_quiet backgrounds +# them; the verbose path wraps them in parentheses). +# shellcheck disable=SC2120 # extra terraform flags (-no-color) come from the quiet path +tf_init() { cd "$TRANSFORMER_DIR" && export PATH="$TF_PATH" TF_IN_AUTOMATION=1 && terraform init -input=false "$@"; } +tf_apply() { cd "$TRANSFORMER_DIR" && export PATH="$TF_PATH" TF_IN_AUTOMATION=1 && terraform apply -auto-approve -compact-warnings -parallelism="$TF_PARALLELISM" -var-file=terraform.tfvars "$@"; } + cmd_apply() { - sg_step "Phase: apply (terraform)" + phase_begin "apply (terraform)" command -v terraform >/dev/null 2>&1 || die "terraform not found on PATH" [ -f "$TFVARS" ] || die "Missing $(sg_rel "$TFVARS"). Run: $PROG init" preflight_run apply # State export (TFC API) calls curl + jq from terraform's local-exec; make sure # both are on PATH for the apply (jq from cache if not already installed). command -v curl >/dev/null 2>&1 || die "curl is required for state export" - local jqdir tflog rc=0 + local tflog rc=0 local -a tfvar_args=() - jqdir="$(dirname "$(sg_resolve jq sg_ensure_jq)")" + TF_PATH="$(dirname "$(sg_resolve jq sg_ensure_jq)"):$PATH" if [ "${#WS_FILTER[@]}" -gt 0 ]; then JQ_BIN="${JQ_BIN:-$(sg_resolve jq sg_ensure_jq)}" tfvar_args=(-var "workspacenames=$(ws_filter_json)") @@ -390,36 +457,32 @@ cmd_apply() { fi if [ "$VERBOSE" -eq 1 ]; then - (cd "$TRANSFORMER_DIR" && export PATH="$jqdir:$PATH" TF_IN_AUTOMATION=1 && - terraform init -input=false && - terraform apply -auto-approve -compact-warnings -parallelism="$TF_PARALLELISM" -var-file=terraform.tfvars "${tfvar_args[@]}") || rc=$? + # shellcheck disable=SC2119 + (tf_init && tf_apply ${tfvar_args[@]+"${tfvar_args[@]}"}) || rc=$? else - # Quiet: capture terraform's verbose plan/output; surface only progress, the - # final summary, and (on failure) the captured log. + # Quiet: terraform's init/plan output goes to a log that is shown only on + # failure; the terminal gets a live progress line per step instead. tflog="$(mktemp)" - sg_log "initializing terraform (providers)..." - (cd "$TRANSFORMER_DIR" && export PATH="$jqdir:$PATH" TF_IN_AUTOMATION=1 && terraform init -input=false -no-color) >"$tflog" 2>&1 || rc=$? + sg_run_quiet "initializing terraform providers" "terraform providers ready" "$tflog" tf_init -no-color || rc=$? if [ "$rc" -eq 0 ]; then - sg_log "reading workspaces, generating payloads, exporting state..." - (cd "$TRANSFORMER_DIR" && export PATH="$jqdir:$PATH" TF_IN_AUTOMATION=1 && - terraform apply -auto-approve -compact-warnings -no-color -parallelism="$TF_PARALLELISM" -var-file=terraform.tfvars "${tfvar_args[@]}") >"$tflog" 2>&1 || rc=$? + sg_run_quiet "reading workspaces, generating payloads, exporting state" "workspaces read, payloads generated, state exported" "$tflog" \ + tf_apply -no-color ${tfvar_args[@]+"${tfvar_args[@]}"} || rc=$? fi if [ "$rc" -ne 0 ]; then sg_err "terraform failed (rc=$rc):" cat "$tflog" >&2 - else - grep -E '^(Apply complete|No changes)' "$tflog" | sed 's/^/ /' >&2 || true fi rm -f "$tflog" fi [ "$rc" -eq 0 ] || return "$rc" - sg_success "apply complete — payloads in $(sg_rel "$EXPORT_DIR")" + payload_files + sg_success "apply complete in $(phase_took) — ${#PF[@]} payload file(s) in $(sg_rel "$EXPORT_DIR")/" show_migration_summary } cmd_enrich() { - sg_step "Phase: variable sets" + phase_begin "variable sets" [ -f "$TFVARS" ] || die "Missing $(sg_rel "$TFVARS") (run: $PROG init)." payload_files [ "${#PF[@]}" -gt 0 ] || die "No payload files in $(sg_rel "$EXPORT_DIR") (run 'apply' first)." @@ -433,12 +496,11 @@ cmd_enrich() { do_convert() { "$SCRIPT_DIR/convert_hcl_to_json.sh" "$1"; } cmd_convert() { - sg_step "Phase: convert (HCL → JSON)" + phase_begin "convert (HCL → JSON)" payload_files [ "${#PF[@]}" -gt 0 ] || die "No payload files in $(sg_rel "$EXPORT_DIR") (run 'apply' first)." - sg_log "converting ${#PF[@]} payload(s), up to $CONC in parallel" - if run_parallel do_convert "$CONC" "${PF[@]}"; then - sg_success "converted ${#PF[@]} payload(s)" + if run_parallel do_convert "$CONC" "converting ${#PF[@]} payload file(s)" "${PF[@]}"; then + sg_success "converted ${#PF[@]} payload file(s) in $(phase_took)" else sg_err "conversion failed for one or more payloads" return 1 @@ -446,13 +508,13 @@ cmd_convert() { } cmd_validate() { - sg_step "Phase: validate" + phase_begin "validate" payload_files [ "${#PF[@]}" -gt 0 ] || die "No payload files in $(sg_rel "$EXPORT_DIR")." if "$SCRIPT_DIR/validate_payload.sh" "${PF[@]}"; then - sg_success "all ${#PF[@]} payload(s) valid" + sg_success "${#PF[@]} payload file(s) valid against schema/sg-payload.schema.json" else - sg_err "validation failed" + sg_err "validation failed — fix the payload(s) above (or the transformer) and re-run '$PROG validate'" return 1 fi } @@ -479,7 +541,7 @@ names_json() { printf '%s\n' "$@" | "$JQ_BIN" -R . | "$JQ_BIN" -s .; } # the trigger pass see what was actually imported); each fallback is appended to # terraform-version-fallbacks.log. Any other per-workflow failure fails the file. do_import() { - local f="$1" seg grp out rc=0 ceiling="" failed=() fb=() name line tmp names work all_names + local f="$1" seg grp out rc=0 ceiling="" failed=() fb=() name line tmp names work all_names patch seg="$(seg_of "$f")" grp="$(group_for "$seg")" # With --workspace, import only the selected workflows (a filtered copy). @@ -512,12 +574,18 @@ do_import() { rm -f "$out" if [ "${#fb[@]}" -gt 0 ]; then - sg_warn "${#fb[@]} workflow(s) pinned above SG's managed Terraform ceiling ($ceiling); re-importing with $SG_DEFAULT_TF_VERSION" + sg_warn "${#fb[@]} workflow(s) pinned above SG's managed Terraform ceiling ($ceiling); re-importing with ${SG_TF_FALLBACK_LABEL:-$SG_DEFAULT_TF_VERSION}" names="$(names_json "${fb[@]}")" tmp="$(mktemp "$EXPORT_DIR/.fallback.$seg.XXXXXX")" - # Patch the affected workflows in the payload and re-import only those. - "$JQ_BIN" --arg v "$SG_DEFAULT_TF_VERSION" --argjson names "$names" \ - 'map(if (.ResourceName as $n | $names | index($n)) != null then .TerraformConfig.terraformVersion = $v else . end)' "$f" >"$tmp.full" && + # Patch the affected workflows in the payload and re-import only those: a + # fixed fallback version, or (SGDefaultTerraformVersion = null) no version at + # all so the API fills it from the org's execution preset. + if [ -n "$SG_DEFAULT_TF_VERSION" ]; then + patch='map(if (.ResourceName as $n | $names | index($n)) != null then .TerraformConfig.terraformVersion = $v else . end)' + else + patch='map(if (.ResourceName as $n | $names | index($n)) != null then del(.TerraformConfig.terraformVersion) else . end)' + fi + "$JQ_BIN" --arg v "$SG_DEFAULT_TF_VERSION" --argjson names "$names" "$patch" "$f" >"$tmp.full" && "$JQ_BIN" --argjson names "$names" \ 'map(select(.ResourceName as $n | $names | index($n) != null))' "$tmp.full" >"$tmp" || { @@ -531,7 +599,7 @@ do_import() { failed+=("$name") else line="$("$JQ_BIN" -r --arg n "$name" '.[] | select(.ResourceName == $n) | .TerraformConfig.terraformVersion' "$f")" - printf '%s/%s: %s -> %s (above SG managed ceiling %s)\n' "$grp" "$name" "$line" "$SG_DEFAULT_TF_VERSION" "$ceiling" >>"$EXPORT_DIR/terraform-version-fallbacks.log" + printf '%s/%s: %s -> %s (above SG managed ceiling %s)\n' "$grp" "$name" "$line" "${SG_DEFAULT_TF_VERSION:-execution preset}" "$ceiling" >>"$EXPORT_DIR/terraform-version-fallbacks.log" fi done rm -f "$out" @@ -566,17 +634,18 @@ tf_fallback_notice() { cat >&2 <<NOTICE StackGuardian ships managed Terraform runtimes only up to the last MPL-licensed (FOSS) release; newer versions are BSL-licensed and are not bundled. The - workflows above were created with $SG_DEFAULT_TF_VERSION instead of the version - pinned in TFC, so they will run a different Terraform than before - verify the - configuration is compatible before the first run. To keep a newer version, set - workspaceOverrides[<name>].terraformVersion to a binary path mounted from a - private runner, or use a custom runtime container template - (wfStepTemplateRevisionId), and re-import. + workflows above were created with ${SG_TF_FALLBACK_LABEL:-$SG_DEFAULT_TF_VERSION} + instead of the version pinned in TFC, so they will run a different Terraform + than before - verify the configuration is compatible before the first run. To + keep a newer version, set workspaceOverrides[<name>].terraformVersion to a + binary path mounted from a private runner, or point the org's execution preset + (or the override) at a custom runtime image (wfStepTemplateRevisionId) that + ships it, and re-import. NOTICE } cmd_import() { - sg_step "Phase: import" + phase_begin "import" [ -n "${SG_API_TOKEN:-}" ] || die "SG_API_TOKEN is not set." [ -n "$ORG" ] || die "StackGuardian org not set (use --org or SG_ORG)." command -v curl >/dev/null 2>&1 || die "curl is required for workflow-group checks/creation." @@ -591,12 +660,12 @@ cmd_import() { # Build the plan: resolve each project's group, check existence, and decide # which groups need creating. Override groups (from the map) must already exist. - local fail=0 to_create=" " f seg grp count override is_override code status - printf '%sImport plan%s (org: %s%s%s, %s)\n' "$C_BOLD" "$C_RESET" "$C_CYAN" "$ORG" "$C_RESET" "$SG_BASE_URL" >&2 - printf ' %s%-34s %-26s %-9s %s%s\n' "$C_BOLD" "FILE" "WORKFLOW GROUP" "WORKFLOWS" "STATUS" "$C_RESET" >&2 + local fail=0 to_create=" " n_create=0 total_wf=0 f seg grp count override is_override code status fw gw q i + local -a files=() groups=() counts=() statuses=() for f in "${PF[@]}"; do seg="$(seg_of "$f")" - count="$("$JQ_BIN" 'length' "$f")" + count="$("$JQ_BIN" --argjson ws "$(ws_filter_json)" '[.[] | select(($ws | length) == 0 or (.ResourceName as $n | $ws | index($n) != null))] | length' "$f")" + total_wf=$((total_wf + count)) override="" [ -f "$MAPPING" ] && override="$("$JQ_BIN" -r --arg k "$seg" '.[$k] // empty' "$MAPPING" 2>/dev/null || true)" if [ -n "$override" ]; then @@ -616,7 +685,11 @@ cmd_import() { fail=1 elif [ "$CREATE_GROUPS" -eq 1 ]; then status="${C_YELLOW}create${C_RESET}" - case "$to_create" in *" $grp "*) ;; *) to_create="$to_create$grp " ;; esac + case "$to_create" in *" $grp "*) ;; *) + to_create="$to_create$grp " + n_create=$((n_create + 1)) + ;; + esac else status="${C_RED}missing!${C_RESET}" fail=1 @@ -626,15 +699,34 @@ cmd_import() { 000) die "could not reach $SG_BASE_URL" ;; *) die "unexpected HTTP $code checking group '$grp'" ;; esac - printf ' %-34s %-26s %-9s %s\n' "$(basename "$f")" "$grp" "$count" "$status" >&2 + files+=("$(basename "$f")") + groups+=("$grp") + counts+=("$count") + statuses+=("$status") + done + fw="$(sg_maxlen 4 "${files[@]}")" + gw="$(sg_maxlen 14 "${groups[@]}")" + printf '%sImport plan%s (org: %s%s%s, %s)\n' "$C_BOLD" "$C_RESET" "$C_CYAN" "$ORG" "$C_RESET" "$SG_BASE_URL" >&2 + printf " %s%-${fw}s %-${gw}s %-9s %s%s\n" "$C_BOLD" "FILE" "WORKFLOW GROUP" "WORKFLOWS" "STATUS" "$C_RESET" >&2 + for ((i = 0; i < ${#files[@]}; i++)); do + printf " %-${fw}s %-${gw}s %-9s %s\n" "${files[i]}" "${groups[i]}" "${counts[i]}" "${statuses[i]}" >&2 done if [ "$fail" -ne 0 ]; then die "some groups are missing (override groups are not auto-created; create them or remove the override)." fi - # Fallback Terraform version for workflows the API rejects as above the - # managed ceiling: SGDefaultTerraformVersion from terraform.tfvars, else 1.5.7. - SG_DEFAULT_TF_VERSION="${SG_DEFAULT_TF_VERSION:-$(tfvars_get '.SGDefaultTerraformVersion' TERRAFORM-1.5.7)}" + # Fallback for workflows the API rejects as above the managed ceiling: a fixed + # SGDefaultTerraformVersion (missing key = 1.5.7), or an explicit null in + # terraform.tfvars = drop the version so the org's execution preset decides. + # The preset itself is read so the plan can show what "preset" resolves to. + if tfvars_is_null SGDefaultTerraformVersion; then + SG_DEFAULT_TF_VERSION="" + else + SG_DEFAULT_TF_VERSION="${SG_DEFAULT_TF_VERSION:-$(tfvars_get '.SGDefaultTerraformVersion' TERRAFORM-1.5.7)}" + fi + # shellcheck disable=SC2034 # read by report.sh (plan) and checklist.sh + SG_PRESET_JSON="$(sg_execution_preset)" || SG_PRESET_JSON="" + preset_labels show_import_plan "${PF[@]}" if [ "$DRY_RUN" -eq 1 ]; then sg_success "dry run — nothing was created or changed" @@ -642,9 +734,14 @@ cmd_import() { fi if [ "$ASSUME_YES" -ne 1 ]; then - printf '%sProceed?%s This imports to %s and creates any "create" groups. [y/N] ' "$C_BOLD$C_YELLOW" "$C_RESET" "$ORG" >&2 - read -r ans || ans="" - case "$ans" in y | Y | yes | YES) ;; *) die "Aborted." ;; esac + q="Import $total_wf workflow(s) into $ORG" + [ "$n_create" -gt 0 ] && q="$q and create $n_create workflow group(s)" + sg_interactive || die "no terminal to confirm the import — re-run with -y to import without a prompt" + if ! sg_confirm "$q?" N; then + sg_warn "import cancelled — nothing was changed in $ORG" + sg_dim "re-run '$PROG all' (or '$PROG import') to come back to this plan; the export phases are saved and skipped" + return 1 + fi fi # Create the missing tfc-* groups before importing into them. @@ -660,7 +757,7 @@ cmd_import() { # (a changed payload or a previous failure re-imports the whole file; # sg-cli updates existing workflows in place). local -a todo=() - local skipped=0 seg + local skipped=0 import_rc=0 for f in "${PF[@]}"; do seg="$(seg_of "$f")" if [ "$FRESH" -eq 0 ] && [ "${#WS_FILTER[@]}" -eq 0 ] && state_import_done "$seg" "$(sg_sha_files "$f")"; then @@ -669,11 +766,9 @@ cmd_import() { fi todo+=("$f") done - [ "$skipped" -gt 0 ] && sg_log "skipping $skipped payload(s) already imported and unchanged (use --fresh to re-import)" - local import_rc=0 + [ "$skipped" -gt 0 ] && sg_log "skipping $skipped payload file(s) already imported and unchanged (--fresh to re-import)" if [ "${#todo[@]}" -gt 0 ]; then - sg_log "importing ${#todo[@]} payload(s), up to $CONC in parallel (retries: $RETRIES)" - run_parallel do_import "$CONC" "${todo[@]}" || import_rc=1 + run_parallel do_import "$CONC" "importing ${#todo[@]} payload file(s) (retries: $RETRIES)" "${todo[@]}" || import_rc=1 for f in "${todo[@]}"; do seg="$(seg_of "$f")" if [ -f "$EXPORT_DIR/.import-result.$seg.json" ]; then @@ -684,7 +779,7 @@ cmd_import() { fi tf_fallback_notice if [ "$import_rc" -eq 0 ]; then - sg_success "import complete (${#todo[@]} payload(s) imported, $skipped skipped)" + sg_success "import complete (${#todo[@]} payload file(s) imported, $skipped skipped)" else sg_err "one or more workflows failed to import (see above); VCS triggers are still registered for the ones that succeeded" fi @@ -696,9 +791,24 @@ cmd_import() { fi [ "$SECRET_STUBS" -eq 1 ] && create_secret_stubs write_checklist + finish_line "$import_rc" return "$import_rc" } +# finish_line <rc> — the last line of an import / all run: outcome, total time, +# and whether the checklist still has items for a human. +finish_line() { + local open="${CHECKLIST_OPEN:-0}" took + took="$(sg_fmt_secs $((SECONDS - RUN_T0)))" + if [ "$1" -ne 0 ]; then + sg_err "finished with failures in $took — fix what is reported above and re-run '$PROG import' (only failed or changed files are retried)" + elif [ "$open" -gt 0 ]; then + sg_success "migration complete in $took — $open item(s) still need a human, see $(sg_rel "$EXPORT_DIR")/post-import-checklist.md" + else + sg_success "migration complete in $took — nothing left to do by hand" + fi +} + # Single source of truth for shell completion (keep in sync with the parser below # and the host-only flags in sg-migrate.sh). SG_COMMANDS="init preflight apply enrich convert validate import triggers checklist all clean completion" @@ -873,7 +983,7 @@ main() { export SG_VERBOSE="$VERBOSE" case "$CMD" in - init) cmd_init ;; + init) cmd_init standalone ;; clean) cmd_clean ;; apply) cmd_apply ;; enrich) cmd_enrich ;; @@ -904,8 +1014,12 @@ main() { preflight_run all [ "$FRESH" -eq 1 ] && { state_reset; sg_log "--fresh: previous run state discarded"; } JQ_BIN="$(sg_resolve jq sg_ensure_jq)" - run_phase apply "$(sg_sha "$(sg_sha_files "$TFVARS")|$(ws_filter_json)")" cmd_apply - if [ "$ENRICH_VARSETS" -eq 1 ]; then run_phase enrich "$(payload_sha)" cmd_enrich; fi + PHASE_TOTAL=4 + [ "$ENRICH_VARSETS" -eq 1 ] && PHASE_TOTAL=5 + local apply_sha + apply_sha="$(sg_sha "$(sg_sha_files "$TFVARS")|$(ws_filter_json)")" + run_phase apply "$apply_sha" cmd_apply + if [ "$ENRICH_VARSETS" -eq 1 ]; then run_phase enrich "$apply_sha" cmd_enrich; fi run_phase convert "$(payload_sha)" cmd_convert run_phase validate "$(payload_sha)" cmd_validate cmd_import diff --git a/scripts/tools.sh b/scripts/tools.sh index 901a9ff..3100080 100755 --- a/scripts/tools.sh +++ b/scripts/tools.sh @@ -53,6 +53,65 @@ sg_err() { printf '%s[sg-migrate] ERROR%s %s\n' "$C_RED$C_BOLD" "$C_RESET" "$*" sg_success() { printf '%s[sg-migrate] ✓%s %s\n' "$C_GREEN$C_BOLD" "$C_RESET" "$*" >&2; } sg_step() { printf '\n%s==> %s%s\n' "$C_CYAN$C_BOLD" "$*" "$C_RESET" >&2; } sg_dim() { printf '%s %s%s\n' "$C_DIM" "$*" "$C_RESET" >&2; } +# sg_row <label> <value> — an aligned " label value" line (review/summary tables). +sg_row() { printf ' %s%-26s%s %s\n' "$C_BOLD" "$1" "$C_RESET" "$2" >&2; } + +# sg_fmt_secs <seconds> — 14s / 1m 12s / 1h 02m, for "done in ..." messages. +sg_fmt_secs() { + local s="${1:-0}" + if [ "$s" -ge 3600 ]; then printf '%dh %02dm' "$((s / 3600))" "$(((s % 3600) / 60))" + elif [ "$s" -ge 60 ]; then printf '%dm %02ds' "$((s / 60))" "$((s % 60))" + else printf '%ds' "$s"; fi +} + +# sg_maxlen <min> <string>... — the longest string's length, but at least <min> +# (used to size table columns to their content). +sg_maxlen() { + local w="$1" s + shift + for s in "$@"; do [ "${#s}" -gt "$w" ] && w="${#s}"; done + printf '%s' "$w" +} + +# In-place progress line for long-running steps. Animated only when stderr is a +# TTY with colors on (same switch as the colors); plain log lines otherwise, so +# CI logs never fill up with spinner frames. +SG_ANIMATE=0 +[ -n "$C_RESET" ] && SG_ANIMATE=1 +_SG_SPIN_FRAMES=('⠋' '⠙' '⠹' '⠸' '⠼' '⠴' '⠦' '⠧' '⠇' '⠏') +_SG_SPIN_I=0 +sg_spin_frame() { + [ "$SG_ANIMATE" = "1" ] || return 0 + printf '\r%s[sg-migrate]%s %s%s%s %s\033[K' "$C_CYAN" "$C_RESET" "$C_CYAN" "${_SG_SPIN_FRAMES[_SG_SPIN_I % 10]}" "$C_RESET" "$1" >&2 + _SG_SPIN_I=$((_SG_SPIN_I + 1)) +} +sg_spin_clear() { + [ "$SG_ANIMATE" = "1" ] && printf '\r\033[K' >&2 + return 0 +} + +# sg_run_quiet <running-label> <done-label> <logfile> <cmd...> — run cmd with +# stdout+stderr captured in logfile, showing "<running-label> (elapsed)" as a +# live line meanwhile, then a "<done-label> (took Ns)" log line. Returns the +# command's exit code; the caller decides what to do with the log. +sg_run_quiet() { + local running="$1" done_label="$2" log="$3" pid rc=0 t0=$SECONDS + shift 3 + "$@" >"$log" 2>&1 & + pid=$! + if [ "$SG_ANIMATE" = "1" ]; then + while kill -0 "$pid" 2>/dev/null; do + sg_spin_frame "$running $C_DIM($(sg_fmt_secs $((SECONDS - t0))))$C_RESET" + sleep 0.2 + done + sg_spin_clear + else + sg_log "$running..." + fi + wait "$pid" || rc=$? + [ "$rc" -eq 0 ] && sg_log "$done_label $C_DIM($(sg_fmt_secs $((SECONDS - t0))))$C_RESET" + return "$rc" +} sg_arch() { case "$(uname -m)" in diff --git a/scripts/validate_payload.sh b/scripts/validate_payload.sh index c5fbd4a..d8545ba 100755 --- a/scripts/validate_payload.sh +++ b/scripts/validate_payload.sh @@ -20,10 +20,21 @@ fi # Resolve yajsv from PATH (Docker image) or download+cache (native). YAJSV_BIN=$(sg_resolve yajsv sg_ensure_yajsv) -sg_log "validating $# file(s) against schema/sg-payload.schema.json" -# yajsv prints "<file>: valid" per file and exits non-zero if any file fails. -# Strip the repo-root prefix from its output for readable, relative paths -# (pipefail off so the pipeline's status is sed's; yajsv's status via PIPESTATUS). +# yajsv prints "<file>: pass" / "<file>: fail: <reason>" per file and exits +# non-zero if any file fails. Re-render its lines in the migrator's style +# (✓/✗ + file name; the raw lines with -v) — pipefail off so the pipeline's +# status is the loop's; yajsv's own status comes back via PIPESTATUS. set +o pipefail -"$YAJSV_BIN" -s "$SCHEMA" "$@" 2>&1 | sed "s#${SG_REPO_ROOT}/##g" | awk '!seen[$0]++' +"$YAJSV_BIN" -s "$SCHEMA" "$@" 2>&1 | sed "s#${SG_REPO_ROOT}/##g" | awk '!seen[$0]++' | while IFS= read -r line; do + if [ "${SG_VERBOSE:-0}" = "1" ]; then + printf ' %s\n' "$line" >&2 + continue + fi + case "$line" in + *": pass") printf ' %s✓%s %s\n' "$C_GREEN" "$C_RESET" "$(basename "${line%: pass}")" >&2 ;; + *": fail: "*) printf ' %s✗%s %s: %s\n' "$C_RED$C_BOLD" "$C_RESET" "$(basename "${line%%: fail: *}")" "${line#*: fail: }" >&2 ;; + *": error: "*) printf ' %s✗%s %s: %s\n' "$C_RED$C_BOLD" "$C_RESET" "$(basename "${line%%: error: *}")" "${line#*: error: }" >&2 ;; + *) printf ' %s\n' "$line" >&2 ;; + esac +done exit "${PIPESTATUS[0]}" diff --git a/transformer/terraform-cloud/example_payload.jsonc b/transformer/terraform-cloud/example_payload.jsonc index 60a67d9..7113182 100644 --- a/transformer/terraform-cloud/example_payload.jsonc +++ b/transformer/terraform-cloud/example_payload.jsonc @@ -34,11 +34,11 @@ "ResourceName": "", // workspace name "RunnerConstraints": { "type": "" - }, // type should be "shared" or "private" i.e. "RunnerConstraints": { "type": "private", "names":["runner-group-name"]} + }, // type should be "shared" or "private" i.e. "RunnerConstraints": { "type": "private", "names":["runner-group-name"]}. Omit the whole key (SGDefaultRunnerConstraints = null) and StackGuardian applies the org's execution preset at import. "Tags": [], // workflow tags "TerraformConfig": { "managedTerraformState": true, // managed state from StackGuardian - "terraformVersion": "TERRAFORM-1.1.6" // version of terraform + "terraformVersion": "TERRAFORM-1.1.6" // version of terraform; omit the key (SGTerraformVersionSource = "preset", or a null fallback) and the org's execution preset decides. The API only fills keys that are absent, never null/"". }, "UserSchedules": [], "VCSConfig": { diff --git a/transformer/terraform-cloud/locals.tf b/transformer/terraform-cloud/locals.tf index 5692f95..471f68e 100644 --- a/transformer/terraform-cloud/locals.tf +++ b/transformer/terraform-cloud/locals.tf @@ -42,13 +42,41 @@ locals { } # Workspaces whose terraform_version is not a pinned semver (e.g. "latest" or - # a constraint) and have no per-workspace override fall back to the default. + # a constraint) and have no per-workspace override fall back to the default + # (SGDefaultTerraformVersion, or the org's execution preset when that is null). versionFallbacks = { for name in local.workflowNames : name => data.tfe_workspace.data[name].terraform_version if !can(regex("^[0-9]+\\.[0-9]+\\.[0-9]+$", data.tfe_workspace.data[name].terraform_version)) && try(var.workspaceOverrides[name].terraformVersion, null) == null } + # Terraform version sent per workflow. An override is always sent as-is. + # Otherwise "carry" keeps a pinned TFC semver (TERRAFORM-x.y.z) and falls back + # to SGDefaultTerraformVersion for anything else; "preset" (or a null default) + # yields null, and the key is then left out of the payload so StackGuardian + # fills it from the org's execution preset at import time. + tfVersion = { + for name in local.workflowNames : + name => ( + try(var.workspaceOverrides[name].terraformVersion, null) != null ? var.workspaceOverrides[name].terraformVersion : + var.SGTerraformVersionSource == "preset" ? null : + can(regex("^[0-9]+\\.[0-9]+\\.[0-9]+$", data.tfe_workspace.data[name].terraform_version)) ? "TERRAFORM-${data.tfe_workspace.data[name].terraform_version}" : + var.SGDefaultTerraformVersion + ) + } + + # Runner constraints sent per workflow: override, else the global default, + # else null (key left out, the execution preset decides). Picked via a tuple + # rather than a conditional: an override with "names" and a default without + # it are different object types, which a conditional refuses to unify. + runnerConstraints = { + for name in local.workflowNames : + name => try([for c in [ + try(var.workspaceOverrides[name].RunnerConstraints, null), + var.SGDefaultRunnerConstraints == null ? null : { for k, v in var.SGDefaultRunnerConstraints : k => v if v != null } + ] : c if c != null][0], null) + } + # Workspaces not using "remote" execution may not store their state in TFC, so # the API state export can come back empty; flag them in the summary. nonRemoteModes = { @@ -67,7 +95,7 @@ locals { # One SG workflow payload per workspace. Per-workspace overrides win over the # SGDefault* values; everything else is derived from the TFC workspace. workflowPayload = { - for wsName, wsId in data.tfe_workspace_ids.data.ids : wsName => { + for wsName, wsId in data.tfe_workspace_ids.data.ids : wsName => merge({ CLIConfiguration = { "WorkflowGroup" : { # SG workflow group per TFC project: tfc-<project> (matches the group @@ -87,7 +115,6 @@ locals { ) DeploymentPlatformConfig = try(var.workspaceOverrides[wsName].DeploymentPlatformConfig, null) != null ? var.workspaceOverrides[wsName].DeploymentPlatformConfig : var.SGDefaultDeploymentPlatformConfig - RunnerConstraints = try(var.workspaceOverrides[wsName].RunnerConstraints, null) != null ? var.workspaceOverrides[wsName].RunnerConstraints : { for k, v in var.SGDefaultRunnerConstraints : k => v if v != null } VCSConfig = { "iacVCSConfig" : { @@ -166,18 +193,18 @@ locals { Approvers = try(var.workspaceOverrides[wsName].Approvers, null) != null ? var.workspaceOverrides[wsName].Approvers : (data.tfe_workspace.data[wsName].auto_apply ? [] : var.SGDefaultWfApprovers) - TerraformConfig = { + # terraformVersion is omitted (not null) when the execution preset should + # decide: the SG API only fills in keys that are absent from the payload. + TerraformConfig = { for k, v in { "managedTerraformState" : true, - "terraformVersion" : ( - try(var.workspaceOverrides[wsName].terraformVersion, null) != null ? var.workspaceOverrides[wsName].terraformVersion : - can(regex("^[0-9]+\\.[0-9]+\\.[0-9]+$", data.tfe_workspace.data[wsName].terraform_version)) ? "TERRAFORM-${data.tfe_workspace.data[wsName].terraform_version}" : var.SGDefaultTerraformVersion - ), + "terraformVersion" : local.tfVersion[wsName], "approvalPreApply" : !data.tfe_workspace.data[wsName].auto_apply - } + } : k => v if v != null } WfType = "TERRAFORM" UserSchedules = [] - } + # RunnerConstraints likewise: present only when we have a value to send. + }, { for k, v in { RunnerConstraints = local.runnerConstraints[wsName] } : k => v if v != null }) } # Group payloads by TFC project so each project imports into its own SG @@ -198,13 +225,18 @@ locals { # Machine-readable migration summary (also rendered to markdown). summary = { - organization = var.tfOrg - workspaceCount = length(local.workflowNames) - projectWorkspaceCounts = { for pid in local.projectsUsed : try(local.projectNames[pid], pid) => length(local.payloadByProject[pid]) } - skippedSensitiveVars = { for name, vars in local.sensitiveVars : name => vars if length(vars) > 0 } - strippedVars = { for name, vars in local.strippedVars : name => vars if length(vars) > 0 } - ignoreVarPatterns = var.ignoreVarPatterns - terraformVersionFallbacks = local.versionFallbacks + organization = var.tfOrg + workspaceCount = length(local.workflowNames) + projectWorkspaceCounts = { for pid in local.projectsUsed : try(local.projectNames[pid], pid) => length(local.payloadByProject[pid]) } + skippedSensitiveVars = { for name, vars in local.sensitiveVars : name => vars if length(vars) > 0 } + strippedVars = { for name, vars in local.strippedVars : name => vars if length(vars) > 0 } + ignoreVarPatterns = var.ignoreVarPatterns + # Version policy, so the later phases can explain what each workflow runs. + terraformVersionSource = var.SGTerraformVersionSource + terraformVersionDefault = var.SGDefaultTerraformVersion # null = the execution preset decides + runnerConstraintsSource = var.SGDefaultRunnerConstraints == null ? "preset" : "config" + tfcTerraformVersions = { for name in local.workflowNames : name => data.tfe_workspace.data[name].terraform_version } + terraformVersionFallbacks = var.SGTerraformVersionSource == "carry" ? local.versionFallbacks : {} nonRemoteExecutionModes = local.nonRemoteModes renamedWorkspaces = { for name in local.workflowNames : name => local.resourceNames[name] if local.resourceNames[name] != name } variableSetsReminder = "TFC Variable Set variables are merged by the 'enrich' step (non-sensitive only). Sensitive set vars can't be read from the API — recreate them as SG secrets; see the enrich step output." diff --git a/transformer/terraform-cloud/summary.tmpl b/transformer/terraform-cloud/summary.tmpl index 7e7f811..d7df0de 100644 --- a/transformer/terraform-cloud/summary.tmpl +++ b/transformer/terraform-cloud/summary.tmpl @@ -28,8 +28,14 @@ Not migrated because they only mean something inside Terraform Cloud (patterns: %{ endfor ~} %{ endif ~} -## Terraform version fallbacks -Workspaces whose version was not a pinned semver; the configured SGDefaultTerraformVersion was used. +## Terraform version +%{ if summary.terraformVersionSource == "preset" ~} +No version is set on the migrated workflows (SGTerraformVersionSource = "preset"): StackGuardian applies the organisation's execution preset (Settings -> Runner groups -> Execution presets), or its platform default (managed Terraform 1.5.7) when none is configured. The workspaces ran these versions in TFC: +%{ for ws, ver in summary.tfcTerraformVersions ~} +- ${ws}: ${ver} +%{ endfor ~} +%{ else ~} +Pinned TFC versions are carried over (SGTerraformVersionSource = "carry"). Workspaces whose version was not a pinned semver use %{ if summary.terraformVersionDefault == null }the organisation's execution preset (SGDefaultTerraformVersion = null)%{ else }the fallback ${summary.terraformVersionDefault}%{ endif }: %{ if length(summary.terraformVersionFallbacks) == 0 ~} - None. %{ else ~} @@ -37,6 +43,14 @@ Workspaces whose version was not a pinned semver; the configured SGDefaultTerraf - ${ws}: reported "${ver}" %{ endfor ~} %{ endif ~} +%{ endif ~} + +## Runners +%{ if summary.runnerConstraintsSource == "preset" ~} +- No runner constraints are set (SGDefaultRunnerConstraints = null); the organisation's execution preset decides (platform default: shared runners). +%{ else ~} +- Runner constraints come from terraform.tfvars (SGDefaultRunnerConstraints / workspaceOverrides). +%{ endif ~} ## Non-remote execution modes TFC may not hold state for these (local/agent execution); state export can be empty. diff --git a/transformer/terraform-cloud/terraform.tfvars.example b/transformer/terraform-cloud/terraform.tfvars.example index f588f18..e20789c 100644 --- a/transformer/terraform-cloud/terraform.tfvars.example +++ b/transformer/terraform-cloud/terraform.tfvars.example @@ -43,17 +43,28 @@ SGDefaultDeploymentPlatformConfig = [ # Runners for every workflow: SG-hosted shared runners (default), or put all # workflows behind a private runner group: # SGDefaultRunnerConstraints = { type = "private", names = ["sg-runner"] } +# Set to null to send no runner constraints, so StackGuardian applies the org's +# execution preset (Settings -> Runner groups -> Execution presets). SGDefaultRunnerConstraints = { type = "shared" } # Choose from: GITHUB_COM, BITBUCKET_ORG, GITLAB_COM, AZURE_DEVOPS, GIT_OTHER SGDefaultSourceConfigDestKind = "GITHUB_COM" -# SG Terraform version used when a workspace's terraform_version is not a pinned -# semver (e.g. "latest" or a constraint). Pinned versions are carried over as-is; -# if the SG API rejects one as above the managed ceiling (1.5.7 - the last MPL/ -# FOSS Terraform release; newer versions are BSL and not bundled), the importer -# re-creates that workflow with this version and logs it in -# export/terraform-version-fallbacks.log. +# Where the workflows get their Terraform version from: +# "carry" - keep each workspace's pinned TFC version; the rest (see below) use +# SGDefaultTerraformVersion +# "preset" - send no version at all: StackGuardian fills it from the org's +# execution preset (Settings -> Runner groups -> Execution presets), +# or its platform default (managed Terraform 1.5.7) +SGTerraformVersionSource = "carry" + +# Fallback for "carry": used when a workspace's terraform_version is not a pinned +# semver (e.g. "latest" or a constraint), and by the importer when the SG API +# rejects a pinned version as above the managed ceiling (1.5.7 - the last MPL/ +# FOSS Terraform release; newer versions are BSL and not bundled) - those +# workflows are re-created with this version and logged in +# export/terraform-version-fallbacks.log. Set to null to leave them to the org's +# execution preset instead. SGDefaultTerraformVersion = "TERRAFORM-1.5.7" # Pre-configure VCS triggers on each VCS-backed workflow, remapped from the diff --git a/transformer/terraform-cloud/variables.tf b/transformer/terraform-cloud/variables.tf index a8a103d..c265db6 100644 --- a/transformer/terraform-cloud/variables.tf +++ b/transformer/terraform-cloud/variables.tf @@ -79,17 +79,18 @@ variable "SGDefaultDeploymentPlatformConfig" { variable "SGDefaultRunnerConstraints" { default = { type = "shared" } - description = "Runner constraints applied to every workflow. Use { type = \"shared\" } for SG-hosted runners, or { type = \"private\", names = [\"<runner-group>\"] } to put every workflow behind a private runner group. Override per workspace via workspaceOverrides[name].RunnerConstraints." + nullable = true + description = "Runner constraints applied to every workflow. Use { type = \"shared\" } for SG-hosted runners, { type = \"private\", names = [\"<runner-group>\"] } to put every workflow behind a private runner group, or null to send no runner constraints at all so StackGuardian applies the org's execution preset (Settings -> Runner groups -> Execution presets; platform default: shared runners). Override per workspace via workspaceOverrides[name].RunnerConstraints." type = object({ type = string names = optional(list(string)) }) validation { - condition = contains(["shared", "private"], var.SGDefaultRunnerConstraints.type) - error_message = "SGDefaultRunnerConstraints.type must be \"shared\" or \"private\"." + condition = var.SGDefaultRunnerConstraints == null || contains(["shared", "private"], try(var.SGDefaultRunnerConstraints.type, "")) + error_message = "SGDefaultRunnerConstraints.type must be \"shared\" or \"private\" (or the whole variable null to defer to the execution preset)." } validation { - condition = var.SGDefaultRunnerConstraints.type != "private" || length(coalesce(var.SGDefaultRunnerConstraints.names, [])) > 0 + condition = var.SGDefaultRunnerConstraints == null || try(var.SGDefaultRunnerConstraints.type, "") != "private" || length(coalesce(try(var.SGDefaultRunnerConstraints.names, null), [])) > 0 error_message = "SGDefaultRunnerConstraints.names must list at least one runner group when type is \"private\"." } } @@ -100,9 +101,20 @@ variable "SGDefaultSourceConfigDestKind" { type = string } +variable "SGTerraformVersionSource" { + default = "carry" + description = "Where the migrated workflows get their Terraform version from. \"carry\": keep each workspace's pinned TFC version; workspaces without a pinned semver (e.g. 'latest') use SGDefaultTerraformVersion, and so does the importer when the SG API rejects a pinned version as above its managed ceiling (1.5.7). \"preset\": send no version at all, so StackGuardian fills it from the org's execution preset (Settings -> Runner groups -> Execution presets), or its platform default when none is configured; SGDefaultTerraformVersion is then ignored. A workspaceOverrides[name].terraformVersion is always sent as-is." + type = string + validation { + condition = contains(["carry", "preset"], var.SGTerraformVersionSource) + error_message = "SGTerraformVersionSource must be \"carry\" or \"preset\"." + } +} + variable "SGDefaultTerraformVersion" { default = "TERRAFORM-1.5.7" - description = "SG Terraform version used when a workspace's terraform_version is not a pinned semver (e.g. 'latest' or a version constraint). Also used by the importer when the SG API rejects a pinned version as above the managed ceiling (1.5.7, the last MPL/FOSS release; newer versions are BSL and not bundled). Use the SG-formatted value, e.g. TERRAFORM-1.5.7." + nullable = true + description = "Fallback SG Terraform version when SGTerraformVersionSource is \"carry\": used for workspaces whose terraform_version is not a pinned semver (e.g. 'latest' or a version constraint), and by the importer when the SG API rejects a pinned version as above the managed ceiling (1.5.7, the last MPL/FOSS release; newer versions are BSL and not bundled). Use the SG-formatted value, e.g. TERRAFORM-1.5.7, or null to leave those workflows to the org's execution preset instead." type = string } From 01405166ed9f36a0eebefb331afd15a6e4dbdafd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adis=20Halilovi=C4=87?= <adis.halilovic@stackguardian.io> Date: Mon, 14 Sep 2026 10:58:49 +0200 Subject: [PATCH 31/71] fix: split _sg_api so callers can read a 4xx body sg_api_raw does the request and returns the body on any status, with SG_HTTP_CODE set in the caller's shell; _sg_api keeps the logging and the body-on-2xx-only behaviour on top of it. --- scripts/lib/sg_api.sh | 40 +++++++++++++++++++++++----------------- 1 file changed, 23 insertions(+), 17 deletions(-) diff --git a/scripts/lib/sg_api.sh b/scripts/lib/sg_api.sh index e5856b6..6783051 100644 --- a/scripts/lib/sg_api.sh +++ b/scripts/lib/sg_api.sh @@ -18,8 +18,10 @@ sg_http_code() { # errors, so never append a fallback to its output; normalize instead. sg_norm_code() { case "$1" in [0-9][0-9][0-9]) printf '%s' "$1" ;; *) printf '000' ;; esac; } -# _sg_api <method> <url> [json-body] — shared request; body on stdout. -_sg_api() { +# sg_api_raw <method> <url> [json-body] — the request itself, no logging: the +# response body goes to stdout whatever the status, SG_HTTP_CODE is set, exit +# code per the contract above. Callers that need the error body use this. +sg_api_raw() { local method="$1" url="$2" body="${3-}" tmp code tmp="$(mktemp)" if [ -n "$body" ]; then @@ -33,24 +35,28 @@ _sg_api() { code="$(sg_norm_code "$code")" # shellcheck disable=SC2034 # read by callers SG_HTTP_CODE="$code" - case "$code" in - 2*) - cat "$tmp" - rm -f "$tmp" - return 0 - ;; - 4*) - sg_err " HTTP $code from ${url#"$SG_BASE_URL"}: $(head -c 400 "$tmp")" + cat "$tmp" + rm -f "$tmp" + case "$code" in 2*) return 0 ;; 4*) return 22 ;; *) return 1 ;; esac +} + +# _sg_api <method> <url> [json-body] — sg_api_raw plus logging; body on stdout +# only on 2xx (4xx/5xx are reported on stderr). +_sg_api() { + local url="$2" tmp rc=0 + tmp="$(mktemp)" + # Not a $(...) capture: SG_HTTP_CODE must survive into this shell. + sg_api_raw "$@" >"$tmp" || rc=$? + case "$rc" in + 0) cat "$tmp" ;; + 22) + sg_err " HTTP $SG_HTTP_CODE from ${url#"$SG_BASE_URL"}: $(head -c 400 "$tmp")" if declare -F explain_api_error >/dev/null; then explain_api_error "$(head -c 2000 "$tmp")"; fi - rm -f "$tmp" - return 22 - ;; - *) - sg_warn " HTTP $code from ${url#"$SG_BASE_URL"}" - rm -f "$tmp" - return 1 ;; + *) sg_warn " HTTP $SG_HTTP_CODE from ${url#"$SG_BASE_URL"}" ;; esac + rm -f "$tmp" + return "$rc" } sg_api_get() { _sg_api GET "$1"; } From 9662748874279688ee7f881f8552b29c6b574fa5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adis=20Halilovi=C4=87?= <adis.halilovic@stackguardian.io> Date: Mon, 14 Sep 2026 10:58:50 +0200 Subject: [PATCH 32/71] fix: import workflows without variables directly; sg-cli drops an empty iacInputData.data sg-cli 2.2.1 round-trips every bulk entry through sg-sdk-go v1.1.0's Workflow struct, whose iacInputData.data map is tagged omitempty, so a workspace with no Terraform variables reaches the API without the key and is rejected with "VCSConfig.iacInputData.data: This field is required." import_bulk now splits each payload file: workflows with variables go through sg-cli as before, the ones without are created straight from the payload via POST .../wfs/ (PATCH when the name exists) followed by the same state upload sg-cli does. The direct path prints sg-cli's "Failed to create <name>: <code>: <body>" line on rejection, so the ceiling fallback, error hints and checklist cover both paths. Workaround only; TODO(sg-cli) markers show what to remove once sg-cli ships with sg-sdk-go >= v1.5.7, where the field became a pointer and "data": {} is sent as given. --- CLAUDE.md | 2 +- README.md | 1 + scripts/lib/sg_api.sh | 42 +++++++++++++++++++++++++++++++++++++ scripts/migrate.sh | 49 +++++++++++++++++++++++++++++++++++++++++-- 4 files changed, 91 insertions(+), 3 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 1a3d3f1..0a8d2c2 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -24,7 +24,7 @@ The migration is a user-driven pipeline, not a single program: The five phases are wrapped by an orchestrator so users don't run them by hand: - `sg-migrate.sh` (host entrypoint, repo root) — runs `scripts/migrate.sh` **inside the Docker image** (`Dockerfile`), bind-mounting the repo at `/app` and forwarding `SG_API_TOKEN`/`SG_ORG`/`SG_BASE_URL`/`TFE_TOKEN`/`TF_TOKEN_*`. For TFC auth it prefers a long-lived `TFE_TOKEN` (env); otherwise it mounts `~/.terraform.d/credentials.tfrc.json` read-only. Runs natively instead when `--native`/`--local` is passed, `SG_NATIVE=1` is set, the command is `clean`/`completion`/help (or no command is given), or Docker is absent. Exports `SG_PROG` so `migrate.sh` shows `./sg-migrate.sh` in its usage/hints. -- `scripts/migrate.sh` — the actual orchestrator (sources `scripts/lib/*.sh`, see below). Subcommands `init|preflight|apply|enrich|convert|validate|import|triggers|checklist|all|clean|completion` (no command prints the help menu; `enrich` runs in `all` unless `--no-variable-sets`). Flags beyond the basics: `--dry-run` (import: plan only), `--fresh` (ignore run state), `--project SEG` / `--workspace NAME` (repeatable filters; apply maps `--workspace` to `-var workspacenames=[...]`, import works on a jq-filtered temp copy), `--skip-preflight`, `--no-secret-stubs`. Resolves each tool via `sg_resolve` (PATH first — the image installs them — else `sg_ensure_*` cache). `apply`/`all` first run `require_tfc_auth`, which resolves the token the tfe provider will use (`TFE_TOKEN`, `TF_TOKEN_<host>`, or the `terraform login` file for `tfHostname`) and verifies it with `GET /api/v2/account/details`, failing fast on a missing or rejected (expired) credential. Runs `convert` and `import` in parallel (`--concurrency`, default 4), raises `terraform apply -parallelism`, and retries `sg-cli` imports with backoff (`sg_retry`, `SG_RETRIES`). Import resolves each project's workflow group, checks existence via the SG API, shows a plan (`exists`/`create`), prompts (skip with `-y`), **creates the missing `tfc-<project>` groups via the API**, then imports. sg-cli exits 0 even when individual workflows fail, so `do_import` parses its output: a workflow rejected as above SG's managed Terraform ceiling (1.5.7, last MPL/FOSS release) is re-imported with `SGDefaultTerraformVersion` (read from `terraform.tfvars`; an explicit `null` there means the `terraformVersion` key is deleted instead, so the org's execution preset decides — `tfvars_is_null` tells an explicit null from a missing key), the payload patched in place, and the case logged to `export/terraform-version-fallbacks.log` plus a printed notice; any other per-workflow failure fails the run. `cmd_import` reads the org's execution preset (`sg_execution_preset`, `GET /orgs/{org}/` → `Settings.workflowDefaults`) and `preset_labels` (report.sh) turns it into the `preset (1.5.7)` / `preset (private:rg)` cells of the plan and the `SG_TF_FALLBACK_LABEL` used in messages. The trigger pass skips workflows that do not exist in SG, and `sg_api_post` returns 22 on 4xx so `sg_retry` (via `SG_NO_RETRY_RC`) does not retry definitive errors. `clean` removes local artifacts (`export/`, TF state, tool cache); `clean --all` also removes config. `completion bash|zsh` prints a completion script (`cmd_completion`; commands/options come from `SG_COMMANDS`/`SG_OPTIONS`, keep them in sync with the parser and the host flags); like `clean` it always runs natively. Output is concise by default — `apply` captures terraform's init/plan output and shows only progress + the final summary (or the full log on failure, `TF_IN_AUTOMATION=1`/`-no-color`); `-v`/`--verbose` (`SG_VERBOSE`) streams everything and un-gates the convert detail lines. Colored logging via `sg_step`/`sg_log`/`sg_success`/`sg_warn`/`sg_err`/`sg_row`; paths shown relative via `sg_rel` (all auto-off when not a TTY / `NO_COLOR`). Long steps show a live progress line (`sg_run_quiet` for terraform init/apply, `run_parallel <fn> <max> <label> <items>` for convert/import/triggers — animated only when `SG_ANIMATE=1`, i.e. colors are on; plain log lines otherwise) and report their duration (`sg_fmt_secs`). Phases start with `phase_begin` — inside `all` the header is numbered (`Phase 2/5: ...`, `PHASE_TOTAL` set in `main`) and `phase_took` gives the elapsed time; `run_parallel` prints a job's output as-is when it is a single line and adds the `── <file> ──` header only for longer or failed output (always with `-v`). The import confirmation goes through `sg_confirm` (so `SG_ANSWERS_FILE` works; a declined prompt is a neutral "cancelled" line, not an error) and `import`/`all` end with `finish_line` (outcome, total time, open checklist items from `CHECKLIST_OPEN`). A Ctrl-C is trapped to clear the progress line and print a resume hint. +- `scripts/migrate.sh` — the actual orchestrator (sources `scripts/lib/*.sh`, see below). Subcommands `init|preflight|apply|enrich|convert|validate|import|triggers|checklist|all|clean|completion` (no command prints the help menu; `enrich` runs in `all` unless `--no-variable-sets`). Flags beyond the basics: `--dry-run` (import: plan only), `--fresh` (ignore run state), `--project SEG` / `--workspace NAME` (repeatable filters; apply maps `--workspace` to `-var workspacenames=[...]`, import works on a jq-filtered temp copy), `--skip-preflight`, `--no-secret-stubs`. Resolves each tool via `sg_resolve` (PATH first — the image installs them — else `sg_ensure_*` cache). `apply`/`all` first run `require_tfc_auth`, which resolves the token the tfe provider will use (`TFE_TOKEN`, `TF_TOKEN_<host>`, or the `terraform login` file for `tfHostname`) and verifies it with `GET /api/v2/account/details`, failing fast on a missing or rejected (expired) credential. Runs `convert` and `import` in parallel (`--concurrency`, default 4), raises `terraform apply -parallelism`, and retries `sg-cli` imports with backoff (`sg_retry`, `SG_RETRIES`). Import resolves each project's workflow group, checks existence via the SG API, shows a plan (`exists`/`create`), prompts (skip with `-y`), **creates the missing `tfc-<project>` groups via the API**, then imports. Imports go through `import_bulk`: workflows with Terraform variables via `sg-cli workflow create --bulk`, workflows **without** via a direct `POST .../wfs/` (`sg_create_workflow` + `sg_upload_tfstate` in `sg_api.sh`) because sg-cli drops an empty `iacInputData.data` and the API rejects the workflow — a workaround (`TODO(sg-cli)` markers) until sg-cli ships with sg-sdk-go >= v1.5.7, which sends the empty object; the direct path prints sg-cli-style `Failed to create <name>: <code>: <body>` lines so the parsing below covers both. sg-cli exits 0 even when individual workflows fail, so `do_import` parses its output: a workflow rejected as above SG's managed Terraform ceiling (1.5.7, last MPL/FOSS release) is re-imported with `SGDefaultTerraformVersion` (read from `terraform.tfvars`; an explicit `null` there means the `terraformVersion` key is deleted instead, so the org's execution preset decides — `tfvars_is_null` tells an explicit null from a missing key), the payload patched in place, and the case logged to `export/terraform-version-fallbacks.log` plus a printed notice; any other per-workflow failure fails the run. `cmd_import` reads the org's execution preset (`sg_execution_preset`, `GET /orgs/{org}/` → `Settings.workflowDefaults`) and `preset_labels` (report.sh) turns it into the `preset (1.5.7)` / `preset (private:rg)` cells of the plan and the `SG_TF_FALLBACK_LABEL` used in messages. The trigger pass skips workflows that do not exist in SG, and `sg_api_post` returns 22 on 4xx so `sg_retry` (via `SG_NO_RETRY_RC`) does not retry definitive errors. `clean` removes local artifacts (`export/`, TF state, tool cache); `clean --all` also removes config. `completion bash|zsh` prints a completion script (`cmd_completion`; commands/options come from `SG_COMMANDS`/`SG_OPTIONS`, keep them in sync with the parser and the host flags); like `clean` it always runs natively. Output is concise by default — `apply` captures terraform's init/plan output and shows only progress + the final summary (or the full log on failure, `TF_IN_AUTOMATION=1`/`-no-color`); `-v`/`--verbose` (`SG_VERBOSE`) streams everything and un-gates the convert detail lines. Colored logging via `sg_step`/`sg_log`/`sg_success`/`sg_warn`/`sg_err`/`sg_row`; paths shown relative via `sg_rel` (all auto-off when not a TTY / `NO_COLOR`). Long steps show a live progress line (`sg_run_quiet` for terraform init/apply, `run_parallel <fn> <max> <label> <items>` for convert/import/triggers — animated only when `SG_ANIMATE=1`, i.e. colors are on; plain log lines otherwise) and report their duration (`sg_fmt_secs`). Phases start with `phase_begin` — inside `all` the header is numbered (`Phase 2/5: ...`, `PHASE_TOTAL` set in `main`) and `phase_took` gives the elapsed time; `run_parallel` prints a job's output as-is when it is a single line and adds the `── <file> ──` header only for longer or failed output (always with `-v`). The import confirmation goes through `sg_confirm` (so `SG_ANSWERS_FILE` works; a declined prompt is a neutral "cancelled" line, not an error) and `import`/`all` end with `finish_line` (outcome, total time, open checklist items from `CHECKLIST_OPEN`). A Ctrl-C is trapped to clear the progress line and print a resume hint. - `scripts/lib/` — sourced libraries (a `lib/` gitignore rule is negated for this dir): `prompt.sh` (`sg_ask`/`sg_confirm`/`sg_select`; tty or `SG_ANSWERS_FILE` for tests; `SG_NONINTERACTIVE=1`/`-y`/no TTY → defaults), `tfvars.sh` (`tfvars_get` cached hcl2json reads, `tfvars_has`/`tfvars_is_null` to tell an explicit `null` from a missing key, `tfvars_write` renders the wizard's `W_*` vars — `W_TF_SOURCE`, and `W_TF_VERSION`/`W_RUNNER_JSON` may be the literal `null`; plain strings go through `_tfvars_str`, so quotes/backslashes in names stay valid HCL), `tfc_api.sh` (`tfc_http`/`tfc_get_all` paginated, `tfc_list_{orgs,projects,workspaces}` — workspaces include `vcs_provider`/`vcs_url`/`vcs_identifier` —, `tfc_select_workspaces` (the module's name-glob / include-tags(all) / exclude-tags filter, shared by wizard and preflight), `tfc_vcs_summary`/`tfc_vcs_kind_for`/`tfc_vcs_label_for`, `tfc_token` with provider precedence, `require_tfc_auth`), `sg_api.sh` (`_sg_api` 0/22/1 contract + `SG_HTTP_CODE`, `sg_list_integrations`, `sg_integration_type`, `sg_vcs_kind_of` (connector type → `sourceConfigDestKind`), `sg_execution_preset` + `sg_preset_{desc,runner_desc,version_desc,runner_provided}` (the org's execution preset and how to describe it), `sg_*_exists`, `sg_list_workflows`, `sg_create_secret`, `sg_patch_workflow`, `wfgroup_*`), `wizard.sh` (`wizard_run`: 4 steps TFC → SG → defaults → review; the TFC step keeps the workspace list and derives, via `tfc_vcs_summary`, which VCS provider the selected workspaces use and the repositories' URL prefix — the SG step lists matching connectors first and takes the repo URL prefix from TFC (never from a previous tfvars written for another provider kind), warns on a kind mismatch, reads the org's execution preset and offers it as the runner choice; the defaults step asks whether to carry TFC versions or use the preset (`SGTerraformVersionSource`) and, for carry, which fallback (`SGDefaultTerraformVersion` or the preset = null); the review shows human-readable rows plus the `tfc-<project>` groups; `_w_select_lines` feeds menus line-by-line so names with spaces/globs survive), `preflight.sh` (`preflight_run <apply|import|all>`, once per process; ✓ lines are plain language and name the connector's role/kind, ✗/! lines name the tfvars field to fix; besides existence it checks consistency — VCS kind vs. the connector's type (fail), repo URL prefix vs. where the TFC repositories live or the kind's well-known host (warn); it also reports the org's execution preset next to the version/runner policy and warns when `carry` meets a preset that mounts a runner-provided binary), `state.sh` (`.sg/state.json`: `phases.<name>.input_sha`, `import.<seg>.{imported,failed,tf_fallback}`, `triggers.<seg>`, `secrets.<name>`; `run_phase` in migrate.sh skips unchanged phases), `report.sh` (`show_migration_summary` after apply — incl. a state-files row —, `show_import_plan` per-workflow table with columns sized to their content), `errors.sh` (`explain_api_error` regex→hint table, `SG_API_HINTS`), `checklist.sh` (`create_secret_stubs` → SG secret `tfc-<wf>-<VAR>` = `CHANGE_ME`, referenced as `${secret::<name>}` via PATCH + payload patch; `write_checklist` → `export/post-import-checklist.md`, printed to the terminal as one ✓/! status line per section plus the imported groups with UI links — the file has the details; sets `CHECKLIST_OPEN`). Parallel jobs (`run_parallel`) run in subshells, so `do_import`/`do_set_triggers` write `.import-result.<seg>.json` / `.triggers-result.<seg>.json` into the export dir and the caller merges them into state. - `scripts/tools.sh` — sourced by the other scripts (sets `SG_REPO_ROOT` to its parent dir). Provides `sg_resolve` (PATH-or-cache), `sg_ensure_{jq,hcl2json,yajsv,sgcli}` (pinned downloads into `.sg/cached/`; `sg-cli` is the Go binary), `sg_retry`, the color vars + log helpers. The Docker image bundles these tools at build time (`yajsv` built from source so arm64 works), so in-container runs never download. - **Variable sets** — `scripts/enrich_variable_sets.sh <tfc-org> <payload...>` (run by `cmd_enrich`, which reads `tfOrg`/`tfHostname` from `terraform.tfvars` via `hcl2json` — variable sets live in the **TFC** org, not `SG_ORG`). Lists all sets + vars via the TFC API (Bearer token from the `terraform login` creds file or `TFE_TOKEN`), computes per-workspace effective vars (scope: global/project/workspace; precedence: priority sets > workspace vars > non-priority sets), and merges them into each workflow (matched by workspace name parsed from `TfStateFilePath`). Sensitive set vars and key conflicts are reported. Skipped by `--no-variable-sets`. diff --git a/README.md b/README.md index 8ada2c7..3c0250b 100644 --- a/README.md +++ b/README.md @@ -168,6 +168,7 @@ To update workflows with different details, re-run the sg-cli command with the m - **Sensitive variables become placeholder secrets.** TFC never returns sensitive values via the API. The export omits them (listed in `migration-summary.md`); after import the orchestrator creates an SG secret `tfc-<workflow>-<VAR>` with the value `CHANGE_ME` for each, references it from the workflow (`${secret::<name>}`, as an environment variable or IaC input) and lists it in `export/post-import-checklist.md`. Set the real values in the SG UI. `--no-secret-stubs` leaves SG secrets untouched. - **Terraform version fallback (FOSS ceiling).** With `SGTerraformVersionSource = "carry"` (the default) pinned versions are carried over as-is and tried first at import, so a custom runtime image or private runner that ships that binary keeps working; workspaces set to `latest` or a version constraint use `SGDefaultTerraformVersion` at export time. StackGuardian's _managed_ runtimes only go up to **1.5.7**, the last MPL-licensed (FOSS) Terraform release; newer versions are BSL-licensed and are not bundled. When the API rejects a workflow for that reason, the importer **automatically re-imports it with `SGDefaultTerraformVersion`** (or, when that is `null`, without any version so the execution preset decides — see the next bullet), patches the payload file to match, prints a notice, and records each case in `export/terraform-version-fallbacks.log`. Those workflows run a different Terraform than they did in TFC, so check compatibility before the first run. To keep a newer version, set `workspaceOverrides[<name>].terraformVersion` to a binary path mounted from a private runner (or use a custom runtime container template) and re-import. - **Execution presets.** StackGuardian org admins can define an execution preset (Settings → Runner groups → Execution presets): default runner constraints plus a Terraform and an OpenTofu configuration that the API applies to any new workflow whose payload does not carry those fields. To lean on it, set `SGTerraformVersionSource = "preset"` (no version is sent at all) and/or `SGDefaultRunnerConstraints = null` (no runner constraints are sent); `SGDefaultTerraformVersion = null` keeps carrying TFC pins but hands the unpinned and rejected ones to the preset. `init` offers these choices with the org's current preset shown inline, preflight prints what the preset would supply, and the import plan marks such cells as `preset (…)`. The preset's custom runtime image or runner-provided binary is inherited in every mode, since the migrator never sets those keys. +- **Workspaces without variables are imported directly.** sg-cli (up to v2.2.1) drops an empty `iacInputData.data` from the request, and the API then rejects the workflow with `VCSConfig.iacInputData.data: This field is required.` The importer therefore creates workflows that have no Terraform variables straight through the API (same create, same state upload) and uses sg-cli for the rest. This is a workaround until sg-cli picks up sg-sdk-go v1.5.7, which fixes the dropped key. - **State export is idempotent.** Re-running `terraform apply` only pulls state for workspaces not yet exported. Set `forceStateRefresh = true` to re-pull everything. Workspaces not using `remote` execution may export incomplete state — see the summary. - **Workflow naming.** `ResourceName` mirrors the TFC workspace name, sanitized to StackGuardian's rules (1-100 chars, `[-a-zA-Z0-9_]`); any rename is listed in the summary and the checklist. - **Preflight.** `apply`, `import` and `all` first verify the TFC and SG tokens, the TFC org and workspace selection, and that every connector, secret and runner group referenced in `terraform.tfvars` exists. Fix what it reports (or re-run `init`); `--skip-preflight` bypasses it. diff --git a/scripts/lib/sg_api.sh b/scripts/lib/sg_api.sh index 6783051..cb16c60 100644 --- a/scripts/lib/sg_api.sh +++ b/scripts/lib/sg_api.sh @@ -95,6 +95,48 @@ sg_list_workflows() { # sg_patch_workflow <group> <wf> <json> — PATCH a workflow. sg_patch_workflow() { sg_api_patch "$(wf_url "$1" "$2")" "$3"; } +# --- direct workflow create (sg-cli workaround) ----------------------------- +# sg-cli (<= v2.2.1, on sg-sdk-go v1.1.0) round-trips every payload entry through +# the SDK's Workflow struct, whose iacInputData.data map is tagged omitempty: a +# workspace with no Terraform variables ("data": {}) reaches the API without the +# key and is rejected with "VCSConfig.iacInputData.data: This field is required." +# Such entries are created straight from the payload JSON instead. +# TODO(sg-cli): workaround — the fix belongs in sg-cli: bump sg-sdk-go to +# >= v1.5.7 (IacInputData.Data became a pointer so "data": {} is sent) or POST +# the raw entry. Drop this block and import_bulk's split once a release has it. + +# sg_create_workflow <group> <entry-json> — POST one payload entry (PATCH when +# the name already exists, as sg-cli does). Silent on success; on failure prints +# "<http-code>: <body>" so the caller can log a sg-cli-style line. 0/22/1. +sg_create_workflow() { + local grp="$1" entry="$2" jq name body tmp rc=0 + jq="$(sg_resolve jq sg_ensure_jq)" + name="$("$jq" -r '.ResourceName' <<<"$entry")" + # CLIConfiguration is sg-cli's own block; VCSTriggers is applied in the + # trigger pass (the create API does not take it). + body="$("$jq" -c 'del(.CLIConfiguration, .VCSTriggers)' <<<"$entry")" + tmp="$(mktemp)" + # Response into a file, not $(...): SG_HTTP_CODE must survive into this shell. + sg_api_raw POST "$(sg_org_url)/wfgrps/$grp/wfs/" "$body" >"$tmp" || rc=$? + if [ "$rc" -eq 22 ] && grep -q "Workflow name not unique" "$tmp"; then + rc=0 + sg_api_raw PATCH "$(wf_url "$grp" "$name")" "$body" >"$tmp" || rc=$? + fi + [ "$rc" -eq 0 ] || printf '%s: %s\n' "$SG_HTTP_CODE" "$(tr -d '\n' <"$tmp")" + rm -f "$tmp" + return "$rc" +} + +# sg_upload_tfstate <group> <wf> <file> — the state upload sg-cli does after a +# bulk create: fetch the presigned URL, PUT the file. 0 on success. +sg_upload_tfstate() { + local url code + url="$(sg_api_get "$(wf_url "$1" "$2")tfstate_upload_url" | "$(sg_resolve jq sg_ensure_jq)" -r '.msg // empty')" || return 1 + [ -n "$url" ] || return 1 + code="$(curl -sS -o /dev/null -w '%{http_code}' -X PUT -H "Content-Type: application/json" -T "$3" "$url" 2>/dev/null)" || true + case "$(sg_norm_code "$code")" in 2*) return 0 ;; *) return 1 ;; esac +} + # --- execution preset (org workflow defaults) ------------------------------ # Settings -> Runner groups -> Execution presets is stored as the org's # Settings.workflowDefaults: RunnerConstraints plus a TerraformConfig each for diff --git a/scripts/migrate.sh b/scripts/migrate.sh index b9e8c9b..63a8fa0 100755 --- a/scripts/migrate.sh +++ b/scripts/migrate.sh @@ -527,6 +527,51 @@ sgcli_bulk() { return "${PIPESTATUS[0]}" } +# import_bulk <group> <file> <out> — import one payload file: sg-cli for the +# workflows that have Terraform variables, a direct API POST for the ones +# without — sg-cli drops an empty iacInputData.data and the API then rejects +# the workflow. TODO(sg-cli): workaround; remove the split once sg-cli ships +# with sg-sdk-go >= v1.5.7 (see sg_create_workflow). The direct path writes +# the same "Failed to create <name>: <code>: <body>" lines sg-cli prints, so +# do_import parses both alike. Like sg-cli, exits 0 even when individual +# workflows were rejected — do_import reads those from <out>; non-zero only +# when a call itself could not be made. +import_bulk() { + local grp="$1" file="$2" out="$3" rc=0 n_direct with_vars entry name err state + : >"$out" + n_direct="$("$JQ_BIN" '[.[] | select((.VCSConfig.iacInputData.data // {}) | length == 0)] | length' "$file")" + if [ "$n_direct" -eq 0 ]; then + sg_retry "$RETRIES" "$RETRY_BASE" -- sgcli_bulk "$grp" "$file" "$out" + return + fi + with_vars="$(mktemp)" + "$JQ_BIN" 'map(select((.VCSConfig.iacInputData.data // {}) | length > 0))' "$file" >"$with_vars" + if [ "$("$JQ_BIN" length "$with_vars")" -gt 0 ]; then + sg_retry "$RETRIES" "$RETRY_BASE" -- sgcli_bulk "$grp" "$with_vars" "$out" || rc=1 + fi + rm -f "$with_vars" + sg_log "$n_direct workflow(s) have no Terraform variables — creating them via the API directly (sg-cli drops an empty iacInputData.data)" + while IFS= read -r entry; do + name="$("$JQ_BIN" -r '.ResourceName' <<<"$entry")" + if err="$(SG_NO_RETRY_RC=22 sg_retry "$RETRIES" "$RETRY_BASE" -- sg_create_workflow "$grp" "$entry" 2>/dev/null)"; then + state="$("$JQ_BIN" -r '.CLIConfiguration.TfStateFilePath // empty' <<<"$entry")" + if [ -z "$state" ]; then + sg_log " $name: created (no state file to upload)" + elif [ ! -f "$state" ]; then + sg_warn " $name: created, but the state file is missing: $(sg_rel "$state")" + elif sg_upload_tfstate "$grp" "$name" "$state"; then + sg_log " $name: created, state file uploaded" + else + sg_warn " $name: created, but the state file upload failed ($(sg_rel "$state"))" + fi + else + # Same shape as sg-cli's failure line (parsed by do_import). + printf 'Failed to create %s: %s\n' "$name" "$(tail -n1 <<<"$err")" | tee -a "$out" + fi + done < <("$JQ_BIN" -c '.[] | select((.VCSConfig.iacInputData.data // {}) | length == 0)' "$file") + return "$rc" +} + # Regex for the API's rejection of a Terraform version above SG's managed # ceiling. SG bundles managed runtimes only up to the last MPL-licensed (FOSS) # Terraform release; newer versions are BSL and are not shipped. @@ -557,7 +602,7 @@ do_import() { fi sg_log "importing $(basename "$f") -> $grp" out="$(mktemp)" - sg_retry "$RETRIES" "$RETRY_BASE" -- sgcli_bulk "$grp" "$work" "$out" || rc=1 + import_bulk "$grp" "$work" "$out" || rc=1 all_names="$("$JQ_BIN" -c '[.[].ResourceName]' "$work")" while IFS= read -r line; do @@ -593,7 +638,7 @@ do_import() { die "could not patch $(basename "$f") for the Terraform version fallback" } out="$(mktemp)" - sg_retry "$RETRIES" "$RETRY_BASE" -- sgcli_bulk "$grp" "$tmp" "$out" || rc=1 + import_bulk "$grp" "$tmp" "$out" || rc=1 for name in "${fb[@]}"; do if grep -q "Failed to create $name:" "$out"; then failed+=("$name") From 1db978ee85f27c2516df22e8fd234014c4159f15 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adis=20Halilovi=C4=87?= <adis.halilovic@stackguardian.io> Date: Mon, 14 Sep 2026 11:12:10 +0200 Subject: [PATCH 33/71] fix: upload Terraform state ourselves and treat a missing state as a failed import sg-cli's state upload sends a PUT without x-ms-blob-type, which Azure Blob rejects (400 MissingRequiredHeader), and it only recognises a literal "HTTP/1.1 200 OK" as success, so on Azure-backed environments every workflow was created without its state while the run reported "state: exported for every selected workspace". import_bulk now reads sg-cli's per-workflow upload lines and re-uploads every reported failure through sg_upload_tfstate (header included, any 2xx accepted); the direct-create path uploads the same way. Each workflow ends up as state_uploaded or state_failed in the import result, a workflow without its state fails the payload file and is retried on the next import, and the checklist reports the SG-side upload next to the TFC-side export. Workaround for sg-cli; TODO(sg-cli) marks what to remove once its upload sends the header and checks the status properly. --- CLAUDE.md | 4 +- README.md | 1 + scripts/lib/checklist.sh | 18 +++++-- scripts/lib/sg_api.sh | 32 +++++++++--- scripts/lib/state.sh | 11 ++-- scripts/migrate.sh | 108 ++++++++++++++++++++++++++++++--------- 6 files changed, 134 insertions(+), 40 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 0a8d2c2..506ba8d 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -24,8 +24,8 @@ The migration is a user-driven pipeline, not a single program: The five phases are wrapped by an orchestrator so users don't run them by hand: - `sg-migrate.sh` (host entrypoint, repo root) — runs `scripts/migrate.sh` **inside the Docker image** (`Dockerfile`), bind-mounting the repo at `/app` and forwarding `SG_API_TOKEN`/`SG_ORG`/`SG_BASE_URL`/`TFE_TOKEN`/`TF_TOKEN_*`. For TFC auth it prefers a long-lived `TFE_TOKEN` (env); otherwise it mounts `~/.terraform.d/credentials.tfrc.json` read-only. Runs natively instead when `--native`/`--local` is passed, `SG_NATIVE=1` is set, the command is `clean`/`completion`/help (or no command is given), or Docker is absent. Exports `SG_PROG` so `migrate.sh` shows `./sg-migrate.sh` in its usage/hints. -- `scripts/migrate.sh` — the actual orchestrator (sources `scripts/lib/*.sh`, see below). Subcommands `init|preflight|apply|enrich|convert|validate|import|triggers|checklist|all|clean|completion` (no command prints the help menu; `enrich` runs in `all` unless `--no-variable-sets`). Flags beyond the basics: `--dry-run` (import: plan only), `--fresh` (ignore run state), `--project SEG` / `--workspace NAME` (repeatable filters; apply maps `--workspace` to `-var workspacenames=[...]`, import works on a jq-filtered temp copy), `--skip-preflight`, `--no-secret-stubs`. Resolves each tool via `sg_resolve` (PATH first — the image installs them — else `sg_ensure_*` cache). `apply`/`all` first run `require_tfc_auth`, which resolves the token the tfe provider will use (`TFE_TOKEN`, `TF_TOKEN_<host>`, or the `terraform login` file for `tfHostname`) and verifies it with `GET /api/v2/account/details`, failing fast on a missing or rejected (expired) credential. Runs `convert` and `import` in parallel (`--concurrency`, default 4), raises `terraform apply -parallelism`, and retries `sg-cli` imports with backoff (`sg_retry`, `SG_RETRIES`). Import resolves each project's workflow group, checks existence via the SG API, shows a plan (`exists`/`create`), prompts (skip with `-y`), **creates the missing `tfc-<project>` groups via the API**, then imports. Imports go through `import_bulk`: workflows with Terraform variables via `sg-cli workflow create --bulk`, workflows **without** via a direct `POST .../wfs/` (`sg_create_workflow` + `sg_upload_tfstate` in `sg_api.sh`) because sg-cli drops an empty `iacInputData.data` and the API rejects the workflow — a workaround (`TODO(sg-cli)` markers) until sg-cli ships with sg-sdk-go >= v1.5.7, which sends the empty object; the direct path prints sg-cli-style `Failed to create <name>: <code>: <body>` lines so the parsing below covers both. sg-cli exits 0 even when individual workflows fail, so `do_import` parses its output: a workflow rejected as above SG's managed Terraform ceiling (1.5.7, last MPL/FOSS release) is re-imported with `SGDefaultTerraformVersion` (read from `terraform.tfvars`; an explicit `null` there means the `terraformVersion` key is deleted instead, so the org's execution preset decides — `tfvars_is_null` tells an explicit null from a missing key), the payload patched in place, and the case logged to `export/terraform-version-fallbacks.log` plus a printed notice; any other per-workflow failure fails the run. `cmd_import` reads the org's execution preset (`sg_execution_preset`, `GET /orgs/{org}/` → `Settings.workflowDefaults`) and `preset_labels` (report.sh) turns it into the `preset (1.5.7)` / `preset (private:rg)` cells of the plan and the `SG_TF_FALLBACK_LABEL` used in messages. The trigger pass skips workflows that do not exist in SG, and `sg_api_post` returns 22 on 4xx so `sg_retry` (via `SG_NO_RETRY_RC`) does not retry definitive errors. `clean` removes local artifacts (`export/`, TF state, tool cache); `clean --all` also removes config. `completion bash|zsh` prints a completion script (`cmd_completion`; commands/options come from `SG_COMMANDS`/`SG_OPTIONS`, keep them in sync with the parser and the host flags); like `clean` it always runs natively. Output is concise by default — `apply` captures terraform's init/plan output and shows only progress + the final summary (or the full log on failure, `TF_IN_AUTOMATION=1`/`-no-color`); `-v`/`--verbose` (`SG_VERBOSE`) streams everything and un-gates the convert detail lines. Colored logging via `sg_step`/`sg_log`/`sg_success`/`sg_warn`/`sg_err`/`sg_row`; paths shown relative via `sg_rel` (all auto-off when not a TTY / `NO_COLOR`). Long steps show a live progress line (`sg_run_quiet` for terraform init/apply, `run_parallel <fn> <max> <label> <items>` for convert/import/triggers — animated only when `SG_ANIMATE=1`, i.e. colors are on; plain log lines otherwise) and report their duration (`sg_fmt_secs`). Phases start with `phase_begin` — inside `all` the header is numbered (`Phase 2/5: ...`, `PHASE_TOTAL` set in `main`) and `phase_took` gives the elapsed time; `run_parallel` prints a job's output as-is when it is a single line and adds the `── <file> ──` header only for longer or failed output (always with `-v`). The import confirmation goes through `sg_confirm` (so `SG_ANSWERS_FILE` works; a declined prompt is a neutral "cancelled" line, not an error) and `import`/`all` end with `finish_line` (outcome, total time, open checklist items from `CHECKLIST_OPEN`). A Ctrl-C is trapped to clear the progress line and print a resume hint. -- `scripts/lib/` — sourced libraries (a `lib/` gitignore rule is negated for this dir): `prompt.sh` (`sg_ask`/`sg_confirm`/`sg_select`; tty or `SG_ANSWERS_FILE` for tests; `SG_NONINTERACTIVE=1`/`-y`/no TTY → defaults), `tfvars.sh` (`tfvars_get` cached hcl2json reads, `tfvars_has`/`tfvars_is_null` to tell an explicit `null` from a missing key, `tfvars_write` renders the wizard's `W_*` vars — `W_TF_SOURCE`, and `W_TF_VERSION`/`W_RUNNER_JSON` may be the literal `null`; plain strings go through `_tfvars_str`, so quotes/backslashes in names stay valid HCL), `tfc_api.sh` (`tfc_http`/`tfc_get_all` paginated, `tfc_list_{orgs,projects,workspaces}` — workspaces include `vcs_provider`/`vcs_url`/`vcs_identifier` —, `tfc_select_workspaces` (the module's name-glob / include-tags(all) / exclude-tags filter, shared by wizard and preflight), `tfc_vcs_summary`/`tfc_vcs_kind_for`/`tfc_vcs_label_for`, `tfc_token` with provider precedence, `require_tfc_auth`), `sg_api.sh` (`_sg_api` 0/22/1 contract + `SG_HTTP_CODE`, `sg_list_integrations`, `sg_integration_type`, `sg_vcs_kind_of` (connector type → `sourceConfigDestKind`), `sg_execution_preset` + `sg_preset_{desc,runner_desc,version_desc,runner_provided}` (the org's execution preset and how to describe it), `sg_*_exists`, `sg_list_workflows`, `sg_create_secret`, `sg_patch_workflow`, `wfgroup_*`), `wizard.sh` (`wizard_run`: 4 steps TFC → SG → defaults → review; the TFC step keeps the workspace list and derives, via `tfc_vcs_summary`, which VCS provider the selected workspaces use and the repositories' URL prefix — the SG step lists matching connectors first and takes the repo URL prefix from TFC (never from a previous tfvars written for another provider kind), warns on a kind mismatch, reads the org's execution preset and offers it as the runner choice; the defaults step asks whether to carry TFC versions or use the preset (`SGTerraformVersionSource`) and, for carry, which fallback (`SGDefaultTerraformVersion` or the preset = null); the review shows human-readable rows plus the `tfc-<project>` groups; `_w_select_lines` feeds menus line-by-line so names with spaces/globs survive), `preflight.sh` (`preflight_run <apply|import|all>`, once per process; ✓ lines are plain language and name the connector's role/kind, ✗/! lines name the tfvars field to fix; besides existence it checks consistency — VCS kind vs. the connector's type (fail), repo URL prefix vs. where the TFC repositories live or the kind's well-known host (warn); it also reports the org's execution preset next to the version/runner policy and warns when `carry` meets a preset that mounts a runner-provided binary), `state.sh` (`.sg/state.json`: `phases.<name>.input_sha`, `import.<seg>.{imported,failed,tf_fallback}`, `triggers.<seg>`, `secrets.<name>`; `run_phase` in migrate.sh skips unchanged phases), `report.sh` (`show_migration_summary` after apply — incl. a state-files row —, `show_import_plan` per-workflow table with columns sized to their content), `errors.sh` (`explain_api_error` regex→hint table, `SG_API_HINTS`), `checklist.sh` (`create_secret_stubs` → SG secret `tfc-<wf>-<VAR>` = `CHANGE_ME`, referenced as `${secret::<name>}` via PATCH + payload patch; `write_checklist` → `export/post-import-checklist.md`, printed to the terminal as one ✓/! status line per section plus the imported groups with UI links — the file has the details; sets `CHECKLIST_OPEN`). Parallel jobs (`run_parallel`) run in subshells, so `do_import`/`do_set_triggers` write `.import-result.<seg>.json` / `.triggers-result.<seg>.json` into the export dir and the caller merges them into state. +- `scripts/migrate.sh` — the actual orchestrator (sources `scripts/lib/*.sh`, see below). Subcommands `init|preflight|apply|enrich|convert|validate|import|triggers|checklist|all|clean|completion` (no command prints the help menu; `enrich` runs in `all` unless `--no-variable-sets`). Flags beyond the basics: `--dry-run` (import: plan only), `--fresh` (ignore run state), `--project SEG` / `--workspace NAME` (repeatable filters; apply maps `--workspace` to `-var workspacenames=[...]`, import works on a jq-filtered temp copy), `--skip-preflight`, `--no-secret-stubs`. Resolves each tool via `sg_resolve` (PATH first — the image installs them — else `sg_ensure_*` cache). `apply`/`all` first run `require_tfc_auth`, which resolves the token the tfe provider will use (`TFE_TOKEN`, `TF_TOKEN_<host>`, or the `terraform login` file for `tfHostname`) and verifies it with `GET /api/v2/account/details`, failing fast on a missing or rejected (expired) credential. Runs `convert` and `import` in parallel (`--concurrency`, default 4), raises `terraform apply -parallelism`, and retries `sg-cli` imports with backoff (`sg_retry`, `SG_RETRIES`). Import resolves each project's workflow group, checks existence via the SG API, shows a plan (`exists`/`create`), prompts (skip with `-y`), **creates the missing `tfc-<project>` groups via the API**, then imports. Imports go through `import_bulk`: workflows with Terraform variables via `sg-cli workflow create --bulk`, workflows **without** via a direct `POST .../wfs/` (`sg_create_workflow` + `sg_upload_tfstate` in `sg_api.sh`) because sg-cli drops an empty `iacInputData.data` and the API rejects the workflow — a workaround (`TODO(sg-cli)` markers) until sg-cli ships with sg-sdk-go >= v1.5.7, which sends the empty object; the direct path prints sg-cli-style `Failed to create <name>: <code>: <body>` lines so the parsing below covers both. State files are the migrator's job on both paths: `import_bulk` tracks sg-cli's per-workflow upload lines (`State file uploaded successfully` / `Failed to upload state file for <wf>`), re-uploads the failures via `upload_state` → `sg_upload_tfstate` (adds the `x-ms-blob-type: BlockBlob` header Azure Blob requires; sg-cli omits it and also treats anything but a literal `HTTP/1.1 200 OK` as failure — second `TODO(sg-cli)`), and writes `[state] uploaded|failed|none <wf>` markers into the parsed output; `do_import` turns them into `state_uploaded`/`state_failed` in the result, a workflow without its state fails the file (and `state_import_done` re-tries it next run). sg-cli exits 0 even when individual workflows fail, so `do_import` parses its output: a workflow rejected as above SG's managed Terraform ceiling (1.5.7, last MPL/FOSS release) is re-imported with `SGDefaultTerraformVersion` (read from `terraform.tfvars`; an explicit `null` there means the `terraformVersion` key is deleted instead, so the org's execution preset decides — `tfvars_is_null` tells an explicit null from a missing key), the payload patched in place, and the case logged to `export/terraform-version-fallbacks.log` plus a printed notice; any other per-workflow failure fails the run. `cmd_import` reads the org's execution preset (`sg_execution_preset`, `GET /orgs/{org}/` → `Settings.workflowDefaults`) and `preset_labels` (report.sh) turns it into the `preset (1.5.7)` / `preset (private:rg)` cells of the plan and the `SG_TF_FALLBACK_LABEL` used in messages. The trigger pass skips workflows that do not exist in SG, and `sg_api_post` returns 22 on 4xx so `sg_retry` (via `SG_NO_RETRY_RC`) does not retry definitive errors. `clean` removes local artifacts (`export/`, TF state, tool cache); `clean --all` also removes config. `completion bash|zsh` prints a completion script (`cmd_completion`; commands/options come from `SG_COMMANDS`/`SG_OPTIONS`, keep them in sync with the parser and the host flags); like `clean` it always runs natively. Output is concise by default — `apply` captures terraform's init/plan output and shows only progress + the final summary (or the full log on failure, `TF_IN_AUTOMATION=1`/`-no-color`); `-v`/`--verbose` (`SG_VERBOSE`) streams everything and un-gates the convert detail lines. Colored logging via `sg_step`/`sg_log`/`sg_success`/`sg_warn`/`sg_err`/`sg_row`; paths shown relative via `sg_rel` (all auto-off when not a TTY / `NO_COLOR`). Long steps show a live progress line (`sg_run_quiet` for terraform init/apply, `run_parallel <fn> <max> <label> <items>` for convert/import/triggers — animated only when `SG_ANIMATE=1`, i.e. colors are on; plain log lines otherwise) and report their duration (`sg_fmt_secs`). Phases start with `phase_begin` — inside `all` the header is numbered (`Phase 2/5: ...`, `PHASE_TOTAL` set in `main`) and `phase_took` gives the elapsed time; `run_parallel` prints a job's output as-is when it is a single line and adds the `── <file> ──` header only for longer or failed output (always with `-v`). The import confirmation goes through `sg_confirm` (so `SG_ANSWERS_FILE` works; a declined prompt is a neutral "cancelled" line, not an error) and `import`/`all` end with `finish_line` (outcome, total time, open checklist items from `CHECKLIST_OPEN`). A Ctrl-C is trapped to clear the progress line and print a resume hint. +- `scripts/lib/` — sourced libraries (a `lib/` gitignore rule is negated for this dir): `prompt.sh` (`sg_ask`/`sg_confirm`/`sg_select`; tty or `SG_ANSWERS_FILE` for tests; `SG_NONINTERACTIVE=1`/`-y`/no TTY → defaults), `tfvars.sh` (`tfvars_get` cached hcl2json reads, `tfvars_has`/`tfvars_is_null` to tell an explicit `null` from a missing key, `tfvars_write` renders the wizard's `W_*` vars — `W_TF_SOURCE`, and `W_TF_VERSION`/`W_RUNNER_JSON` may be the literal `null`; plain strings go through `_tfvars_str`, so quotes/backslashes in names stay valid HCL), `tfc_api.sh` (`tfc_http`/`tfc_get_all` paginated, `tfc_list_{orgs,projects,workspaces}` — workspaces include `vcs_provider`/`vcs_url`/`vcs_identifier` —, `tfc_select_workspaces` (the module's name-glob / include-tags(all) / exclude-tags filter, shared by wizard and preflight), `tfc_vcs_summary`/`tfc_vcs_kind_for`/`tfc_vcs_label_for`, `tfc_token` with provider precedence, `require_tfc_auth`), `sg_api.sh` (`_sg_api` 0/22/1 contract + `SG_HTTP_CODE`, `sg_list_integrations`, `sg_integration_type`, `sg_vcs_kind_of` (connector type → `sourceConfigDestKind`), `sg_execution_preset` + `sg_preset_{desc,runner_desc,version_desc,runner_provided}` (the org's execution preset and how to describe it), `sg_*_exists`, `sg_list_workflows`, `sg_create_secret`, `sg_patch_workflow`, `wfgroup_*`), `wizard.sh` (`wizard_run`: 4 steps TFC → SG → defaults → review; the TFC step keeps the workspace list and derives, via `tfc_vcs_summary`, which VCS provider the selected workspaces use and the repositories' URL prefix — the SG step lists matching connectors first and takes the repo URL prefix from TFC (never from a previous tfvars written for another provider kind), warns on a kind mismatch, reads the org's execution preset and offers it as the runner choice; the defaults step asks whether to carry TFC versions or use the preset (`SGTerraformVersionSource`) and, for carry, which fallback (`SGDefaultTerraformVersion` or the preset = null); the review shows human-readable rows plus the `tfc-<project>` groups; `_w_select_lines` feeds menus line-by-line so names with spaces/globs survive), `preflight.sh` (`preflight_run <apply|import|all>`, once per process; ✓ lines are plain language and name the connector's role/kind, ✗/! lines name the tfvars field to fix; besides existence it checks consistency — VCS kind vs. the connector's type (fail), repo URL prefix vs. where the TFC repositories live or the kind's well-known host (warn); it also reports the org's execution preset next to the version/runner policy and warns when `carry` meets a preset that mounts a runner-provided binary), `state.sh` (`.sg/state.json`: `phases.<name>.input_sha`, `import.<seg>.{imported,failed,tf_fallback,state_uploaded,state_failed}`, `triggers.<seg>`, `secrets.<name>`; `run_phase` in migrate.sh skips unchanged phases), `report.sh` (`show_migration_summary` after apply — incl. a state-files row —, `show_import_plan` per-workflow table with columns sized to their content), `errors.sh` (`explain_api_error` regex→hint table, `SG_API_HINTS`), `checklist.sh` (`create_secret_stubs` → SG secret `tfc-<wf>-<VAR>` = `CHANGE_ME`, referenced as `${secret::<name>}` via PATCH + payload patch; `write_checklist` → `export/post-import-checklist.md`, printed to the terminal as one ✓/! status line per section plus the imported groups with UI links — the file has the details; section 5 covers both the TFC state export and the SG state upload (`state_failed` from the import results); sets `CHECKLIST_OPEN`). Parallel jobs (`run_parallel`) run in subshells, so `do_import`/`do_set_triggers` write `.import-result.<seg>.json` / `.triggers-result.<seg>.json` into the export dir and the caller merges them into state. - `scripts/tools.sh` — sourced by the other scripts (sets `SG_REPO_ROOT` to its parent dir). Provides `sg_resolve` (PATH-or-cache), `sg_ensure_{jq,hcl2json,yajsv,sgcli}` (pinned downloads into `.sg/cached/`; `sg-cli` is the Go binary), `sg_retry`, the color vars + log helpers. The Docker image bundles these tools at build time (`yajsv` built from source so arm64 works), so in-container runs never download. - **Variable sets** — `scripts/enrich_variable_sets.sh <tfc-org> <payload...>` (run by `cmd_enrich`, which reads `tfOrg`/`tfHostname` from `terraform.tfvars` via `hcl2json` — variable sets live in the **TFC** org, not `SG_ORG`). Lists all sets + vars via the TFC API (Bearer token from the `terraform login` creds file or `TFE_TOKEN`), computes per-workspace effective vars (scope: global/project/workspace; precedence: priority sets > workspace vars > non-priority sets), and merges them into each workflow (matched by workspace name parsed from `TfStateFilePath`). Sensitive set vars and key conflicts are reported. Skipped by `--no-variable-sets`. - **Workflow groups** — each TFC project maps to an SG workflow group `tfc-<project-segment>`, created via `POST /api/v1/orgs/{org}/wfgrps/` (auth `Authorization: apikey <token>`) if missing. (Both the API calls and sg-cli honor the undocumented `SG_BASE_URL` for non-prod targets.) Groups are addressed by name, so no generated ID is tracked. `.sg/workflow-groups.json` (gitignored, optional) overrides the target group per segment (`{"<segment>": "<existing-group>"}`); override groups are not auto-created. `--no-create-groups` requires all groups to pre-exist. diff --git a/README.md b/README.md index 3c0250b..707cc46 100644 --- a/README.md +++ b/README.md @@ -169,6 +169,7 @@ To update workflows with different details, re-run the sg-cli command with the m - **Terraform version fallback (FOSS ceiling).** With `SGTerraformVersionSource = "carry"` (the default) pinned versions are carried over as-is and tried first at import, so a custom runtime image or private runner that ships that binary keeps working; workspaces set to `latest` or a version constraint use `SGDefaultTerraformVersion` at export time. StackGuardian's _managed_ runtimes only go up to **1.5.7**, the last MPL-licensed (FOSS) Terraform release; newer versions are BSL-licensed and are not bundled. When the API rejects a workflow for that reason, the importer **automatically re-imports it with `SGDefaultTerraformVersion`** (or, when that is `null`, without any version so the execution preset decides — see the next bullet), patches the payload file to match, prints a notice, and records each case in `export/terraform-version-fallbacks.log`. Those workflows run a different Terraform than they did in TFC, so check compatibility before the first run. To keep a newer version, set `workspaceOverrides[<name>].terraformVersion` to a binary path mounted from a private runner (or use a custom runtime container template) and re-import. - **Execution presets.** StackGuardian org admins can define an execution preset (Settings → Runner groups → Execution presets): default runner constraints plus a Terraform and an OpenTofu configuration that the API applies to any new workflow whose payload does not carry those fields. To lean on it, set `SGTerraformVersionSource = "preset"` (no version is sent at all) and/or `SGDefaultRunnerConstraints = null` (no runner constraints are sent); `SGDefaultTerraformVersion = null` keeps carrying TFC pins but hands the unpinned and rejected ones to the preset. `init` offers these choices with the org's current preset shown inline, preflight prints what the preset would supply, and the import plan marks such cells as `preset (…)`. The preset's custom runtime image or runner-provided binary is inherited in every mode, since the migrator never sets those keys. - **Workspaces without variables are imported directly.** sg-cli (up to v2.2.1) drops an empty `iacInputData.data` from the request, and the API then rejects the workflow with `VCSConfig.iacInputData.data: This field is required.` The importer therefore creates workflows that have no Terraform variables straight through the API (same create, same state upload) and uses sg-cli for the rest. This is a workaround until sg-cli picks up sg-sdk-go v1.5.7, which fixes the dropped key. +- **Terraform state is uploaded by the migrator, and checked.** sg-cli's own upload sends a PUT that Azure-backed StackGuardian environments reject (missing `x-ms-blob-type`) and it reports success only on a literal `HTTP/1.1 200 OK`, so the importer re-uploads every state file sg-cli reports as failed and records per workflow whether the store accepted it. A workflow without its state counts as a failed import: it is listed in the checklist (`state: N workflow(s) are in SG without their state`) and the file is retried on the next `import`. - **State export is idempotent.** Re-running `terraform apply` only pulls state for workspaces not yet exported. Set `forceStateRefresh = true` to re-pull everything. Workspaces not using `remote` execution may export incomplete state — see the summary. - **Workflow naming.** `ResourceName` mirrors the TFC workspace name, sanitized to StackGuardian's rules (1-100 chars, `[-a-zA-Z0-9_]`); any rename is listed in the summary and the checklist. - **Preflight.** `apply`, `import` and `all` first verify the TFC and SG tokens, the TFC org and workspace selection, and that every connector, secret and runner group referenced in `terraform.tfvars` exists. Fix what it reports (or re-run `init`); `--skip-preflight` bypasses it. diff --git a/scripts/lib/checklist.sh b/scripts/lib/checklist.sh index 2bfa9e1..9b9e1b0 100644 --- a/scripts/lib/checklist.sh +++ b/scripts/lib/checklist.sh @@ -108,8 +108,8 @@ _cl_block() { if [ -n "$1" ]; then printf '%s\n\n' "$1"; else printf -- '- None. # links; the terminal gets one status line per section). Sets CHECKLIST_OPEN. write_checklist() { local out="$EXPORT_DIR/post-import-checklist.md" summary="$EXPORT_DIR/migration-summary.json" st - local i_secrets i_unstubbed="" i_failed i_fallback="" i_unpinned="" i_preset="" i_trig i_state="" i_nonremote="" i_renamed="" - local n_secrets n_unstubbed n_failed n_fallback n_unpinned n_preset n_trig n_state n_nonremote n_renamed + local i_secrets i_unstubbed="" i_failed i_fallback="" i_unpinned="" i_preset="" i_trig i_state="" i_stfail i_nonremote="" i_renamed="" + local n_secrets n_unstubbed n_failed n_fallback n_unpinned n_preset n_trig n_state n_stfail n_stok n_nonremote n_renamed local f seg grp n total=0 gw preset_desc="" local -a glines=() st="$(state_read)" @@ -133,9 +133,12 @@ write_checklist() { [ -s "$EXPORT_DIR/terraform-version-fallbacks.log" ] && i_fallback="$(sed 's/^/- [ ] /' "$EXPORT_DIR/terraform-version-fallbacks.log")" i_trig="$(printf '%s' "$st" | "$JQ_BIN" -r '.triggers // {} | to_entries[] | .value.group as $g | (.value.failed[]? | "- [ ] `\($g)/\(.)` — trigger registration failed; check the connector has admin/webhook rights on the repository, then re-run `./sg-migrate.sh triggers`"), (.value.missing[]? | "- [ ] `\($g)/\(.)` — workflow was not imported, so no trigger was registered")')" [ -s "$EXPORT_DIR/state-export-failures.log" ] && i_state="$(sed 's/^/- [ ] /' "$EXPORT_DIR/state-export-failures.log")" + # State that was exported but did not land in SG (upload failed on import). + i_stfail="$(printf '%s' "$st" | "$JQ_BIN" -r --arg ui "$SG_UI_URL" --arg org "$ORG" '.import // {} | to_entries[] | .value.group as $g | .value.state_failed[]? | "- [ ] [`\($g)/\(.)`](\($ui)/orchestrator/orgs/\($org)/wfgrps/\($g)/wfs/\(.)) — created without its Terraform state; re-run `./sg-migrate.sh import` or upload `export/states/<workspace>.tfstate` by hand (Workflow → Settings → State)"')" + n_stok="$(printf '%s' "$st" | "$JQ_BIN" -r '[.import // {} | .[] | .state_uploaded[]?] | length')" n_secrets="$(_cl_count "$i_secrets")"; n_unstubbed="$(_cl_count "$i_unstubbed")"; n_failed="$(_cl_count "$i_failed")" n_fallback="$(_cl_count "$i_fallback")"; n_unpinned="$(_cl_count "$i_unpinned")"; n_preset="$(_cl_count "$i_preset")"; n_trig="$(_cl_count "$i_trig")" - n_state="$(_cl_count "$i_state")"; n_nonremote="$(_cl_count "$i_nonremote")"; n_renamed="$(_cl_count "$i_renamed")" + n_state="$(_cl_count "$i_state")"; n_stfail="$(_cl_count "$i_stfail")"; n_nonremote="$(_cl_count "$i_nonremote")"; n_renamed="$(_cl_count "$i_renamed")" # --- the file ----------------------------------------------------------------- { @@ -158,6 +161,11 @@ write_checklist() { else printf -- '- All selected workspaces had their state exported.\n\n' fi + if [ -n "$i_stfail" ]; then + printf 'These workflows exist in StackGuardian but their state upload failed, so a run would start from an empty state:\n\n%s\n\n' "$i_stfail" + else + printf -- '- State uploaded to StackGuardian for %s workflow(s).\n\n' "$n_stok" + fi [ -n "$i_nonremote" ] && printf '%s\n\n' "$i_nonremote" if [ -n "$i_renamed" ]; then printf '## 6. Renamed workflows\n\nThese TFC workspace names were not valid StackGuardian workflow names and were adjusted:\n\n%s\n\n' "$i_renamed" @@ -191,9 +199,11 @@ write_checklist() { _cl_status "$n_trig" "VCS triggers: registered for every workflow that had them" "VCS triggers: $n_trig workflow(s) without triggers — see the checklist, then '$PROG triggers'" _cl_status "$((n_state + n_nonremote))" "state: exported for every selected workspace" \ "state: $n_state workspace(s) without exported state${i_nonremote:+, $n_nonremote with non-remote execution} — upload by hand" + _cl_status "$n_stfail" "state: uploaded to SG for $n_stok workflow(s)" \ + "state: $n_stfail workflow(s) are in SG without their state (upload failed) — re-run '$PROG import' or upload by hand" [ "$n_renamed" -gt 0 ] && _cl_status "$n_renamed" "" "names: $n_renamed workflow(s) were renamed to valid SG names" # shellcheck disable=SC2034 - CHECKLIST_OPEN=$((n_secrets + n_unstubbed + n_failed + n_fallback + n_unpinned + n_preset + n_trig + n_state + n_nonremote)) + CHECKLIST_OPEN=$((n_secrets + n_unstubbed + n_failed + n_fallback + n_unpinned + n_preset + n_trig + n_state + n_stfail + n_nonremote)) sg_dim "full checklist with links: $(sg_rel "$out")" } diff --git a/scripts/lib/sg_api.sh b/scripts/lib/sg_api.sh index cb16c60..3b54e2c 100644 --- a/scripts/lib/sg_api.sh +++ b/scripts/lib/sg_api.sh @@ -127,14 +127,32 @@ sg_create_workflow() { return "$rc" } -# sg_upload_tfstate <group> <wf> <file> — the state upload sg-cli does after a -# bulk create: fetch the presigned URL, PUT the file. 0 on success. +# sg_upload_tfstate <group> <wf> <file> — upload a workflow's Terraform state: +# fetch the presigned URL, PUT the file. 0 on a 2xx from the store; otherwise +# prints a one-line reason (for the log) and returns 1. The store behind the +# URL differs per environment: Azure Blob requires x-ms-blob-type on every PUT +# (S3/GCS ignore it) and answers 201, not 200. +# TODO(sg-cli): sg-cli's own upload (uploadTfState) sends no x-ms-blob-type and +# only accepts a literal "HTTP/1.1 200 OK", so the migrator re-uploads whatever +# sg-cli reports as failed (import_bulk) — drop that once sg-cli is fixed. sg_upload_tfstate() { - local url code - url="$(sg_api_get "$(wf_url "$1" "$2")tfstate_upload_url" | "$(sg_resolve jq sg_ensure_jq)" -r '.msg // empty')" || return 1 - [ -n "$url" ] || return 1 - code="$(curl -sS -o /dev/null -w '%{http_code}' -X PUT -H "Content-Type: application/json" -T "$3" "$url" 2>/dev/null)" || true - case "$(sg_norm_code "$code")" in 2*) return 0 ;; *) return 1 ;; esac + local url code body + body="$(sg_api_get "$(wf_url "$1" "$2")tfstate_upload_url" 2>/dev/null)" || { + printf 'no upload URL (HTTP %s)' "$SG_HTTP_CODE" + return 1 + } + url="$(printf '%s' "$body" | "$(sg_resolve jq sg_ensure_jq)" -r '.msg // empty' 2>/dev/null)" + [ -n "$url" ] || { + printf 'no upload URL in the API response' + return 1 + } + code="$(curl -sS -o /dev/null -w '%{http_code}' -X PUT \ + -H "Content-Type: application/json" -H "x-ms-blob-type: BlockBlob" \ + -T "$3" "$url" 2>/dev/null)" || true + code="$(sg_norm_code "$code")" + case "$code" in 2*) return 0 ;; esac + printf 'store answered HTTP %s' "$code" + return 1 } # --- execution preset (org workflow defaults) ------------------------------ diff --git a/scripts/lib/state.sh b/scripts/lib/state.sh index c69040d..567aabc 100644 --- a/scripts/lib/state.sh +++ b/scripts/lib/state.sh @@ -9,7 +9,9 @@ # # Layout: { "phases": { "<phase>": {"at": iso, "input_sha": sha} }, # "import": { "<seg>": {"at": iso, "payload_sha": sha, "group": g, -# "imported": [...], "failed": [...]} } } +# "imported": [...], "failed": [...], +# "tf_fallback": [...], +# "state_uploaded": [...], "state_failed": [...]} } } STATE_FILE="${SG_STATE_FILE:-$SG_REPO_ROOT/.sg/state.json}" @@ -52,14 +54,15 @@ state_phase_done() { state_mark_phase() { state_update '.phases[$p] = {at: $at, input_sha: $s}' --arg p "$1" --arg s "$2" --arg at "$(state_now)"; } # state_import_done <seg> <payload-sha> — exit 0 when this payload file was -# fully imported (no failures) with exactly this content. +# fully imported (no failed workflows, no missing state) with exactly this content. state_import_done() { state_read | "$(sg_resolve jq sg_ensure_jq)" -e --arg s "$1" --arg sha "$2" \ - '.import[$s] | select(.payload_sha == $sha and ((.failed // []) | length) == 0)' >/dev/null 2>&1 + '.import[$s] | select(.payload_sha == $sha and ((.failed // []) | length) == 0 and ((.state_failed // []) | length) == 0)' >/dev/null 2>&1 } # state_record_import <seg> <result-json> — merge a do_import result -# ({group, payload_sha, imported, failed}) for a payload file. +# ({group, payload_sha, imported, failed, tf_fallback, state_uploaded, +# state_failed}) for a payload file. state_record_import() { state_update '.import[$s] = ($r + {at: $at})' --arg s "$1" --argjson r "$2" --arg at "$(state_now)"; } state_reset() { rm -f "$STATE_FILE"; } diff --git a/scripts/migrate.sh b/scripts/migrate.sh index 63a8fa0..58cce61 100755 --- a/scripts/migrate.sh +++ b/scripts/migrate.sh @@ -527,43 +527,92 @@ sgcli_bulk() { return "${PIPESTATUS[0]}" } +# state_path_of <file> <wf> — the payload entry's TfStateFilePath ("" if none). +state_path_of() { "$JQ_BIN" -r --arg n "$2" '.[] | select(.ResourceName == $n) | .CLIConfiguration.TfStateFilePath // empty' "$1"; } + +# upload_state <group> <wf> <file> <out> — upload the workflow's state file +# ourselves and append a "[state] uploaded|failed|none <wf>[: why]" marker to +# <out> for do_import. Returns 1 only when the workflow has a state file that +# did not land. +upload_state() { + local grp="$1" name="$2" file="$3" out="$4" path why + path="$(state_path_of "$file" "$name")" + if [ -z "$path" ]; then + printf '[state] none %s\n' "$name" >>"$out" + return 0 + fi + if [ ! -f "$path" ]; then + sg_warn " $name: state file missing: $(sg_rel "$path")" + printf '[state] failed %s: state file missing (%s)\n' "$name" "$path" >>"$out" + return 1 + fi + if why="$(sg_upload_tfstate "$grp" "$name" "$path")"; then + sg_log " $name: state file uploaded" + printf '[state] uploaded %s\n' "$name" >>"$out" + return 0 + fi + sg_warn " $name: state file upload failed — $why" + printf '[state] failed %s: %s\n' "$name" "$why" >>"$out" + return 1 +} + # import_bulk <group> <file> <out> — import one payload file: sg-cli for the # workflows that have Terraform variables, a direct API POST for the ones # without — sg-cli drops an empty iacInputData.data and the API then rejects # the workflow. TODO(sg-cli): workaround; remove the split once sg-cli ships # with sg-sdk-go >= v1.5.7 (see sg_create_workflow). The direct path writes # the same "Failed to create <name>: <code>: <body>" lines sg-cli prints, so -# do_import parses both alike. Like sg-cli, exits 0 even when individual -# workflows were rejected — do_import reads those from <out>; non-zero only -# when a call itself could not be made. +# do_import parses both alike. +# +# State files are the migrator's responsibility on both paths: whatever +# sg-cli reports as a failed upload is uploaded again by upload_state, and +# every workflow ends up with a "[state] ..." marker in <out>. +# +# Like sg-cli, exits 0 even when individual workflows were rejected — +# do_import reads those from <out>; non-zero only when a call itself could +# not be made. import_bulk() { - local grp="$1" file="$2" out="$3" rc=0 n_direct with_vars entry name err state + local grp="$1" file="$2" out="$3" rc=0 n_direct with_vars entry name err line cur="" cli_out + local -a redo=() : >"$out" n_direct="$("$JQ_BIN" '[.[] | select((.VCSConfig.iacInputData.data // {}) | length == 0)] | length' "$file")" - if [ "$n_direct" -eq 0 ]; then - sg_retry "$RETRIES" "$RETRY_BASE" -- sgcli_bulk "$grp" "$file" "$out" - return + with_vars="$file" + if [ "$n_direct" -gt 0 ]; then + with_vars="$(mktemp)" + "$JQ_BIN" 'map(select((.VCSConfig.iacInputData.data // {}) | length > 0))' "$file" >"$with_vars" fi - with_vars="$(mktemp)" - "$JQ_BIN" 'map(select((.VCSConfig.iacInputData.data // {}) | length > 0))' "$file" >"$with_vars" if [ "$("$JQ_BIN" length "$with_vars")" -gt 0 ]; then - sg_retry "$RETRIES" "$RETRY_BASE" -- sgcli_bulk "$grp" "$with_vars" "$out" || rc=1 + cli_out="$(mktemp)" + sg_retry "$RETRIES" "$RETRY_BASE" -- sgcli_bulk "$grp" "$with_vars" "$cli_out" || rc=1 + cat "$cli_out" >>"$out" + # sg-cli's own state upload, per workflow: trust its success, redo its + # failures (it sends no x-ms-blob-type, and misreads anything but a + # literal "HTTP/1.1 200 OK" as a failure). + while IFS= read -r line; do + case "$line" in + *"Processing workflow: "*) cur="${line##*Processing workflow: }" ;; + *"Failed to create "*) cur="" ;; + *"State file uploaded successfully"*) [ -n "$cur" ] && printf '[state] uploaded %s\n' "$cur" >>"$out" ;; + *"Failed to upload state file for "*) name="${line##*Failed to upload state file for }"; redo+=("${name%%:*}") ;; + *"cannot access state file"*) [ -n "$cur" ] && redo+=("$cur") ;; + *"TfStateFilePath not provided for "*) name="${line##*TfStateFilePath not provided for }"; printf '[state] none %s\n' "${name%%:*}" >>"$out" ;; + esac + done <"$cli_out" + rm -f "$cli_out" + if [ "${#redo[@]}" -gt 0 ]; then + sg_log "${#redo[@]} state upload(s) reported failed by sg-cli — uploading them directly" + for name in "${redo[@]}"; do upload_state "$grp" "$name" "$file" "$out" || true; done + fi fi - rm -f "$with_vars" + [ "$with_vars" != "$file" ] && rm -f "$with_vars" + [ "$n_direct" -gt 0 ] || return "$rc" + sg_log "$n_direct workflow(s) have no Terraform variables — creating them via the API directly (sg-cli drops an empty iacInputData.data)" while IFS= read -r entry; do name="$("$JQ_BIN" -r '.ResourceName' <<<"$entry")" if err="$(SG_NO_RETRY_RC=22 sg_retry "$RETRIES" "$RETRY_BASE" -- sg_create_workflow "$grp" "$entry" 2>/dev/null)"; then - state="$("$JQ_BIN" -r '.CLIConfiguration.TfStateFilePath // empty' <<<"$entry")" - if [ -z "$state" ]; then - sg_log " $name: created (no state file to upload)" - elif [ ! -f "$state" ]; then - sg_warn " $name: created, but the state file is missing: $(sg_rel "$state")" - elif sg_upload_tfstate "$grp" "$name" "$state"; then - sg_log " $name: created, state file uploaded" - else - sg_warn " $name: created, but the state file upload failed ($(sg_rel "$state"))" - fi + sg_log " $name: created" + upload_state "$grp" "$name" "$file" "$out" || true else # Same shape as sg-cli's failure line (parsed by do_import). printf 'Failed to create %s: %s\n' "$name" "$(tail -n1 <<<"$err")" | tee -a "$out" @@ -586,7 +635,7 @@ names_json() { printf '%s\n' "$@" | "$JQ_BIN" -R . | "$JQ_BIN" -s .; } # the trigger pass see what was actually imported); each fallback is appended to # terraform-version-fallbacks.log. Any other per-workflow failure fails the file. do_import() { - local f="$1" seg grp out rc=0 ceiling="" failed=() fb=() name line tmp names work all_names patch + local f="$1" seg grp out rc=0 ceiling="" failed=() fb=() st_ok=() st_failed=() name line tmp names work all_names patch seg="$(seg_of "$f")" grp="$(group_for "$seg")" # With --workspace, import only the selected workflows (a filtered copy). @@ -614,6 +663,10 @@ do_import() { explain_api_error "${BASH_REMATCH[2]}" elif [[ "$line" =~ Failed\ to\ create\ ([^:]+): ]]; then failed+=("${BASH_REMATCH[1]}") + elif [[ "$line" =~ ^\[state\]\ uploaded\ (.+)$ ]]; then + st_ok+=("${BASH_REMATCH[1]}") + elif [[ "$line" =~ ^\[state\]\ failed\ ([^:]+): ]]; then + st_failed+=("${BASH_REMATCH[1]}") fi done <"$out" rm -f "$out" @@ -645,6 +698,8 @@ do_import() { else line="$("$JQ_BIN" -r --arg n "$name" '.[] | select(.ResourceName == $n) | .TerraformConfig.terraformVersion' "$f")" printf '%s/%s: %s -> %s (above SG managed ceiling %s)\n' "$grp" "$name" "$line" "${SG_DEFAULT_TF_VERSION:-execution preset}" "$ceiling" >>"$EXPORT_DIR/terraform-version-fallbacks.log" + grep -q "^\[state\] uploaded $name\$" "$out" && st_ok+=("$name") + grep -q "^\[state\] failed $name:" "$out" && st_failed+=("$name") fi done rm -f "$out" @@ -659,13 +714,20 @@ do_import() { "$JQ_BIN" -nc --arg g "$grp" --arg sha "$(sg_sha_files "$f")" --argjson all "$all_names" \ --argjson failed "$([ "${#failed[@]}" -gt 0 ] && names_json "${failed[@]}" || echo '[]')" \ --argjson fallback "$([ "${#fb[@]}" -gt 0 ] && names_json "${fb[@]}" || echo '[]')" \ - '{group: $g, payload_sha: $sha, imported: ($all - $failed), failed: $failed, tf_fallback: ($fallback - $failed)}' \ + --argjson st_ok "$([ "${#st_ok[@]}" -gt 0 ] && names_json "${st_ok[@]}" || echo '[]')" \ + --argjson st_failed "$([ "${#st_failed[@]}" -gt 0 ] && names_json "${st_failed[@]}" || echo '[]')" \ + '{group: $g, payload_sha: $sha, imported: ($all - $failed), failed: $failed, tf_fallback: ($fallback - $failed), + state_uploaded: ($st_ok - $failed - $st_failed | unique), state_failed: ($st_failed - $failed | unique)}' \ >"$EXPORT_DIR/.import-result.$seg.json" if [ "${#failed[@]}" -gt 0 ]; then sg_err "$(basename "$f"): ${#failed[@]} workflow(s) failed to import: ${failed[*]}" return 1 fi + if [ "${#st_failed[@]}" -gt 0 ]; then + sg_err "$(basename "$f"): ${#st_failed[@]} workflow(s) created but without their Terraform state in SG: ${st_failed[*]}" + return 1 + fi return "$rc" } From 927f546891a4259edfd92c0d31bfcb0d1731544c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adis=20Halilovi=C4=87?= <adis.halilovic@stackguardian.io> Date: Mon, 14 Sep 2026 11:12:11 +0200 Subject: [PATCH 34/71] feat: import one workflow first and stop if it or its state upload fails With more than one workflow to import, probe_import imports the first selected workflow of the first payload file on its own and requires both the create and the state upload to succeed before the rest is imported in parallel. An environment problem (a store that rejects uploads, a read-only token, a wrong connector kind) then costs one workflow instead of all of them; the probe workflow is updated again with its file afterwards. --- CLAUDE.md | 2 +- README.md | 1 + scripts/migrate.sh | 34 ++++++++++++++++++++++++++++++++++ 3 files changed, 36 insertions(+), 1 deletion(-) diff --git a/CLAUDE.md b/CLAUDE.md index 506ba8d..fc602d9 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -24,7 +24,7 @@ The migration is a user-driven pipeline, not a single program: The five phases are wrapped by an orchestrator so users don't run them by hand: - `sg-migrate.sh` (host entrypoint, repo root) — runs `scripts/migrate.sh` **inside the Docker image** (`Dockerfile`), bind-mounting the repo at `/app` and forwarding `SG_API_TOKEN`/`SG_ORG`/`SG_BASE_URL`/`TFE_TOKEN`/`TF_TOKEN_*`. For TFC auth it prefers a long-lived `TFE_TOKEN` (env); otherwise it mounts `~/.terraform.d/credentials.tfrc.json` read-only. Runs natively instead when `--native`/`--local` is passed, `SG_NATIVE=1` is set, the command is `clean`/`completion`/help (or no command is given), or Docker is absent. Exports `SG_PROG` so `migrate.sh` shows `./sg-migrate.sh` in its usage/hints. -- `scripts/migrate.sh` — the actual orchestrator (sources `scripts/lib/*.sh`, see below). Subcommands `init|preflight|apply|enrich|convert|validate|import|triggers|checklist|all|clean|completion` (no command prints the help menu; `enrich` runs in `all` unless `--no-variable-sets`). Flags beyond the basics: `--dry-run` (import: plan only), `--fresh` (ignore run state), `--project SEG` / `--workspace NAME` (repeatable filters; apply maps `--workspace` to `-var workspacenames=[...]`, import works on a jq-filtered temp copy), `--skip-preflight`, `--no-secret-stubs`. Resolves each tool via `sg_resolve` (PATH first — the image installs them — else `sg_ensure_*` cache). `apply`/`all` first run `require_tfc_auth`, which resolves the token the tfe provider will use (`TFE_TOKEN`, `TF_TOKEN_<host>`, or the `terraform login` file for `tfHostname`) and verifies it with `GET /api/v2/account/details`, failing fast on a missing or rejected (expired) credential. Runs `convert` and `import` in parallel (`--concurrency`, default 4), raises `terraform apply -parallelism`, and retries `sg-cli` imports with backoff (`sg_retry`, `SG_RETRIES`). Import resolves each project's workflow group, checks existence via the SG API, shows a plan (`exists`/`create`), prompts (skip with `-y`), **creates the missing `tfc-<project>` groups via the API**, then imports. Imports go through `import_bulk`: workflows with Terraform variables via `sg-cli workflow create --bulk`, workflows **without** via a direct `POST .../wfs/` (`sg_create_workflow` + `sg_upload_tfstate` in `sg_api.sh`) because sg-cli drops an empty `iacInputData.data` and the API rejects the workflow — a workaround (`TODO(sg-cli)` markers) until sg-cli ships with sg-sdk-go >= v1.5.7, which sends the empty object; the direct path prints sg-cli-style `Failed to create <name>: <code>: <body>` lines so the parsing below covers both. State files are the migrator's job on both paths: `import_bulk` tracks sg-cli's per-workflow upload lines (`State file uploaded successfully` / `Failed to upload state file for <wf>`), re-uploads the failures via `upload_state` → `sg_upload_tfstate` (adds the `x-ms-blob-type: BlockBlob` header Azure Blob requires; sg-cli omits it and also treats anything but a literal `HTTP/1.1 200 OK` as failure — second `TODO(sg-cli)`), and writes `[state] uploaded|failed|none <wf>` markers into the parsed output; `do_import` turns them into `state_uploaded`/`state_failed` in the result, a workflow without its state fails the file (and `state_import_done` re-tries it next run). sg-cli exits 0 even when individual workflows fail, so `do_import` parses its output: a workflow rejected as above SG's managed Terraform ceiling (1.5.7, last MPL/FOSS release) is re-imported with `SGDefaultTerraformVersion` (read from `terraform.tfvars`; an explicit `null` there means the `terraformVersion` key is deleted instead, so the org's execution preset decides — `tfvars_is_null` tells an explicit null from a missing key), the payload patched in place, and the case logged to `export/terraform-version-fallbacks.log` plus a printed notice; any other per-workflow failure fails the run. `cmd_import` reads the org's execution preset (`sg_execution_preset`, `GET /orgs/{org}/` → `Settings.workflowDefaults`) and `preset_labels` (report.sh) turns it into the `preset (1.5.7)` / `preset (private:rg)` cells of the plan and the `SG_TF_FALLBACK_LABEL` used in messages. The trigger pass skips workflows that do not exist in SG, and `sg_api_post` returns 22 on 4xx so `sg_retry` (via `SG_NO_RETRY_RC`) does not retry definitive errors. `clean` removes local artifacts (`export/`, TF state, tool cache); `clean --all` also removes config. `completion bash|zsh` prints a completion script (`cmd_completion`; commands/options come from `SG_COMMANDS`/`SG_OPTIONS`, keep them in sync with the parser and the host flags); like `clean` it always runs natively. Output is concise by default — `apply` captures terraform's init/plan output and shows only progress + the final summary (or the full log on failure, `TF_IN_AUTOMATION=1`/`-no-color`); `-v`/`--verbose` (`SG_VERBOSE`) streams everything and un-gates the convert detail lines. Colored logging via `sg_step`/`sg_log`/`sg_success`/`sg_warn`/`sg_err`/`sg_row`; paths shown relative via `sg_rel` (all auto-off when not a TTY / `NO_COLOR`). Long steps show a live progress line (`sg_run_quiet` for terraform init/apply, `run_parallel <fn> <max> <label> <items>` for convert/import/triggers — animated only when `SG_ANIMATE=1`, i.e. colors are on; plain log lines otherwise) and report their duration (`sg_fmt_secs`). Phases start with `phase_begin` — inside `all` the header is numbered (`Phase 2/5: ...`, `PHASE_TOTAL` set in `main`) and `phase_took` gives the elapsed time; `run_parallel` prints a job's output as-is when it is a single line and adds the `── <file> ──` header only for longer or failed output (always with `-v`). The import confirmation goes through `sg_confirm` (so `SG_ANSWERS_FILE` works; a declined prompt is a neutral "cancelled" line, not an error) and `import`/`all` end with `finish_line` (outcome, total time, open checklist items from `CHECKLIST_OPEN`). A Ctrl-C is trapped to clear the progress line and print a resume hint. +- `scripts/migrate.sh` — the actual orchestrator (sources `scripts/lib/*.sh`, see below). Subcommands `init|preflight|apply|enrich|convert|validate|import|triggers|checklist|all|clean|completion` (no command prints the help menu; `enrich` runs in `all` unless `--no-variable-sets`). Flags beyond the basics: `--dry-run` (import: plan only), `--fresh` (ignore run state), `--project SEG` / `--workspace NAME` (repeatable filters; apply maps `--workspace` to `-var workspacenames=[...]`, import works on a jq-filtered temp copy), `--skip-preflight`, `--no-secret-stubs`. Resolves each tool via `sg_resolve` (PATH first — the image installs them — else `sg_ensure_*` cache). `apply`/`all` first run `require_tfc_auth`, which resolves the token the tfe provider will use (`TFE_TOKEN`, `TF_TOKEN_<host>`, or the `terraform login` file for `tfHostname`) and verifies it with `GET /api/v2/account/details`, failing fast on a missing or rejected (expired) credential. Runs `convert` and `import` in parallel (`--concurrency`, default 4), raises `terraform apply -parallelism`, and retries `sg-cli` imports with backoff (`sg_retry`, `SG_RETRIES`). Import resolves each project's workflow group, checks existence via the SG API, shows a plan (`exists`/`create`), prompts (skip with `-y`), **creates the missing `tfc-<project>` groups via the API**, then imports. Imports go through `import_bulk`: workflows with Terraform variables via `sg-cli workflow create --bulk`, workflows **without** via a direct `POST .../wfs/` (`sg_create_workflow` + `sg_upload_tfstate` in `sg_api.sh`) because sg-cli drops an empty `iacInputData.data` and the API rejects the workflow — a workaround (`TODO(sg-cli)` markers) until sg-cli ships with sg-sdk-go >= v1.5.7, which sends the empty object; the direct path prints sg-cli-style `Failed to create <name>: <code>: <body>` lines so the parsing below covers both. State files are the migrator's job on both paths: `import_bulk` tracks sg-cli's per-workflow upload lines (`State file uploaded successfully` / `Failed to upload state file for <wf>`), re-uploads the failures via `upload_state` → `sg_upload_tfstate` (adds the `x-ms-blob-type: BlockBlob` header Azure Blob requires; sg-cli omits it and also treats anything but a literal `HTTP/1.1 200 OK` as failure — second `TODO(sg-cli)`), and writes `[state] uploaded|failed|none <wf>` markers into the parsed output; `do_import` turns them into `state_uploaded`/`state_failed` in the result, a workflow without its state fails the file (and `state_import_done` re-tries it next run). Before the parallel import, `probe_import` imports one workflow alone (first selected of the first file) and dies unless the create and the state upload both succeeded — the fail-fast for environment problems. sg-cli exits 0 even when individual workflows fail, so `do_import` parses its output: a workflow rejected as above SG's managed Terraform ceiling (1.5.7, last MPL/FOSS release) is re-imported with `SGDefaultTerraformVersion` (read from `terraform.tfvars`; an explicit `null` there means the `terraformVersion` key is deleted instead, so the org's execution preset decides — `tfvars_is_null` tells an explicit null from a missing key), the payload patched in place, and the case logged to `export/terraform-version-fallbacks.log` plus a printed notice; any other per-workflow failure fails the run. `cmd_import` reads the org's execution preset (`sg_execution_preset`, `GET /orgs/{org}/` → `Settings.workflowDefaults`) and `preset_labels` (report.sh) turns it into the `preset (1.5.7)` / `preset (private:rg)` cells of the plan and the `SG_TF_FALLBACK_LABEL` used in messages. The trigger pass skips workflows that do not exist in SG, and `sg_api_post` returns 22 on 4xx so `sg_retry` (via `SG_NO_RETRY_RC`) does not retry definitive errors. `clean` removes local artifacts (`export/`, TF state, tool cache); `clean --all` also removes config. `completion bash|zsh` prints a completion script (`cmd_completion`; commands/options come from `SG_COMMANDS`/`SG_OPTIONS`, keep them in sync with the parser and the host flags); like `clean` it always runs natively. Output is concise by default — `apply` captures terraform's init/plan output and shows only progress + the final summary (or the full log on failure, `TF_IN_AUTOMATION=1`/`-no-color`); `-v`/`--verbose` (`SG_VERBOSE`) streams everything and un-gates the convert detail lines. Colored logging via `sg_step`/`sg_log`/`sg_success`/`sg_warn`/`sg_err`/`sg_row`; paths shown relative via `sg_rel` (all auto-off when not a TTY / `NO_COLOR`). Long steps show a live progress line (`sg_run_quiet` for terraform init/apply, `run_parallel <fn> <max> <label> <items>` for convert/import/triggers — animated only when `SG_ANIMATE=1`, i.e. colors are on; plain log lines otherwise) and report their duration (`sg_fmt_secs`). Phases start with `phase_begin` — inside `all` the header is numbered (`Phase 2/5: ...`, `PHASE_TOTAL` set in `main`) and `phase_took` gives the elapsed time; `run_parallel` prints a job's output as-is when it is a single line and adds the `── <file> ──` header only for longer or failed output (always with `-v`). The import confirmation goes through `sg_confirm` (so `SG_ANSWERS_FILE` works; a declined prompt is a neutral "cancelled" line, not an error) and `import`/`all` end with `finish_line` (outcome, total time, open checklist items from `CHECKLIST_OPEN`). A Ctrl-C is trapped to clear the progress line and print a resume hint. - `scripts/lib/` — sourced libraries (a `lib/` gitignore rule is negated for this dir): `prompt.sh` (`sg_ask`/`sg_confirm`/`sg_select`; tty or `SG_ANSWERS_FILE` for tests; `SG_NONINTERACTIVE=1`/`-y`/no TTY → defaults), `tfvars.sh` (`tfvars_get` cached hcl2json reads, `tfvars_has`/`tfvars_is_null` to tell an explicit `null` from a missing key, `tfvars_write` renders the wizard's `W_*` vars — `W_TF_SOURCE`, and `W_TF_VERSION`/`W_RUNNER_JSON` may be the literal `null`; plain strings go through `_tfvars_str`, so quotes/backslashes in names stay valid HCL), `tfc_api.sh` (`tfc_http`/`tfc_get_all` paginated, `tfc_list_{orgs,projects,workspaces}` — workspaces include `vcs_provider`/`vcs_url`/`vcs_identifier` —, `tfc_select_workspaces` (the module's name-glob / include-tags(all) / exclude-tags filter, shared by wizard and preflight), `tfc_vcs_summary`/`tfc_vcs_kind_for`/`tfc_vcs_label_for`, `tfc_token` with provider precedence, `require_tfc_auth`), `sg_api.sh` (`_sg_api` 0/22/1 contract + `SG_HTTP_CODE`, `sg_list_integrations`, `sg_integration_type`, `sg_vcs_kind_of` (connector type → `sourceConfigDestKind`), `sg_execution_preset` + `sg_preset_{desc,runner_desc,version_desc,runner_provided}` (the org's execution preset and how to describe it), `sg_*_exists`, `sg_list_workflows`, `sg_create_secret`, `sg_patch_workflow`, `wfgroup_*`), `wizard.sh` (`wizard_run`: 4 steps TFC → SG → defaults → review; the TFC step keeps the workspace list and derives, via `tfc_vcs_summary`, which VCS provider the selected workspaces use and the repositories' URL prefix — the SG step lists matching connectors first and takes the repo URL prefix from TFC (never from a previous tfvars written for another provider kind), warns on a kind mismatch, reads the org's execution preset and offers it as the runner choice; the defaults step asks whether to carry TFC versions or use the preset (`SGTerraformVersionSource`) and, for carry, which fallback (`SGDefaultTerraformVersion` or the preset = null); the review shows human-readable rows plus the `tfc-<project>` groups; `_w_select_lines` feeds menus line-by-line so names with spaces/globs survive), `preflight.sh` (`preflight_run <apply|import|all>`, once per process; ✓ lines are plain language and name the connector's role/kind, ✗/! lines name the tfvars field to fix; besides existence it checks consistency — VCS kind vs. the connector's type (fail), repo URL prefix vs. where the TFC repositories live or the kind's well-known host (warn); it also reports the org's execution preset next to the version/runner policy and warns when `carry` meets a preset that mounts a runner-provided binary), `state.sh` (`.sg/state.json`: `phases.<name>.input_sha`, `import.<seg>.{imported,failed,tf_fallback,state_uploaded,state_failed}`, `triggers.<seg>`, `secrets.<name>`; `run_phase` in migrate.sh skips unchanged phases), `report.sh` (`show_migration_summary` after apply — incl. a state-files row —, `show_import_plan` per-workflow table with columns sized to their content), `errors.sh` (`explain_api_error` regex→hint table, `SG_API_HINTS`), `checklist.sh` (`create_secret_stubs` → SG secret `tfc-<wf>-<VAR>` = `CHANGE_ME`, referenced as `${secret::<name>}` via PATCH + payload patch; `write_checklist` → `export/post-import-checklist.md`, printed to the terminal as one ✓/! status line per section plus the imported groups with UI links — the file has the details; section 5 covers both the TFC state export and the SG state upload (`state_failed` from the import results); sets `CHECKLIST_OPEN`). Parallel jobs (`run_parallel`) run in subshells, so `do_import`/`do_set_triggers` write `.import-result.<seg>.json` / `.triggers-result.<seg>.json` into the export dir and the caller merges them into state. - `scripts/tools.sh` — sourced by the other scripts (sets `SG_REPO_ROOT` to its parent dir). Provides `sg_resolve` (PATH-or-cache), `sg_ensure_{jq,hcl2json,yajsv,sgcli}` (pinned downloads into `.sg/cached/`; `sg-cli` is the Go binary), `sg_retry`, the color vars + log helpers. The Docker image bundles these tools at build time (`yajsv` built from source so arm64 works), so in-container runs never download. - **Variable sets** — `scripts/enrich_variable_sets.sh <tfc-org> <payload...>` (run by `cmd_enrich`, which reads `tfOrg`/`tfHostname` from `terraform.tfvars` via `hcl2json` — variable sets live in the **TFC** org, not `SG_ORG`). Lists all sets + vars via the TFC API (Bearer token from the `terraform login` creds file or `TFE_TOKEN`), computes per-workspace effective vars (scope: global/project/workspace; precedence: priority sets > workspace vars > non-priority sets), and merges them into each workflow (matched by workspace name parsed from `TfStateFilePath`). Sensitive set vars and key conflicts are reported. Skipped by `--no-variable-sets`. diff --git a/README.md b/README.md index 707cc46..bf7e6e4 100644 --- a/README.md +++ b/README.md @@ -170,6 +170,7 @@ To update workflows with different details, re-run the sg-cli command with the m - **Execution presets.** StackGuardian org admins can define an execution preset (Settings → Runner groups → Execution presets): default runner constraints plus a Terraform and an OpenTofu configuration that the API applies to any new workflow whose payload does not carry those fields. To lean on it, set `SGTerraformVersionSource = "preset"` (no version is sent at all) and/or `SGDefaultRunnerConstraints = null` (no runner constraints are sent); `SGDefaultTerraformVersion = null` keeps carrying TFC pins but hands the unpinned and rejected ones to the preset. `init` offers these choices with the org's current preset shown inline, preflight prints what the preset would supply, and the import plan marks such cells as `preset (…)`. The preset's custom runtime image or runner-provided binary is inherited in every mode, since the migrator never sets those keys. - **Workspaces without variables are imported directly.** sg-cli (up to v2.2.1) drops an empty `iacInputData.data` from the request, and the API then rejects the workflow with `VCSConfig.iacInputData.data: This field is required.` The importer therefore creates workflows that have no Terraform variables straight through the API (same create, same state upload) and uses sg-cli for the rest. This is a workaround until sg-cli picks up sg-sdk-go v1.5.7, which fixes the dropped key. - **Terraform state is uploaded by the migrator, and checked.** sg-cli's own upload sends a PUT that Azure-backed StackGuardian environments reject (missing `x-ms-blob-type`) and it reports success only on a literal `HTTP/1.1 200 OK`, so the importer re-uploads every state file sg-cli reports as failed and records per workflow whether the store accepted it. A workflow without its state counts as a failed import: it is listed in the checklist (`state: N workflow(s) are in SG without their state`) and the file is retried on the next `import`. +- **Fail fast.** When more than one workflow is to be imported, the importer first imports a single one (the first selected workflow of the first payload file) and requires both the create and its state upload to succeed before the rest is imported in parallel. An environment problem then costs one workflow, not all of them; the probe workflow is simply updated again with its file. - **State export is idempotent.** Re-running `terraform apply` only pulls state for workspaces not yet exported. Set `forceStateRefresh = true` to re-pull everything. Workspaces not using `remote` execution may export incomplete state — see the summary. - **Workflow naming.** `ResourceName` mirrors the TFC workspace name, sanitized to StackGuardian's rules (1-100 chars, `[-a-zA-Z0-9_]`); any rename is listed in the summary and the checklist. - **Preflight.** `apply`, `import` and `all` first verify the TFC and SG tokens, the TFC org and workspace selection, and that every connector, secret and runner group referenced in `terraform.tfvars` exists. Fix what it reports (or re-run `init`); `--skip-preflight` bypasses it. diff --git a/scripts/migrate.sh b/scripts/migrate.sh index 58cce61..b7e2e96 100755 --- a/scripts/migrate.sh +++ b/scripts/migrate.sh @@ -731,6 +731,38 @@ do_import() { return "$rc" } +# probe_import <payload>... — fail fast: import a single workflow (the first +# selected one of the first file) and require both the create and its state +# upload to succeed before the rest is imported in parallel. An environment +# problem (wrong connector kind, a store that rejects the upload, a read-only +# token) then costs one workflow instead of all of them. The probe workflow is +# re-imported with its file afterwards (sg-cli updates it in place). +probe_import() { + local f="$1" seg grp name dir probe res + seg="$(seg_of "$f")" + grp="$(group_for "$seg")" + name="$("$JQ_BIN" -r --argjson ws "$(ws_filter_json)" 'first(.[] | select(($ws | length) == 0 or (.ResourceName as $n | $ws | index($n) != null)) | .ResourceName) // empty' "$f")" + [ -n "$name" ] || return 0 + dir="$(mktemp -d "$EXPORT_DIR/.probe.XXXXXX")" + probe="$dir/$(basename "$f")" + "$JQ_BIN" --arg n "$name" 'map(select(.ResourceName == $n))' "$f" >"$probe" + sg_log "probing with one workflow before importing the rest: $grp/$name" + do_import "$probe" || true + res="$EXPORT_DIR/.import-result.$seg.json" + if [ -f "$res" ] && [ "$("$JQ_BIN" '(.failed | length) + (.state_failed | length)' "$res")" -eq 0 ]; then + if [ "$("$JQ_BIN" '.state_uploaded | length' "$res")" -gt 0 ]; then + sg_success "probe ok — $name created and its state uploaded; importing the rest" + else + sg_success "probe ok — $name created (no state file to upload); importing the rest" + fi + rm -rf "$dir" "$res" + return 0 + fi + [ -f "$res" ] && state_record_import "$seg" "$(cat "$res")" + rm -rf "$dir" "$res" + die "probe failed for $grp/$name (see above) — nothing else was imported; fix the cause and re-run '$PROG import'" +} + # Print the customer-facing notice when any workflow fell back to the default # Terraform version during this import. tf_fallback_notice() { @@ -875,6 +907,8 @@ cmd_import() { done [ "$skipped" -gt 0 ] && sg_log "skipping $skipped payload file(s) already imported and unchanged (--fresh to re-import)" if [ "${#todo[@]}" -gt 0 ]; then + # More than one workflow to import: try a single one first (fail fast). + if [ "$total_wf" -gt 1 ]; then probe_import "${todo[0]}"; fi run_parallel do_import "$CONC" "importing ${#todo[@]} payload file(s) (retries: $RETRIES)" "${todo[@]}" || import_rc=1 for f in "${todo[@]}"; do seg="$(seg_of "$f")" From e487f370f73dffa6714289449f2ff139d345cd6f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adis=20Halilovi=C4=87?= <adis.halilovic@stackguardian.io> Date: Mon, 14 Sep 2026 11:16:44 +0200 Subject: [PATCH 35/71] fix: update existing workflows on re-import; sg-cli never takes its update path sg-cli switches from create to update only when the API's message says "Workflow name not unique", but the API answers 409 "Workflow ID not unique", so every re-import of an existing workflow (a retry, --fresh, the fail-fast probe) failed with that 409 and no state was uploaded. import_bulk now treats a 409 / "not unique" create failure as an existing workflow: it PATCHes the payload entry via sg_update_workflow, uploads its state and drops the failure line, so re-runs are idempotent. The direct-create path does the same fallback in sg_create_workflow. Workaround for sg-cli, marked TODO(sg-cli). --- CLAUDE.md | 2 +- README.md | 1 + scripts/lib/errors.sh | 2 +- scripts/lib/sg_api.sh | 53 +++++++++++++++++++++++++++++++------------ scripts/migrate.sh | 53 ++++++++++++++++++++++++++++++++++--------- 5 files changed, 83 insertions(+), 28 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index fc602d9..066a48b 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -24,7 +24,7 @@ The migration is a user-driven pipeline, not a single program: The five phases are wrapped by an orchestrator so users don't run them by hand: - `sg-migrate.sh` (host entrypoint, repo root) — runs `scripts/migrate.sh` **inside the Docker image** (`Dockerfile`), bind-mounting the repo at `/app` and forwarding `SG_API_TOKEN`/`SG_ORG`/`SG_BASE_URL`/`TFE_TOKEN`/`TF_TOKEN_*`. For TFC auth it prefers a long-lived `TFE_TOKEN` (env); otherwise it mounts `~/.terraform.d/credentials.tfrc.json` read-only. Runs natively instead when `--native`/`--local` is passed, `SG_NATIVE=1` is set, the command is `clean`/`completion`/help (or no command is given), or Docker is absent. Exports `SG_PROG` so `migrate.sh` shows `./sg-migrate.sh` in its usage/hints. -- `scripts/migrate.sh` — the actual orchestrator (sources `scripts/lib/*.sh`, see below). Subcommands `init|preflight|apply|enrich|convert|validate|import|triggers|checklist|all|clean|completion` (no command prints the help menu; `enrich` runs in `all` unless `--no-variable-sets`). Flags beyond the basics: `--dry-run` (import: plan only), `--fresh` (ignore run state), `--project SEG` / `--workspace NAME` (repeatable filters; apply maps `--workspace` to `-var workspacenames=[...]`, import works on a jq-filtered temp copy), `--skip-preflight`, `--no-secret-stubs`. Resolves each tool via `sg_resolve` (PATH first — the image installs them — else `sg_ensure_*` cache). `apply`/`all` first run `require_tfc_auth`, which resolves the token the tfe provider will use (`TFE_TOKEN`, `TF_TOKEN_<host>`, or the `terraform login` file for `tfHostname`) and verifies it with `GET /api/v2/account/details`, failing fast on a missing or rejected (expired) credential. Runs `convert` and `import` in parallel (`--concurrency`, default 4), raises `terraform apply -parallelism`, and retries `sg-cli` imports with backoff (`sg_retry`, `SG_RETRIES`). Import resolves each project's workflow group, checks existence via the SG API, shows a plan (`exists`/`create`), prompts (skip with `-y`), **creates the missing `tfc-<project>` groups via the API**, then imports. Imports go through `import_bulk`: workflows with Terraform variables via `sg-cli workflow create --bulk`, workflows **without** via a direct `POST .../wfs/` (`sg_create_workflow` + `sg_upload_tfstate` in `sg_api.sh`) because sg-cli drops an empty `iacInputData.data` and the API rejects the workflow — a workaround (`TODO(sg-cli)` markers) until sg-cli ships with sg-sdk-go >= v1.5.7, which sends the empty object; the direct path prints sg-cli-style `Failed to create <name>: <code>: <body>` lines so the parsing below covers both. State files are the migrator's job on both paths: `import_bulk` tracks sg-cli's per-workflow upload lines (`State file uploaded successfully` / `Failed to upload state file for <wf>`), re-uploads the failures via `upload_state` → `sg_upload_tfstate` (adds the `x-ms-blob-type: BlockBlob` header Azure Blob requires; sg-cli omits it and also treats anything but a literal `HTTP/1.1 200 OK` as failure — second `TODO(sg-cli)`), and writes `[state] uploaded|failed|none <wf>` markers into the parsed output; `do_import` turns them into `state_uploaded`/`state_failed` in the result, a workflow without its state fails the file (and `state_import_done` re-tries it next run). Before the parallel import, `probe_import` imports one workflow alone (first selected of the first file) and dies unless the create and the state upload both succeeded — the fail-fast for environment problems. sg-cli exits 0 even when individual workflows fail, so `do_import` parses its output: a workflow rejected as above SG's managed Terraform ceiling (1.5.7, last MPL/FOSS release) is re-imported with `SGDefaultTerraformVersion` (read from `terraform.tfvars`; an explicit `null` there means the `terraformVersion` key is deleted instead, so the org's execution preset decides — `tfvars_is_null` tells an explicit null from a missing key), the payload patched in place, and the case logged to `export/terraform-version-fallbacks.log` plus a printed notice; any other per-workflow failure fails the run. `cmd_import` reads the org's execution preset (`sg_execution_preset`, `GET /orgs/{org}/` → `Settings.workflowDefaults`) and `preset_labels` (report.sh) turns it into the `preset (1.5.7)` / `preset (private:rg)` cells of the plan and the `SG_TF_FALLBACK_LABEL` used in messages. The trigger pass skips workflows that do not exist in SG, and `sg_api_post` returns 22 on 4xx so `sg_retry` (via `SG_NO_RETRY_RC`) does not retry definitive errors. `clean` removes local artifacts (`export/`, TF state, tool cache); `clean --all` also removes config. `completion bash|zsh` prints a completion script (`cmd_completion`; commands/options come from `SG_COMMANDS`/`SG_OPTIONS`, keep them in sync with the parser and the host flags); like `clean` it always runs natively. Output is concise by default — `apply` captures terraform's init/plan output and shows only progress + the final summary (or the full log on failure, `TF_IN_AUTOMATION=1`/`-no-color`); `-v`/`--verbose` (`SG_VERBOSE`) streams everything and un-gates the convert detail lines. Colored logging via `sg_step`/`sg_log`/`sg_success`/`sg_warn`/`sg_err`/`sg_row`; paths shown relative via `sg_rel` (all auto-off when not a TTY / `NO_COLOR`). Long steps show a live progress line (`sg_run_quiet` for terraform init/apply, `run_parallel <fn> <max> <label> <items>` for convert/import/triggers — animated only when `SG_ANIMATE=1`, i.e. colors are on; plain log lines otherwise) and report their duration (`sg_fmt_secs`). Phases start with `phase_begin` — inside `all` the header is numbered (`Phase 2/5: ...`, `PHASE_TOTAL` set in `main`) and `phase_took` gives the elapsed time; `run_parallel` prints a job's output as-is when it is a single line and adds the `── <file> ──` header only for longer or failed output (always with `-v`). The import confirmation goes through `sg_confirm` (so `SG_ANSWERS_FILE` works; a declined prompt is a neutral "cancelled" line, not an error) and `import`/`all` end with `finish_line` (outcome, total time, open checklist items from `CHECKLIST_OPEN`). A Ctrl-C is trapped to clear the progress line and print a resume hint. +- `scripts/migrate.sh` — the actual orchestrator (sources `scripts/lib/*.sh`, see below). Subcommands `init|preflight|apply|enrich|convert|validate|import|triggers|checklist|all|clean|completion` (no command prints the help menu; `enrich` runs in `all` unless `--no-variable-sets`). Flags beyond the basics: `--dry-run` (import: plan only), `--fresh` (ignore run state), `--project SEG` / `--workspace NAME` (repeatable filters; apply maps `--workspace` to `-var workspacenames=[...]`, import works on a jq-filtered temp copy), `--skip-preflight`, `--no-secret-stubs`. Resolves each tool via `sg_resolve` (PATH first — the image installs them — else `sg_ensure_*` cache). `apply`/`all` first run `require_tfc_auth`, which resolves the token the tfe provider will use (`TFE_TOKEN`, `TF_TOKEN_<host>`, or the `terraform login` file for `tfHostname`) and verifies it with `GET /api/v2/account/details`, failing fast on a missing or rejected (expired) credential. Runs `convert` and `import` in parallel (`--concurrency`, default 4), raises `terraform apply -parallelism`, and retries `sg-cli` imports with backoff (`sg_retry`, `SG_RETRIES`). Import resolves each project's workflow group, checks existence via the SG API, shows a plan (`exists`/`create`), prompts (skip with `-y`), **creates the missing `tfc-<project>` groups via the API**, then imports. Imports go through `import_bulk`: workflows with Terraform variables via `sg-cli workflow create --bulk`, workflows **without** via a direct `POST .../wfs/` (`sg_create_workflow` + `sg_upload_tfstate` in `sg_api.sh`) because sg-cli drops an empty `iacInputData.data` and the API rejects the workflow — a workaround (`TODO(sg-cli)` markers) until sg-cli ships with sg-sdk-go >= v1.5.7, which sends the empty object; the direct path prints sg-cli-style `Failed to create <name>: <code>: <body>` lines so the parsing below covers both. State files are the migrator's job on both paths: `import_bulk` tracks sg-cli's per-workflow upload lines (`State file uploaded successfully` / `Failed to upload state file for <wf>`), re-uploads the failures via `upload_state` → `sg_upload_tfstate` (adds the `x-ms-blob-type: BlockBlob` header Azure Blob requires; sg-cli omits it and also treats anything but a literal `HTTP/1.1 200 OK` as failure — second `TODO(sg-cli)`), and writes `[state] uploaded|failed|none <wf>` markers into the parsed output; `do_import` turns them into `state_uploaded`/`state_failed` in the result, a workflow without its state fails the file (and `state_import_done` re-tries it next run). Existing workflows come back from sg-cli as `Failed to create <wf>: 409: Workflow ID not unique` (its update branch matches the outdated text `Workflow name not unique` — third `TODO(sg-cli)`); `import_bulk` collects those, PATCHes them via `sg_update_workflow`, re-uploads their state and drops the failure line, and `sg_create_workflow` does the same fallback on the direct path (prints `updated`). Before the parallel import, `probe_import` imports one workflow alone (first selected of the first file) and dies unless the create and the state upload both succeeded — the fail-fast for environment problems. sg-cli exits 0 even when individual workflows fail, so `do_import` parses its output: a workflow rejected as above SG's managed Terraform ceiling (1.5.7, last MPL/FOSS release) is re-imported with `SGDefaultTerraformVersion` (read from `terraform.tfvars`; an explicit `null` there means the `terraformVersion` key is deleted instead, so the org's execution preset decides — `tfvars_is_null` tells an explicit null from a missing key), the payload patched in place, and the case logged to `export/terraform-version-fallbacks.log` plus a printed notice; any other per-workflow failure fails the run. `cmd_import` reads the org's execution preset (`sg_execution_preset`, `GET /orgs/{org}/` → `Settings.workflowDefaults`) and `preset_labels` (report.sh) turns it into the `preset (1.5.7)` / `preset (private:rg)` cells of the plan and the `SG_TF_FALLBACK_LABEL` used in messages. The trigger pass skips workflows that do not exist in SG, and `sg_api_post` returns 22 on 4xx so `sg_retry` (via `SG_NO_RETRY_RC`) does not retry definitive errors. `clean` removes local artifacts (`export/`, TF state, tool cache); `clean --all` also removes config. `completion bash|zsh` prints a completion script (`cmd_completion`; commands/options come from `SG_COMMANDS`/`SG_OPTIONS`, keep them in sync with the parser and the host flags); like `clean` it always runs natively. Output is concise by default — `apply` captures terraform's init/plan output and shows only progress + the final summary (or the full log on failure, `TF_IN_AUTOMATION=1`/`-no-color`); `-v`/`--verbose` (`SG_VERBOSE`) streams everything and un-gates the convert detail lines. Colored logging via `sg_step`/`sg_log`/`sg_success`/`sg_warn`/`sg_err`/`sg_row`; paths shown relative via `sg_rel` (all auto-off when not a TTY / `NO_COLOR`). Long steps show a live progress line (`sg_run_quiet` for terraform init/apply, `run_parallel <fn> <max> <label> <items>` for convert/import/triggers — animated only when `SG_ANIMATE=1`, i.e. colors are on; plain log lines otherwise) and report their duration (`sg_fmt_secs`). Phases start with `phase_begin` — inside `all` the header is numbered (`Phase 2/5: ...`, `PHASE_TOTAL` set in `main`) and `phase_took` gives the elapsed time; `run_parallel` prints a job's output as-is when it is a single line and adds the `── <file> ──` header only for longer or failed output (always with `-v`). The import confirmation goes through `sg_confirm` (so `SG_ANSWERS_FILE` works; a declined prompt is a neutral "cancelled" line, not an error) and `import`/`all` end with `finish_line` (outcome, total time, open checklist items from `CHECKLIST_OPEN`). A Ctrl-C is trapped to clear the progress line and print a resume hint. - `scripts/lib/` — sourced libraries (a `lib/` gitignore rule is negated for this dir): `prompt.sh` (`sg_ask`/`sg_confirm`/`sg_select`; tty or `SG_ANSWERS_FILE` for tests; `SG_NONINTERACTIVE=1`/`-y`/no TTY → defaults), `tfvars.sh` (`tfvars_get` cached hcl2json reads, `tfvars_has`/`tfvars_is_null` to tell an explicit `null` from a missing key, `tfvars_write` renders the wizard's `W_*` vars — `W_TF_SOURCE`, and `W_TF_VERSION`/`W_RUNNER_JSON` may be the literal `null`; plain strings go through `_tfvars_str`, so quotes/backslashes in names stay valid HCL), `tfc_api.sh` (`tfc_http`/`tfc_get_all` paginated, `tfc_list_{orgs,projects,workspaces}` — workspaces include `vcs_provider`/`vcs_url`/`vcs_identifier` —, `tfc_select_workspaces` (the module's name-glob / include-tags(all) / exclude-tags filter, shared by wizard and preflight), `tfc_vcs_summary`/`tfc_vcs_kind_for`/`tfc_vcs_label_for`, `tfc_token` with provider precedence, `require_tfc_auth`), `sg_api.sh` (`_sg_api` 0/22/1 contract + `SG_HTTP_CODE`, `sg_list_integrations`, `sg_integration_type`, `sg_vcs_kind_of` (connector type → `sourceConfigDestKind`), `sg_execution_preset` + `sg_preset_{desc,runner_desc,version_desc,runner_provided}` (the org's execution preset and how to describe it), `sg_*_exists`, `sg_list_workflows`, `sg_create_secret`, `sg_patch_workflow`, `wfgroup_*`), `wizard.sh` (`wizard_run`: 4 steps TFC → SG → defaults → review; the TFC step keeps the workspace list and derives, via `tfc_vcs_summary`, which VCS provider the selected workspaces use and the repositories' URL prefix — the SG step lists matching connectors first and takes the repo URL prefix from TFC (never from a previous tfvars written for another provider kind), warns on a kind mismatch, reads the org's execution preset and offers it as the runner choice; the defaults step asks whether to carry TFC versions or use the preset (`SGTerraformVersionSource`) and, for carry, which fallback (`SGDefaultTerraformVersion` or the preset = null); the review shows human-readable rows plus the `tfc-<project>` groups; `_w_select_lines` feeds menus line-by-line so names with spaces/globs survive), `preflight.sh` (`preflight_run <apply|import|all>`, once per process; ✓ lines are plain language and name the connector's role/kind, ✗/! lines name the tfvars field to fix; besides existence it checks consistency — VCS kind vs. the connector's type (fail), repo URL prefix vs. where the TFC repositories live or the kind's well-known host (warn); it also reports the org's execution preset next to the version/runner policy and warns when `carry` meets a preset that mounts a runner-provided binary), `state.sh` (`.sg/state.json`: `phases.<name>.input_sha`, `import.<seg>.{imported,failed,tf_fallback,state_uploaded,state_failed}`, `triggers.<seg>`, `secrets.<name>`; `run_phase` in migrate.sh skips unchanged phases), `report.sh` (`show_migration_summary` after apply — incl. a state-files row —, `show_import_plan` per-workflow table with columns sized to their content), `errors.sh` (`explain_api_error` regex→hint table, `SG_API_HINTS`), `checklist.sh` (`create_secret_stubs` → SG secret `tfc-<wf>-<VAR>` = `CHANGE_ME`, referenced as `${secret::<name>}` via PATCH + payload patch; `write_checklist` → `export/post-import-checklist.md`, printed to the terminal as one ✓/! status line per section plus the imported groups with UI links — the file has the details; section 5 covers both the TFC state export and the SG state upload (`state_failed` from the import results); sets `CHECKLIST_OPEN`). Parallel jobs (`run_parallel`) run in subshells, so `do_import`/`do_set_triggers` write `.import-result.<seg>.json` / `.triggers-result.<seg>.json` into the export dir and the caller merges them into state. - `scripts/tools.sh` — sourced by the other scripts (sets `SG_REPO_ROOT` to its parent dir). Provides `sg_resolve` (PATH-or-cache), `sg_ensure_{jq,hcl2json,yajsv,sgcli}` (pinned downloads into `.sg/cached/`; `sg-cli` is the Go binary), `sg_retry`, the color vars + log helpers. The Docker image bundles these tools at build time (`yajsv` built from source so arm64 works), so in-container runs never download. - **Variable sets** — `scripts/enrich_variable_sets.sh <tfc-org> <payload...>` (run by `cmd_enrich`, which reads `tfOrg`/`tfHostname` from `terraform.tfvars` via `hcl2json` — variable sets live in the **TFC** org, not `SG_ORG`). Lists all sets + vars via the TFC API (Bearer token from the `terraform login` creds file or `TFE_TOKEN`), computes per-workspace effective vars (scope: global/project/workspace; precedence: priority sets > workspace vars > non-priority sets), and merges them into each workflow (matched by workspace name parsed from `TfStateFilePath`). Sensitive set vars and key conflicts are reported. Skipped by `--no-variable-sets`. diff --git a/README.md b/README.md index bf7e6e4..1686818 100644 --- a/README.md +++ b/README.md @@ -170,6 +170,7 @@ To update workflows with different details, re-run the sg-cli command with the m - **Execution presets.** StackGuardian org admins can define an execution preset (Settings → Runner groups → Execution presets): default runner constraints plus a Terraform and an OpenTofu configuration that the API applies to any new workflow whose payload does not carry those fields. To lean on it, set `SGTerraformVersionSource = "preset"` (no version is sent at all) and/or `SGDefaultRunnerConstraints = null` (no runner constraints are sent); `SGDefaultTerraformVersion = null` keeps carrying TFC pins but hands the unpinned and rejected ones to the preset. `init` offers these choices with the org's current preset shown inline, preflight prints what the preset would supply, and the import plan marks such cells as `preset (…)`. The preset's custom runtime image or runner-provided binary is inherited in every mode, since the migrator never sets those keys. - **Workspaces without variables are imported directly.** sg-cli (up to v2.2.1) drops an empty `iacInputData.data` from the request, and the API then rejects the workflow with `VCSConfig.iacInputData.data: This field is required.` The importer therefore creates workflows that have no Terraform variables straight through the API (same create, same state upload) and uses sg-cli for the rest. This is a workaround until sg-cli picks up sg-sdk-go v1.5.7, which fixes the dropped key. - **Terraform state is uploaded by the migrator, and checked.** sg-cli's own upload sends a PUT that Azure-backed StackGuardian environments reject (missing `x-ms-blob-type`) and it reports success only on a literal `HTTP/1.1 200 OK`, so the importer re-uploads every state file sg-cli reports as failed and records per workflow whether the store accepted it. A workflow without its state counts as a failed import: it is listed in the checklist (`state: N workflow(s) are in SG without their state`) and the file is retried on the next `import`. +- **Re-runs update existing workflows.** sg-cli answers `409 Workflow ID not unique` for a workflow that already exists instead of updating it (its update path waits for a message the API no longer sends), so the importer updates such workflows itself via PATCH and re-uploads their state. Re-running `import` (or `import --fresh`) is therefore safe and idempotent. - **Fail fast.** When more than one workflow is to be imported, the importer first imports a single one (the first selected workflow of the first payload file) and requires both the create and its state upload to succeed before the rest is imported in parallel. An environment problem then costs one workflow, not all of them; the probe workflow is simply updated again with its file. - **State export is idempotent.** Re-running `terraform apply` only pulls state for workspaces not yet exported. Set `forceStateRefresh = true` to re-pull everything. Workspaces not using `remote` execution may export incomplete state — see the summary. - **Workflow naming.** `ResourceName` mirrors the TFC workspace name, sanitized to StackGuardian's rules (1-100 chars, `[-a-zA-Z0-9_]`); any rename is listed in the summary and the checklist. diff --git a/scripts/lib/errors.sh b/scripts/lib/errors.sh index bed4439..d6afccc 100644 --- a/scripts/lib/errors.sh +++ b/scripts/lib/errors.sh @@ -16,7 +16,7 @@ SG_API_HINTS=( 'runner ?group.*(not found|does not exist|invalid)|(not found|does not exist|invalid).*runner => Runner group not found. Check SGDefaultRunnerConstraints.names / workspaceOverrides[<ws>].RunnerConstraints against the runner groups in the SG org.' '(repo|repository).*(not found|access|permission|denied|unable|could not|cannot)|(clone|checkout).*(fail|denied|unable) => The VCS connector cannot reach the repository. Check SGDefaultIACVCSRepoPrefix + the workspace repo path, and that the connector (SGDefaultVCSAuthIntegrationID) has access to that repository.' 'approver => Approvers must be e-mail addresses of existing SG users. Check SGDefaultWfApprovers / workspaceOverrides[<ws>].Approvers.' - 'already exists => A resource with that name already exists. Re-running the import updates workflows in place; for workflow groups this is harmless.' + 'already exists => A resource with that name already exists. Existing workflows are updated by the importer; for workflow groups this is harmless.' 'ResourceName => Invalid workflow name: 1-100 chars, letters/digits/-/_ only. The transformer sanitizes names (see renamedWorkspaces in migration-summary.md); adjust local.resourceNames if a rule is missing.' 'sourceConfigDestKind => Invalid VCS kind. SGDefaultSourceConfigDestKind must be one of GITHUB_COM, GITLAB_COM, BITBUCKET_ORG, AZURE_DEVOPS, GIT_OTHER.' '(unauthori[sz]ed|forbidden|invalid token|authentication) => SG_API_TOKEN was rejected or lacks permission for this org. Regenerate it under Org settings -> API keys and re-export SG_API_TOKEN.' diff --git a/scripts/lib/sg_api.sh b/scripts/lib/sg_api.sh index 3b54e2c..e908517 100644 --- a/scripts/lib/sg_api.sh +++ b/scripts/lib/sg_api.sh @@ -105,28 +105,51 @@ sg_patch_workflow() { sg_api_patch "$(wf_url "$1" "$2")" "$3"; } # >= v1.5.7 (IacInputData.Data became a pointer so "data": {} is sent) or POST # the raw entry. Drop this block and import_bulk's split once a release has it. -# sg_create_workflow <group> <entry-json> — POST one payload entry (PATCH when -# the name already exists, as sg-cli does). Silent on success; on failure prints -# "<http-code>: <body>" so the caller can log a sg-cli-style line. 0/22/1. -sg_create_workflow() { - local grp="$1" entry="$2" jq name body tmp rc=0 - jq="$(sg_resolve jq sg_ensure_jq)" - name="$("$jq" -r '.ResourceName' <<<"$entry")" - # CLIConfiguration is sg-cli's own block; VCSTriggers is applied in the - # trigger pass (the create API does not take it). - body="$("$jq" -c 'del(.CLIConfiguration, .VCSTriggers)' <<<"$entry")" +# _sg_wf_body <entry-json> — a payload entry as the workflow API takes it: +# CLIConfiguration is sg-cli's own block; VCSTriggers is applied in the +# trigger pass (the create API does not take it). +_sg_wf_body() { "$(sg_resolve jq sg_ensure_jq)" -c 'del(.CLIConfiguration, .VCSTriggers)' <<<"$1"; } + +# _sg_wf_call <method> <url> <body> — request; silent on success, on failure +# prints "<http-code>: <body>" (one line) so the caller can log a sg-cli-style +# line. 0/22/1. +_sg_wf_call() { + local tmp rc=0 tmp="$(mktemp)" # Response into a file, not $(...): SG_HTTP_CODE must survive into this shell. - sg_api_raw POST "$(sg_org_url)/wfgrps/$grp/wfs/" "$body" >"$tmp" || rc=$? - if [ "$rc" -eq 22 ] && grep -q "Workflow name not unique" "$tmp"; then - rc=0 - sg_api_raw PATCH "$(wf_url "$grp" "$name")" "$body" >"$tmp" || rc=$? - fi + sg_api_raw "$1" "$2" "$3" >"$tmp" || rc=$? [ "$rc" -eq 0 ] || printf '%s: %s\n' "$SG_HTTP_CODE" "$(tr -d '\n' <"$tmp")" rm -f "$tmp" return "$rc" } +# sg_update_workflow <group> <entry-json> — PATCH an existing workflow with the +# payload entry. Output/exit as _sg_wf_call. +# TODO(sg-cli): sg-cli only switches to its update path on the message +# "Workflow name not unique", but the API answers 409 "Workflow ID not unique", +# so re-importing an existing workflow fails inside sg-cli; import_bulk +# catches that 409 and updates through here. Remove once sg-cli handles it. +sg_update_workflow() { + local name + name="$("$(sg_resolve jq sg_ensure_jq)" -r '.ResourceName' <<<"$2")" + _sg_wf_call PATCH "$(wf_url "$1" "$name")" "$(_sg_wf_body "$2")" +} + +# sg_create_workflow <group> <entry-json> — POST one payload entry; when the +# workflow already exists (409 / "not unique") it is updated instead, as +# sg-cli intends to, and "updated" is printed. Otherwise output/exit as +# _sg_wf_call (callers run this in a subshell, so results travel via stdout). +sg_create_workflow() { + local grp="$1" entry="$2" err rc=0 + err="$(_sg_wf_call POST "$(sg_org_url)/wfgrps/$grp/wfs/" "$(_sg_wf_body "$entry")")" || rc=$? + if [ "$rc" -eq 22 ] && { [ "$SG_HTTP_CODE" = "409" ] || [[ "$err" == *"not unique"* ]]; }; then + sg_update_workflow "$grp" "$entry" && echo updated + return + fi + [ "$rc" -eq 0 ] || printf '%s\n' "$err" + return "$rc" +} + # sg_upload_tfstate <group> <wf> <file> — upload a workflow's Terraform state: # fetch the presigned URL, PUT the file. 0 on a 2xx from the store; otherwise # prints a one-line reason (for the log) and returns 1. The store behind the diff --git a/scripts/migrate.sh b/scripts/migrate.sh index b7e2e96..895214b 100755 --- a/scripts/migrate.sh +++ b/scripts/migrate.sh @@ -568,12 +568,17 @@ upload_state() { # sg-cli reports as a failed upload is uploaded again by upload_state, and # every workflow ends up with a "[state] ..." marker in <out>. # +# Workflows that already exist (re-runs, retries, the probe) come back from +# sg-cli as "Failed to create <wf>: 409: Workflow ID not unique" — its own +# update path never triggers (TODO(sg-cli), see sg_update_workflow) — and are +# updated here via PATCH, state included; their failure line is dropped. +# # Like sg-cli, exits 0 even when individual workflows were rejected — # do_import reads those from <out>; non-zero only when a call itself could # not be made. import_bulk() { - local grp="$1" file="$2" out="$3" rc=0 n_direct with_vars entry name err line cur="" cli_out - local -a redo=() + local grp="$1" file="$2" out="$3" rc=0 n_direct with_vars entry name err line cur="" cli_out pat + local -a redo=() exists=() updated=() : >"$out" n_direct="$("$JQ_BIN" '[.[] | select((.VCSConfig.iacInputData.data // {}) | length == 0)] | length' "$file")" with_vars="$file" @@ -584,23 +589,49 @@ import_bulk() { if [ "$("$JQ_BIN" length "$with_vars")" -gt 0 ]; then cli_out="$(mktemp)" sg_retry "$RETRIES" "$RETRY_BASE" -- sgcli_bulk "$grp" "$with_vars" "$cli_out" || rc=1 - cat "$cli_out" >>"$out" # sg-cli's own state upload, per workflow: trust its success, redo its # failures (it sends no x-ms-blob-type, and misreads anything but a - # literal "HTTP/1.1 200 OK" as a failure). + # literal "HTTP/1.1 200 OK" as a failure). A 409 is an existing workflow. while IFS= read -r line; do case "$line" in *"Processing workflow: "*) cur="${line##*Processing workflow: }" ;; - *"Failed to create "*) cur="" ;; + *"Failed to create "*) + cur="" + name="${line##*Failed to create }" + name="${name%%:*}" + case "$line" in *": 409: "* | *"not unique"*) exists+=("$name") ;; esac + ;; *"State file uploaded successfully"*) [ -n "$cur" ] && printf '[state] uploaded %s\n' "$cur" >>"$out" ;; *"Failed to upload state file for "*) name="${line##*Failed to upload state file for }"; redo+=("${name%%:*}") ;; *"cannot access state file"*) [ -n "$cur" ] && redo+=("$cur") ;; *"TfStateFilePath not provided for "*) name="${line##*TfStateFilePath not provided for }"; printf '[state] none %s\n' "${name%%:*}" >>"$out" ;; esac done <"$cli_out" + if [ "${#exists[@]}" -gt 0 ]; then + sg_log "${#exists[@]} workflow(s) already exist — updating them via the API (sg-cli's update path does not trigger on the 409)" + for name in "${exists[@]}"; do + entry="$("$JQ_BIN" -c --arg n "$name" 'first(.[] | select(.ResourceName == $n))' "$file")" + if err="$(SG_NO_RETRY_RC=22 sg_retry "$RETRIES" "$RETRY_BASE" -- sg_update_workflow "$grp" "$entry" 2>/dev/null)"; then + sg_log " $name: updated" + updated+=("$name") + redo+=("$name") + else + sg_warn " $name: update failed — $(tail -n1 <<<"$err")" + fi + done + fi + if [ "${#updated[@]}" -gt 0 ]; then + # Drop the create-failure line of every workflow that was updated instead. + pat="$(mktemp)" + for name in "${updated[@]}"; do printf 'Failed to create %s: ' "$name" >>"$pat"; echo >>"$pat"; done + grep -v -F -f "$pat" "$cli_out" >>"$out" || true + rm -f "$pat" + else + cat "$cli_out" >>"$out" + fi rm -f "$cli_out" if [ "${#redo[@]}" -gt 0 ]; then - sg_log "${#redo[@]} state upload(s) reported failed by sg-cli — uploading them directly" + sg_log "uploading the state of ${#redo[@]} workflow(s) directly (sg-cli reported the upload failed, or did not attempt it)" for name in "${redo[@]}"; do upload_state "$grp" "$name" "$file" "$out" || true; done fi fi @@ -611,7 +642,7 @@ import_bulk() { while IFS= read -r entry; do name="$("$JQ_BIN" -r '.ResourceName' <<<"$entry")" if err="$(SG_NO_RETRY_RC=22 sg_retry "$RETRIES" "$RETRY_BASE" -- sg_create_workflow "$grp" "$entry" 2>/dev/null)"; then - sg_log " $name: created" + if [ "$(tail -n1 <<<"$err")" = "updated" ]; then sg_log " $name: already existed — updated"; else sg_log " $name: created"; fi upload_state "$grp" "$name" "$file" "$out" || true else # Same shape as sg-cli's failure line (parsed by do_import). @@ -736,7 +767,7 @@ do_import() { # upload to succeed before the rest is imported in parallel. An environment # problem (wrong connector kind, a store that rejects the upload, a read-only # token) then costs one workflow instead of all of them. The probe workflow is -# re-imported with its file afterwards (sg-cli updates it in place). +# re-imported with its file afterwards (updated via PATCH, state included). probe_import() { local f="$1" seg grp name dir probe res seg="$(seg_of "$f")" @@ -751,9 +782,9 @@ probe_import() { res="$EXPORT_DIR/.import-result.$seg.json" if [ -f "$res" ] && [ "$("$JQ_BIN" '(.failed | length) + (.state_failed | length)' "$res")" -eq 0 ]; then if [ "$("$JQ_BIN" '.state_uploaded | length' "$res")" -gt 0 ]; then - sg_success "probe ok — $name created and its state uploaded; importing the rest" + sg_success "probe ok — $name is in SG with its state; importing the rest" else - sg_success "probe ok — $name created (no state file to upload); importing the rest" + sg_success "probe ok — $name is in SG (no state file to upload); importing the rest" fi rm -rf "$dir" "$res" return 0 @@ -894,7 +925,7 @@ cmd_import() { # Resume: skip payload files already imported in full with identical content # (a changed payload or a previous failure re-imports the whole file; - # sg-cli updates existing workflows in place). + # existing workflows are updated via PATCH). local -a todo=() local skipped=0 import_rc=0 for f in "${PF[@]}"; do From 1d37135f212008845ed59b7bb21843b2620351b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adis=20Halilovi=C4=87?= <adis.halilovic@stackguardian.io> Date: Mon, 14 Sep 2026 11:58:57 +0200 Subject: [PATCH 36/71] feat: update command pulls the latest version and rebuilds the image if needed Customers cloned or forked the repo per release and had no way to pick up fixes. ./sg-migrate.sh update runs on the host: it refuses a non-git checkout, a detached HEAD or dirty tracked files, then fast-forwards the current branch and rebuilds the Docker image only when the Dockerfile changed. Config and output are untracked, so a pull never touches them. --- CLAUDE.md | 2 +- README.md | 6 ++++- scripts/migrate.sh | 9 ++++++- sg-migrate.sh | 64 +++++++++++++++++++++++++++++++++++++++++++--- 4 files changed, 75 insertions(+), 6 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 066a48b..6492119 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -23,7 +23,7 @@ The migration is a user-driven pipeline, not a single program: The five phases are wrapped by an orchestrator so users don't run them by hand: -- `sg-migrate.sh` (host entrypoint, repo root) — runs `scripts/migrate.sh` **inside the Docker image** (`Dockerfile`), bind-mounting the repo at `/app` and forwarding `SG_API_TOKEN`/`SG_ORG`/`SG_BASE_URL`/`TFE_TOKEN`/`TF_TOKEN_*`. For TFC auth it prefers a long-lived `TFE_TOKEN` (env); otherwise it mounts `~/.terraform.d/credentials.tfrc.json` read-only. Runs natively instead when `--native`/`--local` is passed, `SG_NATIVE=1` is set, the command is `clean`/`completion`/help (or no command is given), or Docker is absent. Exports `SG_PROG` so `migrate.sh` shows `./sg-migrate.sh` in its usage/hints. +- `sg-migrate.sh` (host entrypoint, repo root) — runs `scripts/migrate.sh` **inside the Docker image** (`Dockerfile`), bind-mounting the repo at `/app` and forwarding `SG_API_TOKEN`/`SG_ORG`/`SG_BASE_URL`/`TFE_TOKEN`/`TF_TOKEN_*`. For TFC auth it prefers a long-lived `TFE_TOKEN` (env); otherwise it mounts `~/.terraform.d/credentials.tfrc.json` read-only. Runs natively instead when `--native`/`--local` is passed, `SG_NATIVE=1` is set, the command is `clean`/`completion`/help (or no command is given), or Docker is absent. Exports `SG_PROG` so `migrate.sh` shows `./sg-migrate.sh` in its usage/hints. `update` is host-only too (`cmd_update` in the wrapper, never passed to `migrate.sh`): refuses a non-git checkout, a detached HEAD or dirty tracked files, then `git pull --ff-only` and rebuilds the image only when the `Dockerfile` changed between the old and new HEAD — this is how customers pick up fixes from a plain clone (untracked config/output survive). `migrate.sh` lists `update` in its usage/`SG_COMMANDS` and rejects it with a host-only hint. `.gitattributes` forces LF so a Git-for-Windows clone does not CRLF the scripts. - `scripts/migrate.sh` — the actual orchestrator (sources `scripts/lib/*.sh`, see below). Subcommands `init|preflight|apply|enrich|convert|validate|import|triggers|checklist|all|clean|completion` (no command prints the help menu; `enrich` runs in `all` unless `--no-variable-sets`). Flags beyond the basics: `--dry-run` (import: plan only), `--fresh` (ignore run state), `--project SEG` / `--workspace NAME` (repeatable filters; apply maps `--workspace` to `-var workspacenames=[...]`, import works on a jq-filtered temp copy), `--skip-preflight`, `--no-secret-stubs`. Resolves each tool via `sg_resolve` (PATH first — the image installs them — else `sg_ensure_*` cache). `apply`/`all` first run `require_tfc_auth`, which resolves the token the tfe provider will use (`TFE_TOKEN`, `TF_TOKEN_<host>`, or the `terraform login` file for `tfHostname`) and verifies it with `GET /api/v2/account/details`, failing fast on a missing or rejected (expired) credential. Runs `convert` and `import` in parallel (`--concurrency`, default 4), raises `terraform apply -parallelism`, and retries `sg-cli` imports with backoff (`sg_retry`, `SG_RETRIES`). Import resolves each project's workflow group, checks existence via the SG API, shows a plan (`exists`/`create`), prompts (skip with `-y`), **creates the missing `tfc-<project>` groups via the API**, then imports. Imports go through `import_bulk`: workflows with Terraform variables via `sg-cli workflow create --bulk`, workflows **without** via a direct `POST .../wfs/` (`sg_create_workflow` + `sg_upload_tfstate` in `sg_api.sh`) because sg-cli drops an empty `iacInputData.data` and the API rejects the workflow — a workaround (`TODO(sg-cli)` markers) until sg-cli ships with sg-sdk-go >= v1.5.7, which sends the empty object; the direct path prints sg-cli-style `Failed to create <name>: <code>: <body>` lines so the parsing below covers both. State files are the migrator's job on both paths: `import_bulk` tracks sg-cli's per-workflow upload lines (`State file uploaded successfully` / `Failed to upload state file for <wf>`), re-uploads the failures via `upload_state` → `sg_upload_tfstate` (adds the `x-ms-blob-type: BlockBlob` header Azure Blob requires; sg-cli omits it and also treats anything but a literal `HTTP/1.1 200 OK` as failure — second `TODO(sg-cli)`), and writes `[state] uploaded|failed|none <wf>` markers into the parsed output; `do_import` turns them into `state_uploaded`/`state_failed` in the result, a workflow without its state fails the file (and `state_import_done` re-tries it next run). Existing workflows come back from sg-cli as `Failed to create <wf>: 409: Workflow ID not unique` (its update branch matches the outdated text `Workflow name not unique` — third `TODO(sg-cli)`); `import_bulk` collects those, PATCHes them via `sg_update_workflow`, re-uploads their state and drops the failure line, and `sg_create_workflow` does the same fallback on the direct path (prints `updated`). Before the parallel import, `probe_import` imports one workflow alone (first selected of the first file) and dies unless the create and the state upload both succeeded — the fail-fast for environment problems. sg-cli exits 0 even when individual workflows fail, so `do_import` parses its output: a workflow rejected as above SG's managed Terraform ceiling (1.5.7, last MPL/FOSS release) is re-imported with `SGDefaultTerraformVersion` (read from `terraform.tfvars`; an explicit `null` there means the `terraformVersion` key is deleted instead, so the org's execution preset decides — `tfvars_is_null` tells an explicit null from a missing key), the payload patched in place, and the case logged to `export/terraform-version-fallbacks.log` plus a printed notice; any other per-workflow failure fails the run. `cmd_import` reads the org's execution preset (`sg_execution_preset`, `GET /orgs/{org}/` → `Settings.workflowDefaults`) and `preset_labels` (report.sh) turns it into the `preset (1.5.7)` / `preset (private:rg)` cells of the plan and the `SG_TF_FALLBACK_LABEL` used in messages. The trigger pass skips workflows that do not exist in SG, and `sg_api_post` returns 22 on 4xx so `sg_retry` (via `SG_NO_RETRY_RC`) does not retry definitive errors. `clean` removes local artifacts (`export/`, TF state, tool cache); `clean --all` also removes config. `completion bash|zsh` prints a completion script (`cmd_completion`; commands/options come from `SG_COMMANDS`/`SG_OPTIONS`, keep them in sync with the parser and the host flags); like `clean` it always runs natively. Output is concise by default — `apply` captures terraform's init/plan output and shows only progress + the final summary (or the full log on failure, `TF_IN_AUTOMATION=1`/`-no-color`); `-v`/`--verbose` (`SG_VERBOSE`) streams everything and un-gates the convert detail lines. Colored logging via `sg_step`/`sg_log`/`sg_success`/`sg_warn`/`sg_err`/`sg_row`; paths shown relative via `sg_rel` (all auto-off when not a TTY / `NO_COLOR`). Long steps show a live progress line (`sg_run_quiet` for terraform init/apply, `run_parallel <fn> <max> <label> <items>` for convert/import/triggers — animated only when `SG_ANIMATE=1`, i.e. colors are on; plain log lines otherwise) and report their duration (`sg_fmt_secs`). Phases start with `phase_begin` — inside `all` the header is numbered (`Phase 2/5: ...`, `PHASE_TOTAL` set in `main`) and `phase_took` gives the elapsed time; `run_parallel` prints a job's output as-is when it is a single line and adds the `── <file> ──` header only for longer or failed output (always with `-v`). The import confirmation goes through `sg_confirm` (so `SG_ANSWERS_FILE` works; a declined prompt is a neutral "cancelled" line, not an error) and `import`/`all` end with `finish_line` (outcome, total time, open checklist items from `CHECKLIST_OPEN`). A Ctrl-C is trapped to clear the progress line and print a resume hint. - `scripts/lib/` — sourced libraries (a `lib/` gitignore rule is negated for this dir): `prompt.sh` (`sg_ask`/`sg_confirm`/`sg_select`; tty or `SG_ANSWERS_FILE` for tests; `SG_NONINTERACTIVE=1`/`-y`/no TTY → defaults), `tfvars.sh` (`tfvars_get` cached hcl2json reads, `tfvars_has`/`tfvars_is_null` to tell an explicit `null` from a missing key, `tfvars_write` renders the wizard's `W_*` vars — `W_TF_SOURCE`, and `W_TF_VERSION`/`W_RUNNER_JSON` may be the literal `null`; plain strings go through `_tfvars_str`, so quotes/backslashes in names stay valid HCL), `tfc_api.sh` (`tfc_http`/`tfc_get_all` paginated, `tfc_list_{orgs,projects,workspaces}` — workspaces include `vcs_provider`/`vcs_url`/`vcs_identifier` —, `tfc_select_workspaces` (the module's name-glob / include-tags(all) / exclude-tags filter, shared by wizard and preflight), `tfc_vcs_summary`/`tfc_vcs_kind_for`/`tfc_vcs_label_for`, `tfc_token` with provider precedence, `require_tfc_auth`), `sg_api.sh` (`_sg_api` 0/22/1 contract + `SG_HTTP_CODE`, `sg_list_integrations`, `sg_integration_type`, `sg_vcs_kind_of` (connector type → `sourceConfigDestKind`), `sg_execution_preset` + `sg_preset_{desc,runner_desc,version_desc,runner_provided}` (the org's execution preset and how to describe it), `sg_*_exists`, `sg_list_workflows`, `sg_create_secret`, `sg_patch_workflow`, `wfgroup_*`), `wizard.sh` (`wizard_run`: 4 steps TFC → SG → defaults → review; the TFC step keeps the workspace list and derives, via `tfc_vcs_summary`, which VCS provider the selected workspaces use and the repositories' URL prefix — the SG step lists matching connectors first and takes the repo URL prefix from TFC (never from a previous tfvars written for another provider kind), warns on a kind mismatch, reads the org's execution preset and offers it as the runner choice; the defaults step asks whether to carry TFC versions or use the preset (`SGTerraformVersionSource`) and, for carry, which fallback (`SGDefaultTerraformVersion` or the preset = null); the review shows human-readable rows plus the `tfc-<project>` groups; `_w_select_lines` feeds menus line-by-line so names with spaces/globs survive), `preflight.sh` (`preflight_run <apply|import|all>`, once per process; ✓ lines are plain language and name the connector's role/kind, ✗/! lines name the tfvars field to fix; besides existence it checks consistency — VCS kind vs. the connector's type (fail), repo URL prefix vs. where the TFC repositories live or the kind's well-known host (warn); it also reports the org's execution preset next to the version/runner policy and warns when `carry` meets a preset that mounts a runner-provided binary), `state.sh` (`.sg/state.json`: `phases.<name>.input_sha`, `import.<seg>.{imported,failed,tf_fallback,state_uploaded,state_failed}`, `triggers.<seg>`, `secrets.<name>`; `run_phase` in migrate.sh skips unchanged phases), `report.sh` (`show_migration_summary` after apply — incl. a state-files row —, `show_import_plan` per-workflow table with columns sized to their content), `errors.sh` (`explain_api_error` regex→hint table, `SG_API_HINTS`), `checklist.sh` (`create_secret_stubs` → SG secret `tfc-<wf>-<VAR>` = `CHANGE_ME`, referenced as `${secret::<name>}` via PATCH + payload patch; `write_checklist` → `export/post-import-checklist.md`, printed to the terminal as one ✓/! status line per section plus the imported groups with UI links — the file has the details; section 5 covers both the TFC state export and the SG state upload (`state_failed` from the import results); sets `CHECKLIST_OPEN`). Parallel jobs (`run_parallel`) run in subshells, so `do_import`/`do_set_triggers` write `.import-result.<seg>.json` / `.triggers-result.<seg>.json` into the export dir and the caller merges them into state. - `scripts/tools.sh` — sourced by the other scripts (sets `SG_REPO_ROOT` to its parent dir). Provides `sg_resolve` (PATH-or-cache), `sg_ensure_{jq,hcl2json,yajsv,sgcli}` (pinned downloads into `.sg/cached/`; `sg-cli` is the Go binary), `sg_retry`, the color vars + log helpers. The Docker image bundles these tools at build time (`yajsv` built from source so arm64 works), so in-container runs never download. diff --git a/README.md b/README.md index 1686818..bf6336f 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,9 @@ Migrate workloads from other platforms to [StackGuardian Platform](https://app.s `./sg-migrate.sh` runs the whole flow — `terraform apply` → HCL→JSON conversion → schema validation → bulk import — with all tooling (Terraform, `jq`, `hcl2json`, `yajsv`, `sg-cli`) isolated in a Docker image, so it behaves identically on Linux, macOS, and Windows. Docker is required for this path; without it the script automatically falls back to running natively (downloading pinned tools into `.sg/cached/`). ```shell +git clone https://github.com/StackGuardian/stackguardian-migrator.git +cd stackguardian-migrator + export TFE_TOKEN=<TFC/TFE token> # long-lived API token (User/Team/Org token from the TFC UI) export SG_API_TOKEN=<your SG token> export SG_ORG=<your SG org> @@ -27,13 +30,14 @@ export SG_ORG=<your SG org> That's it — no IDs to look up and no workflow-group mapping to fill in. `init` lists what the tokens can see (TFC organisations and workspaces, SG VCS/cloud connectors and runner groups, the org's execution preset) and writes `terraform.tfvars` from your picks — it also reads which VCS provider your TFC workspaces are connected to, lists the matching connectors first and takes the repository URL prefix from TFC. `all` verifies every reference **before** running terraform (preflight — including that the VCS connector, the VCS kind and the repo URL prefix agree with each other and with the TFC repositories), prints a migration summary after the export, shows a per-workflow import plan (create/update, Terraform version, runner, triggers, secrets), and ends with a **post-import checklist** of what still needs a human. Phases are numbered, long steps show a live progress line, and every phase reports how long it took. Each TFC project is imported into an SG workflow group named `tfc-<project>`, **created automatically via the API** if it doesn't exist. +- **Updating.** Clone the repo (don't fork it or download the release zip) and run `./sg-migrate.sh update` to pull the latest version; it fast-forwards the checkout and rebuilds the Docker image only if the `Dockerfile` changed. Your `terraform.tfvars`, `export/` and `.sg/` are never tracked, so they survive every update. To stay on a fixed release instead, `git checkout v1.2.2` (then `git checkout master` to follow the latest again). - Single phase: `./sg-migrate.sh preflight|apply|enrich|convert|validate|import|triggers|checklist`. Running `./sg-migrate.sh` with no command prints the help menu. - **Resume.** `all` remembers what it completed (`.sg/state.json`) and skips phases whose inputs have not changed, so after a failure you just re-run it; files already imported in full are skipped and files with failures are retried. `--fresh` redoes everything. - **Scope.** `--project <segment>` and `--workspace <name>` (repeatable) limit every phase to a subset — migrate one team first, then the rest. - **Dry run.** `./sg-migrate.sh import --dry-run` prints the per-workflow plan and stops; nothing is created. - **Sensitive variables** (which TFC never exposes) are recreated as SG secrets with the value `CHANGE_ME` and referenced from the workflows as `${secret::<name>}`; the checklist lists each one to fill in. Opt out with `--no-secret-stubs`. - TFC **Variable Set** variables are merged into the payloads automatically (the `enrich` phase, via the TFC API); skip it with `--no-variable-sets`. -- `./sg-migrate.sh clean` removes local working artifacts (`export/`, Terraform state, run state, tool cache) for a fresh start; add `--all` to also remove config. `clean` always runs locally. +- `./sg-migrate.sh clean` removes local working artifacts (`export/`, Terraform state, run state, tool cache) for a fresh start; add `--all` to also remove config. `clean` (like `update` and `completion`) always runs locally. - **Override** a project's target group (to reuse an existing group) in `.sg/workflow-groups.json`: `{"<project-segment>": "<existing-group>"}`. Override groups must already exist (they're not auto-created). - Output is concise by default (terraform's plan/init noise is hidden; shown on error). Add `-v`/`--verbose` for full output. Known API errors come with a hint naming the `terraform.tfvars` field to fix. - Flags: `-y` skip the import prompt (CI; also makes `init` non-interactive), `--concurrency N` parallel jobs, `--org NAME`, `--no-create-groups` require groups to pre-exist, `--skip-preflight`, `--build` rebuild the image, `--native`/`--local` force a local run even when Docker is available. diff --git a/scripts/migrate.sh b/scripts/migrate.sh index 895214b..471c1a1 100755 --- a/scripts/migrate.sh +++ b/scripts/migrate.sh @@ -98,6 +98,8 @@ Commands: tool cache). Add --all to also remove config (terraform.tfvars, mapping). completion Print a completion script for your shell (bash/zsh auto-detected): \`source <($PROG completion)\` + update Pull the latest version of the migrator (git pull --ff-only) and rebuild + the Docker image if the Dockerfile changed. Runs on the host. Each TFC project maps to an SG workflow group named tfc-<project>, created via the API if missing. Override a project's target group in .sg/workflow-groups.json @@ -983,7 +985,7 @@ finish_line() { # Single source of truth for shell completion (keep in sync with the parser below # and the host-only flags in sg-migrate.sh). -SG_COMMANDS="init preflight apply enrich convert validate import triggers checklist all clean completion" +SG_COMMANDS="init preflight apply enrich convert validate import triggers checklist all clean completion update" SG_OPTIONS="--org --export-dir --mapping --concurrency --no-create-groups --no-variable-sets --no-vcs-triggers --skip-preflight --dry-run --no-secret-stubs --fresh --project --workspace --all -v --verbose -y --yes -h --help --native --local --build" # cmd_completion <bash|zsh> — print a completion script for sg-migrate.sh / @@ -1139,6 +1141,11 @@ main() { cmd_completion "${2:-}" exit 0 ;; + update) + # Host-only: needs the checkout's git, not the container. + sg_err "'update' runs on the host: use ./sg-migrate.sh update" + exit 2 + ;; *) echo "Unknown argument: $1" >&2 usage diff --git a/sg-migrate.sh b/sg-migrate.sh index 9374ceb..54c622c 100755 --- a/sg-migrate.sh +++ b/sg-migrate.sh @@ -6,7 +6,8 @@ # credentials file read-only, and forwards SG_API_TOKEN/SG_ORG. # # Runs natively (no Docker) when: --native/--local is passed, SG_NATIVE=1 is set, -# the command is 'clean' (a local filesystem op), or docker is unavailable. +# the command is 'clean'/'completion'/'update' (local filesystem/git ops), or +# docker is unavailable. set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" @@ -27,11 +28,13 @@ for a in "$@"; do esac done -# Help, 'clean' and 'completion' only touch the local shell/filesystem — no -# container. With no command at all, migrate.sh prints the help menu. +# Help, 'clean', 'completion' and 'update' only touch the local shell/filesystem +# (or git) — no container. With no command at all, migrate.sh prints the help menu. HAS_CMD=0 +UPDATE=0 for a in ${ARGS[@]+"${ARGS[@]}"}; do case "$a" in + update) NATIVE=1; HAS_CMD=1; UPDATE=1 ;; clean | completion | -h | --help) NATIVE=1; HAS_CMD=1 ;; init | preflight | apply | enrich | convert | validate | import | triggers | checklist | all) HAS_CMD=1 ;; esac @@ -46,6 +49,61 @@ SG_SHELL="$(ps -p "$PPID" -o comm= 2>/dev/null | sed 's/^-//; s#.*/##')" case "$SG_SHELL" in bash | zsh) ;; *) SG_SHELL="$(basename "${SHELL:-zsh}")" ;; esac export SG_SHELL +# 'update' pulls the latest version of this checkout (fast-forward only) and +# rebuilds the image when the Dockerfile changed. Everything the user edits or +# generates (terraform.tfvars, export/, .sg/) is untracked, so a pull never +# touches it. Needs the host's git, hence never runs in the container. +cmd_update() { + local repo="https://github.com/StackGuardian/stackguardian-migrator.git" + local g=(git -C "$SCRIPT_DIR") + if ! "${g[@]}" rev-parse --git-dir >/dev/null 2>&1; then + sg_err "$SCRIPT_DIR is not a git checkout (downloaded archive?). Clone the repo instead, then updates are one command:" + sg_dim "git clone $repo" + exit 1 + fi + local branch + if ! branch="$("${g[@]}" symbolic-ref -q --short HEAD)"; then + sg_err "this checkout is pinned to $("${g[@]}" describe --tags --always 2>/dev/null) (detached HEAD); nothing to pull." + sg_dim "To follow the latest version: git checkout master && $SG_PROG update" + exit 1 + fi + local dirty + dirty="$("${g[@]}" status --porcelain --untracked-files=no)" + if [ -n "$dirty" ]; then + sg_err "local changes to tracked files block the update:" + printf '%s\n' "$dirty" | sed 's/^/ /' >&2 + sg_dim "Keep them with 'git stash' or discard with 'git checkout -- <file>', then re-run $SG_PROG update" + exit 1 + fi + local old new + old="$("${g[@]}" rev-parse HEAD)" + sg_step "Updating $branch" + if ! "${g[@]}" pull --ff-only; then + sg_err "git pull failed. If '$branch' has no upstream or local commits, reset it to the published branch: git checkout -B master origin/master" + exit 1 + fi + new="$("${g[@]}" rev-parse HEAD)" + if [ "$old" = "$new" ]; then + sg_success "already up to date (${new:0:7})" + return 0 + fi + sg_success "updated ${old:0:7}..${new:0:7}" + "${g[@]}" log --oneline --no-decorate "$old..$new" | sed 's/^/ /' >&2 + if [ -z "$("${g[@]}" diff --name-only "$old" "$new" -- Dockerfile)" ]; then + sg_dim "image $IMAGE unchanged (Dockerfile untouched)" + elif command -v docker >/dev/null 2>&1; then + sg_log "Dockerfile changed; rebuilding image $IMAGE ..." + docker build -t "$IMAGE" "$SCRIPT_DIR" + sg_success "image $IMAGE rebuilt" + else + sg_dim "Dockerfile changed, but docker is not available here; native runs need no rebuild" + fi +} +if [ "$UPDATE" = "1" ]; then + cmd_update + exit 0 +fi + if [ "$NATIVE" = "1" ] || ! command -v docker >/dev/null 2>&1; then [ "$NATIVE" = "1" ] || sg_warn "docker not found; running natively" exec "$SCRIPT_DIR/scripts/migrate.sh" ${ARGS[@]+"${ARGS[@]}"} From d360f6dacc8fa39ff2a057a6179f6179b88c6d2c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adis=20Halilovi=C4=87?= <adis.halilovic@stackguardian.io> Date: Mon, 14 Sep 2026 11:58:58 +0200 Subject: [PATCH 37/71] fix: force LF line endings; document the Windows (WSL 2) setup A Git-for-Windows clone with core.autocrlf turns the bash scripts into CRLF and they fail under WSL/Docker. --- .gitattributes | 3 +++ README.md | 10 ++++++++++ 2 files changed, 13 insertions(+) create mode 100644 .gitattributes diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..bba1bf5 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,3 @@ +# Keep LF everywhere: a Git-for-Windows clone with core.autocrlf would turn the +# bash scripts into CRLF and break them under WSL/Docker. +* text=auto eol=lf diff --git a/README.md b/README.md index bf6336f..97ca78d 100644 --- a/README.md +++ b/README.md @@ -54,6 +54,16 @@ The manual, step-by-step flow below remains supported for fine-grained control a - Terraform - [sg-cli](https://github.com/StackGuardian/sg-cli) +With the orchestrated flow the last two come from the Docker image; only `git` and Docker are needed on the host. + +### Windows (WSL 2) + +Run the migrator from a WSL 2 distribution (Ubuntu from the Microsoft Store is fine); nothing is needed on the Windows side. + +- Install [Docker Desktop](https://docs.docker.com/desktop/features/wsl/) with the WSL 2 backend and enable *WSL integration* for your distro (Settings → Resources → WSL integration). `docker` is then on the PATH inside WSL and `./sg-migrate.sh` works unchanged. +- Clone inside the Linux filesystem (e.g. `~/stackguardian-migrator`), not under `/mnt/c/...`: bind mounts from the Windows drive are slow, and a Git-for-Windows checkout may convert the scripts to CRLF line endings. +- Set `TFE_TOKEN` (or run `terraform login`) **inside WSL**. The wrapper mounts `~/.terraform.d/credentials.tfrc.json` from the WSL home; a `terraform login` done on the Windows side is not seen. + ### Authenticate to Terraform Cloud/Enterprise Set `TFE_TOKEN` to a long-lived API token (create one under **User Settings → Tokens**, or use a Team/Organization token) — this is the recommended path and avoids session expiry. Alternatively run `terraform login`, which writes `~/.terraform.d/credentials.tfrc.json`. The `tfe` provider, the API state export, and variable-set enrichment all use whichever is present. From 76bdb857c2b2c96e413c4e578e53a86789b9267c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adis=20Halilovi=C4=87?= <adis.halilovic@stackguardian.io> Date: Mon, 14 Sep 2026 12:33:13 +0200 Subject: [PATCH 38/71] =?UTF-8?q?feat:=20projectOverrides=20=E2=80=94=20pe?= =?UTF-8?q?r-TFC-project=20connectors,=20runners,=20approvers=20and=20work?= =?UTF-8?q?flow=20group?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Connectors and the other SGDefault* values could only be set globally or per workspace. projectOverrides, keyed by the TFC project name, applies to every workspace of a project; precedence is workspaceOverrides > projectOverrides > SGDefault*. Its workflowGroup field picks the StackGuardian workflow group for the whole project (default tfc-<project>) and is written into every payload entry's CLIConfiguration.WorkflowGroup.name, so the importer has one source of truth; the payload file keeps its sg-payload.<project>.json name. Overrides now resolve once per workspace into local.effective (each layer null-filtered before merge()). Object-shaped fields are picked with the tuple idiom already used for RunnerConstraints, since a partial VCSTriggers or DeploymentPlatformConfig override does not unify with the derived value in a conditional. Both override maps are typed any with a field-name validation: map(object({... = optional(any)})) rejects entries whose object-shaped fields differ ("attribute types must all match for conversion to map"), which already bit workspaceOverrides as soon as two workspaces set different fields. The summary gains projects (name -> segment, group, count), workspaceProjects, workflowGroups and unknownProjectOverrides (keys that match no TFC project, also flagged in migration-summary.md). --- CLAUDE.md | 2 +- .../terraform-cloud/example_payload.jsonc | 6 +- transformer/terraform-cloud/locals.tf | 97 ++++++++++++++----- transformer/terraform-cloud/summary.tmpl | 7 +- .../terraform-cloud/terraform.tfvars.example | 20 +++- transformer/terraform-cloud/variables.tf | 48 ++++++--- 6 files changed, 139 insertions(+), 41 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 6492119..8624927 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -42,7 +42,7 @@ The whole transformation lives in `locals.tf` — there are no `outputs.tf`/`mai - `terraform_version` → with `SGTerraformVersionSource = "carry"` (default) `TERRAFORM-<version>` for a pinned semver, otherwise `SGDefaultTerraformVersion`; with `"preset"`, or when that default is `null`, the `terraformVersion` key is **left out** of `TerraformConfig` so the SG API fills it from the org's **execution preset** (`Settings.workflowDefaults`, UI: Settings → Runner groups → Execution presets; platform default: managed Terraform 1.5.7). The API only fills keys that are absent, never `null`/`""`, so the transformer filters nulls out (`local.tfVersion`). Likewise `RunnerConstraints` is omitted when `SGDefaultRunnerConstraints = null` (`local.runnerConstraints`, picked via a tuple because an override with `names` and a default without it are different object types). The summary records `terraformVersionSource`, `terraformVersionDefault`, `runnerConstraintsSource` and `tfcTerraformVersions` so later phases can explain what each workflow runs. - `project_id` → `CLIConfiguration.WorkflowGroup.name` = `tfc-<project-segment>` (matches the per-project filename and the group the importer creates/targets). - Sensitive variables (terraform + env) are skipped and recorded in the summary. - - Per-workspace `var.workspaceOverrides[<name>]` fields take precedence over the `SGDefault*` values (resolved via `try(var.workspaceOverrides[name].<field>, null) != null ? ... : <default>`). + - Overrides resolve once per workspace into `local.effective[<name>]` (workspaceOverrides > projectOverrides[<raw TFC project name>] > null = the `SGDefault*` value decides; each layer is null-filtered before `merge()`). Both maps are typed `any` with a field-name validation, because `map(object({... = optional(any)}))` refuses entries whose object-shaped fields differ. Object-shaped fields (`DeploymentPlatformConfig`, `RunnerConstraints`, `VCSTriggers`) are picked with the tuple idiom, not `? :`, since a partial override object does not unify with the derived one. `projectOverrides[<project>].workflowGroup` sets `CLIConfiguration.WorkflowGroup.name` for the whole project (default `tfc-<segment>`; `local.projectGroups`/`local.workflowGroups`); the summary records `projects` (name → segment/group/count), `workspaceProjects`, `workflowGroups` and `unknownProjectOverrides` (keys matching no TFC project). - `local.resourceNames` sanitizes workspace names to a valid SG `ResourceName` (≤100 chars, `^[-a-zA-Z0-9_]+$`, collision-disambiguated). For normal TFC names this is a no-op; any actual rename is reported in the summary. This is the single place to adjust naming rules. - `resources.tf` — writes one `sg-payload.<project>.json` per project directly via `for_each` (no `mv`), plus `migration-summary.{md,json}`. When `exportStateFiles=true`, `null_resource.exportState` pulls each workspace's state **directly from the TFC/TFE API** (`GET /api/v2/workspaces/{id}/current-state-version` → `hosted-state-download-url`) via a `local-exec` `curl`/`jq` script — no `terraform init` or providers per workspace (avoids the plugin-cache concurrency bug and per-workspace provider downloads). The token is read at runtime from `~/.terraform.d/credentials.tfrc.json` (the `terraform login` file) or `TFE_TOKEN`, so it never enters TF state. Idempotent (keyed by workspace name/id; `forceStateRefresh` re-pulls), with per-workspace failures (no token / no state / download error) recorded in `state-export-failures.log` instead of aborting. `cmd_apply` ensures `jq`/`curl` are on PATH for the apply. - `summary.tmpl` — renders `local.summary` to `migration-summary.md`. diff --git a/transformer/terraform-cloud/example_payload.jsonc b/transformer/terraform-cloud/example_payload.jsonc index 7113182..977a67f 100644 --- a/transformer/terraform-cloud/example_payload.jsonc +++ b/transformer/terraform-cloud/example_payload.jsonc @@ -3,15 +3,15 @@ "CLIConfiguration": { "TfStateFilePath": "", // path to the state file "WorkflowGroup": { - "name": "" // workflowgroup name - will be created if it does not exist + "name": "" // workflow group: tfc-<project>, or projectOverrides[<project>].workflowGroup; reused if it exists, created otherwise } }, "DeploymentPlatformConfig": [ { - "kind": "AWS_RBAC", // value corresponds to "AWS_STATIC", "GCP_STATIC" , "AWS_RBAC" OR "AZURE_STATIC" + "kind": "AWS_RBAC", // one of AWS_STATIC, AWS_RBAC, AWS_OIDC, AZURE_STATIC, AZURE_OIDC, AZURE_MANAGED_ID_OIDC, GCP_STATIC, GCP_OIDC "config": { "integrationId": "/integrations/aws-rbac", // value corresponds to /integrations/<your-integration-from-stackguardian> - "profileName": "default" // // value corresponds to the profile if "AWS_RBAC" + "profileName": "default" // AWS only: the profile name for AWS_RBAC } } ], diff --git a/transformer/terraform-cloud/locals.tf b/transformer/terraform-cloud/locals.tf index 471f68e..a5eeb98 100644 --- a/transformer/terraform-cloud/locals.tf +++ b/transformer/terraform-cloud/locals.tf @@ -7,6 +7,36 @@ locals { # to set a human-readable WorkflowGroup name in the payload. projectNames = { for p in data.tfe_projects.data.projects : p.id => p.name } + # TFC project per workspace: id, and the raw name (falls back to the id when + # the project is not visible to the token). + workflowProject = { for name in local.workflowNames : name => data.tfe_workspace.data[name].project_id } + workspaceProjects = { for name, pid in local.workflowProject : name => try(local.projectNames[pid], pid) } + + # Effective override per workspace: workspaceOverrides > projectOverrides (by + # the workspace's project name) > null, where null means "the SGDefault* + # value decides". Null attributes are dropped from each layer before merge() + # - merge() does not skip them - and the all-null base makes every + # local.effective[name].<field> safe to reference. + effective = { + for name in local.workflowNames : + name => merge( + { for f in local.overrideFieldNames : f => null }, + { for k, v in try(var.projectOverrides[local.workspaceProjects[name]], {}) : k => v if v != null }, + { for k, v in try(var.workspaceOverrides[name], {}) : k => v if v != null }, + ) + } + + # projectOverrides keys that name no project of the TFC org (typo guard). + unknownProjectOverrides = sort([for k in keys(var.projectOverrides) : k if !contains(values(local.projectNames), k)]) + + # Effective cloud connector per workflow. Picked via a tuple, not a + # conditional: an AWS and an Azure DeploymentPlatformConfig have different + # config shapes. + deploymentPlatformConfig = { + for name in local.workflowNames : + name => try([for c in [local.effective[name].DeploymentPlatformConfig, var.SGDefaultDeploymentPlatformConfig] : c if c != null][0], var.SGDefaultDeploymentPlatformConfig) + } + # SG workflow-name (ResourceName) sanitization. Per the SG OpenAPI spec, # ResourceName must be 1-100 chars; SG's name convention is ^[-a-zA-Z0-9_]+$. # TFC workspace names already satisfy both, so for normal inputs this is a @@ -47,7 +77,7 @@ locals { versionFallbacks = { for name in local.workflowNames : name => data.tfe_workspace.data[name].terraform_version - if !can(regex("^[0-9]+\\.[0-9]+\\.[0-9]+$", data.tfe_workspace.data[name].terraform_version)) && try(var.workspaceOverrides[name].terraformVersion, null) == null + if !can(regex("^[0-9]+\\.[0-9]+\\.[0-9]+$", data.tfe_workspace.data[name].terraform_version)) && local.effective[name].terraformVersion == null } # Terraform version sent per workflow. An override is always sent as-is. @@ -58,7 +88,7 @@ locals { tfVersion = { for name in local.workflowNames : name => ( - try(var.workspaceOverrides[name].terraformVersion, null) != null ? var.workspaceOverrides[name].terraformVersion : + local.effective[name].terraformVersion != null ? local.effective[name].terraformVersion : var.SGTerraformVersionSource == "preset" ? null : can(regex("^[0-9]+\\.[0-9]+\\.[0-9]+$", data.tfe_workspace.data[name].terraform_version)) ? "TERRAFORM-${data.tfe_workspace.data[name].terraform_version}" : var.SGDefaultTerraformVersion @@ -72,7 +102,7 @@ locals { runnerConstraints = { for name in local.workflowNames : name => try([for c in [ - try(var.workspaceOverrides[name].RunnerConstraints, null), + local.effective[name].RunnerConstraints, var.SGDefaultRunnerConstraints == null ? null : { for k, v in var.SGDefaultRunnerConstraints : k => v if v != null } ] : c if c != null][0], null) } @@ -89,18 +119,20 @@ locals { # Drives both the source config kind and which workspaces get VCS triggers. sourceKind = { for name in local.workflowNames : - name => try(var.workspaceOverrides[name].sourceConfigDestKind, null) != null ? var.workspaceOverrides[name].sourceConfigDestKind : var.SGDefaultSourceConfigDestKind + name => local.effective[name].sourceConfigDestKind != null ? local.effective[name].sourceConfigDestKind : var.SGDefaultSourceConfigDestKind } - # One SG workflow payload per workspace. Per-workspace overrides win over the - # SGDefault* values; everything else is derived from the TFC workspace. + # One SG workflow payload per workspace. Overrides (workspace, then project) + # win over the SGDefault* values; everything else is derived from the TFC + # workspace. workflowPayload = { for wsName, wsId in data.tfe_workspace_ids.data.ids : wsName => merge({ CLIConfiguration = { "WorkflowGroup" : { - # SG workflow group per TFC project: tfc-<project> (matches the group - # the importer creates/targets and the per-project payload filename). - "name" : "tfc-${local.projectFileSegment[data.tfe_workspace.data[wsName].project_id]}" + # SG workflow group per TFC project: projectOverrides[<project>].workflowGroup, + # else tfc-<project>. The importer reads it from here (reused when it + # exists, created otherwise); the payload file is still named by project. + "name" : local.workflowGroups[wsName] }, "TfStateFilePath" : "${abspath(path.root)}/../../${var.exportPath}/states/${data.tfe_workspace.data[wsName].name}.tfstate" } @@ -111,10 +143,10 @@ locals { [for v in data.tfe_variables.data[wsId].variables : { "config" : { "textValue" : v.value, "varName" : v.name }, "kind" : "PLAIN_TEXT" } if v.category == "env" && v.sensitive == false && !anytrue([for p in var.ignoreVarPatterns : can(regex(p, v.name))])], - try(var.workspaceOverrides[wsName].extraEnvironmentVariables, []) + local.effective[wsName].extraEnvironmentVariables != null ? local.effective[wsName].extraEnvironmentVariables : [] ) - DeploymentPlatformConfig = try(var.workspaceOverrides[wsName].DeploymentPlatformConfig, null) != null ? var.workspaceOverrides[wsName].DeploymentPlatformConfig : var.SGDefaultDeploymentPlatformConfig + DeploymentPlatformConfig = local.deploymentPlatformConfig[wsName] VCSConfig = { "iacVCSConfig" : { @@ -125,9 +157,9 @@ locals { "includeSubModule" : false, "ref" : length(data.tfe_workspace.data[wsName].vcs_repo) > 0 ? data.tfe_workspace.data[wsName].vcs_repo[0].branch : "", "isPrivate" : length(data.tfe_workspace.data[wsName].vcs_repo) > 0 ? length(data.tfe_workspace.data[wsName].vcs_repo[0].oauth_token_id) > 0 || length(data.tfe_workspace.data[wsName].vcs_repo[0].github_app_installation_id) > 0 : false, - "auth" : length(data.tfe_workspace.data[wsName].vcs_repo) > 0 ? (length(data.tfe_workspace.data[wsName].vcs_repo[0].oauth_token_id) > 0 || length(data.tfe_workspace.data[wsName].vcs_repo[0].github_app_installation_id) > 0 ? (try(var.workspaceOverrides[wsName].vcsAuthIntegrationID, null) != null ? var.workspaceOverrides[wsName].vcsAuthIntegrationID : var.SGDefaultVCSAuthIntegrationID) : "") : "", + "auth" : length(data.tfe_workspace.data[wsName].vcs_repo) > 0 ? (length(data.tfe_workspace.data[wsName].vcs_repo[0].oauth_token_id) > 0 || length(data.tfe_workspace.data[wsName].vcs_repo[0].github_app_installation_id) > 0 ? (local.effective[wsName].vcsAuthIntegrationID != null ? local.effective[wsName].vcsAuthIntegrationID : var.SGDefaultVCSAuthIntegrationID) : "") : "", "workingDir" : data.tfe_workspace.data[wsName].working_directory, - "repo" : length(data.tfe_workspace.data[wsName].vcs_repo) > 0 ? format("%s/%s", try(var.workspaceOverrides[wsName].vcsRepoPrefix, null) != null ? var.workspaceOverrides[wsName].vcsRepoPrefix : var.SGDefaultIACVCSRepoPrefix, data.tfe_workspace.data[wsName].vcs_repo[0].identifier) : "" + "repo" : length(data.tfe_workspace.data[wsName].vcs_repo) > 0 ? format("%s/%s", local.effective[wsName].vcsRepoPrefix != null ? local.effective[wsName].vcsRepoPrefix : var.SGDefaultIACVCSRepoPrefix, data.tfe_workspace.data[wsName].vcs_repo[0].identifier) : "" } } }, @@ -145,8 +177,10 @@ locals { # cmd_triggers), which is what actually registers the repo webhook. Field # shape matches a real SG workflow + the landfast set_vcs_triggers call; # only the truly server-assigned fields (gh_webhook_url, *_hook_id, - # github_app_installation_id) are omitted. - VCSTriggers = try(var.workspaceOverrides[wsName].VCSTriggers, null) != null ? var.workspaceOverrides[wsName].VCSTriggers : ( + # github_app_installation_id) are omitted. An override replaces the remap + # entirely; picked via a tuple because an override object rarely has the + # exact attribute set of the derived one (a conditional would refuse). + VCSTriggers = try([for c in [local.effective[wsName].VCSTriggers, ( var.SGDefaultEnableVCSTriggers && length(data.tfe_workspace.data[wsName].vcs_repo) > 0 && contains(["GITHUB_COM", "GITLAB_COM", "BITBUCKET_ORG", "AZURE_DEVOPS"], local.sourceKind[wsName]) @@ -174,7 +208,7 @@ locals { ) } : null - ) + )] : c if c != null][0], null) MiniSteps = { "wfChaining" : { @@ -191,7 +225,7 @@ locals { } } - Approvers = try(var.workspaceOverrides[wsName].Approvers, null) != null ? var.workspaceOverrides[wsName].Approvers : (data.tfe_workspace.data[wsName].auto_apply ? [] : var.SGDefaultWfApprovers) + Approvers = local.effective[wsName].Approvers != null ? local.effective[wsName].Approvers : (data.tfe_workspace.data[wsName].auto_apply ? [] : var.SGDefaultWfApprovers) # terraformVersion is omitted (not null) when the execution preset should # decide: the SG API only fills in keys that are absent from the payload. @@ -209,8 +243,7 @@ locals { # Group payloads by TFC project so each project imports into its own SG # workflow group (the bulk import takes a single --workflow-group per file). - workflowProject = { for wsName, wsId in data.tfe_workspace_ids.data.ids : wsName => data.tfe_workspace.data[wsName].project_id } - projectsUsed = toset(values(local.workflowProject)) + projectsUsed = toset(values(local.workflowProject)) payloadByProject = { for pid in local.projectsUsed : @@ -223,14 +256,34 @@ locals { pid => replace(lower(try(local.projectNames[pid], pid)), "/[^a-z0-9-]+/", "-") } + # SG workflow group per project: projectOverrides[<name>].workflowGroup, else + # tfc-<segment>; and the same per workspace for the payload and the summary. + projectGroups = { + for pid in local.projectsUsed : + pid => try(var.projectOverrides[try(local.projectNames[pid], pid)].workflowGroup, null) != null ? var.projectOverrides[try(local.projectNames[pid], pid)].workflowGroup : "tfc-${local.projectFileSegment[pid]}" + } + workflowGroups = { for name, pid in local.workflowProject : name => local.projectGroups[pid] } + # Machine-readable migration summary (also rendered to markdown). summary = { organization = var.tfOrg workspaceCount = length(local.workflowNames) projectWorkspaceCounts = { for pid in local.projectsUsed : try(local.projectNames[pid], pid) => length(local.payloadByProject[pid]) } - skippedSensitiveVars = { for name, vars in local.sensitiveVars : name => vars if length(vars) > 0 } - strippedVars = { for name, vars in local.strippedVars : name => vars if length(vars) > 0 } - ignoreVarPatterns = var.ignoreVarPatterns + # Per project (raw TFC name): payload file segment, SG workflow group, size. + projects = { + for pid in local.projectsUsed : + try(local.projectNames[pid], pid) => { + segment = local.projectFileSegment[pid] + workflowGroup = local.projectGroups[pid] + workspaceCount = length(local.payloadByProject[pid]) + } + } + workspaceProjects = local.workspaceProjects # ws => raw TFC project name + workflowGroups = local.workflowGroups # ws => SG workflow group + unknownProjectOverrides = local.unknownProjectOverrides + skippedSensitiveVars = { for name, vars in local.sensitiveVars : name => vars if length(vars) > 0 } + strippedVars = { for name, vars in local.strippedVars : name => vars if length(vars) > 0 } + ignoreVarPatterns = var.ignoreVarPatterns # Version policy, so the later phases can explain what each workflow runs. terraformVersionSource = var.SGTerraformVersionSource terraformVersionDefault = var.SGDefaultTerraformVersion # null = the execution preset decides diff --git a/transformer/terraform-cloud/summary.tmpl b/transformer/terraform-cloud/summary.tmpl index d7df0de..ba257b2 100644 --- a/transformer/terraform-cloud/summary.tmpl +++ b/transformer/terraform-cloud/summary.tmpl @@ -4,9 +4,12 @@ - Workspaces processed: ${summary.workspaceCount} ## Workflows per project -%{ for project, count in summary.projectWorkspaceCounts ~} -- ${project}: ${count} +%{ for project, info in summary.projects ~} +- ${project}: ${info.workspaceCount} -> workflow group ${info.workflowGroup} (sg-payload.${info.segment}.json) %{ endfor ~} +%{ if length(summary.unknownProjectOverrides) > 0 ~} +- WARNING: projectOverrides keys matching no TFC project (ignored): ${join(", ", summary.unknownProjectOverrides)} +%{ endif ~} ## Skipped sensitive variables These are not migrated (TFC never returns sensitive values). Recreate them as SG secrets after import. diff --git a/transformer/terraform-cloud/terraform.tfvars.example b/transformer/terraform-cloud/terraform.tfvars.example index e20789c..27b1864 100644 --- a/transformer/terraform-cloud/terraform.tfvars.example +++ b/transformer/terraform-cloud/terraform.tfvars.example @@ -75,8 +75,26 @@ SGDefaultEnableVCSTriggers = true # Re-pull state for every workspace on each apply. Leave false for idempotent runs. forceStateRefresh = false +# Per-project settings, keyed by the TFC project name (as shown in TFC). Every +# workspace of the project gets them; precedence is +# workspaceOverrides > projectOverrides > SGDefault*. +# workflowGroup picks the StackGuardian workflow group for the whole project +# (default tfc-<project>): an existing group is reused, a missing one created. +# projectOverrides = { +# "platform-azure" = { +# workflowGroup = "azure-landing-zone" +# DeploymentPlatformConfig = [ +# { "kind" : "AZURE_OIDC", "config" : { "integrationId" : "/integrations/azure-prod" } } +# ] +# vcsAuthIntegrationID = "/integrations/github_platform" +# Approvers = ["platform-lead@example.com"] +# RunnerConstraints = { "type" : "private", "names" : ["azure-runners"] } +# } +# } + # Per-workspace overrides, keyed by workspace name. Any field set here wins over -# the SGDefault* value above, for that workspace only. All fields are optional. +# the projectOverrides and SGDefault* values, for that workspace only. All +# fields are optional. # workspaceOverrides = { # "prod-networking" = { # DeploymentPlatformConfig = [ diff --git a/transformer/terraform-cloud/variables.tf b/transformer/terraform-cloud/variables.tf index c265db6..eb5ce56 100644 --- a/transformer/terraform-cloud/variables.tf +++ b/transformer/terraform-cloud/variables.tf @@ -130,18 +130,42 @@ variable "forceStateRefresh" { type = bool } +# The override maps are typed "any" on purpose: with map(object({ ... = +# optional(any) })) Terraform requires every entry to resolve each attribute to +# one common type, so one workspace setting RunnerConstraints while another +# omits it fails with "attribute types must all match for conversion to map". +# The validations below check the field names instead; the values are shaped +# like the SGDefault* variables they override. +locals { + overrideFieldNames = [ + "DeploymentPlatformConfig", # list, like SGDefaultDeploymentPlatformConfig + "RunnerConstraints", # object { type, names }, like SGDefaultRunnerConstraints + "Approvers", # list(string) + "vcsAuthIntegrationID", # string + "vcsRepoPrefix", # string + "sourceConfigDestKind", # string + "terraformVersion", # string, sent as-is (e.g. TERRAFORM-1.7.5) + "extraEnvironmentVariables", # list of SG EnvironmentVariables entries + "VCSTriggers", # object, replaces the derived triggers entirely + ] +} + variable "workspaceOverrides" { default = {} - description = "Per-workspace overrides keyed by TFC/TFE workspace name. Any field set here takes precedence over the matching SGDefault* value for that workspace only." - type = map(object({ - DeploymentPlatformConfig = optional(list(any)) - RunnerConstraints = optional(any) - Approvers = optional(list(string)) - vcsAuthIntegrationID = optional(string) - vcsRepoPrefix = optional(string) - sourceConfigDestKind = optional(string) - terraformVersion = optional(string) - extraEnvironmentVariables = optional(list(any), []) - VCSTriggers = optional(any) - })) + description = "Per-workspace overrides keyed by TFC/TFE workspace name. Any field set here wins over the matching projectOverrides and SGDefault* values for that workspace only. Fields: DeploymentPlatformConfig, RunnerConstraints, Approvers, vcsAuthIntegrationID, vcsRepoPrefix, sourceConfigDestKind, terraformVersion, extraEnvironmentVariables, VCSTriggers (see terraform.tfvars.example)." + type = any + validation { + condition = can([for name, o in var.workspaceOverrides : keys(o)]) && alltrue([for name, o in var.workspaceOverrides : length(setsubtract(keys(o), ["DeploymentPlatformConfig", "RunnerConstraints", "Approvers", "vcsAuthIntegrationID", "vcsRepoPrefix", "sourceConfigDestKind", "terraformVersion", "extraEnvironmentVariables", "VCSTriggers"])) == 0]) + error_message = "workspaceOverrides must map workspace names to objects with only these fields: DeploymentPlatformConfig, RunnerConstraints, Approvers, vcsAuthIntegrationID, vcsRepoPrefix, sourceConfigDestKind, terraformVersion, extraEnvironmentVariables, VCSTriggers." + } +} + +variable "projectOverrides" { + default = {} + description = "Per-project overrides keyed by the TFC/TFE project name (as shown in TFC, case-sensitive). Applied to every workspace of that project: workspaceOverrides win over these, these win over the SGDefault* values. Same fields as workspaceOverrides plus workflowGroup, which replaces the default StackGuardian workflow group tfc-<project> for the whole project (the payload file keeps its sg-payload.<project>.json name). Keys that match no project are listed in migration-summary.md (unknownProjectOverrides)." + type = any + validation { + condition = can([for name, o in var.projectOverrides : keys(o)]) && alltrue([for name, o in var.projectOverrides : length(setsubtract(keys(o), ["DeploymentPlatformConfig", "RunnerConstraints", "Approvers", "vcsAuthIntegrationID", "vcsRepoPrefix", "sourceConfigDestKind", "terraformVersion", "extraEnvironmentVariables", "VCSTriggers", "workflowGroup"])) == 0]) + error_message = "projectOverrides must map TFC project names to objects with only these fields: DeploymentPlatformConfig, RunnerConstraints, Approvers, vcsAuthIntegrationID, vcsRepoPrefix, sourceConfigDestKind, terraformVersion, extraEnvironmentVariables, VCSTriggers, workflowGroup." + } } From 6a244b4d0d4481bd9c0ecc9ad4e80d531f3d34fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adis=20Halilovi=C4=87?= <adis.halilovic@stackguardian.io> Date: Mon, 14 Sep 2026 12:33:40 +0200 Subject: [PATCH 39/71] feat: strip cloud credential env vars the connector replaces (stripCloudAuthVars) TFC workspaces carry ARM_*, AWS_ACCESS_KEY_ID, GOOGLE_CREDENTIALS and the like; in StackGuardian the cloud connector provides these, and migrated copies fight it. Only TFC_*/TFE_* were stripped so far. stripCloudAuthVars (default true) drops the credential env variables of the family matching each workflow's effective connector kind (AWS_* -> AWS, AZURE_* -> AZURE, GCP_* -> GCP); cloudAuthVarPatterns holds the regexes per family and can be replaced. Env variables only; terraform inputs are untouched. A sensitive credential is stripped as well, so the post-import step does not create a CHANGE_ME placeholder secret that would shadow the connector. Stripped variables are listed in the summary (strippedCloudAuthVars, with the connector kind) and the enrich step applies the same rule to variable-set variables. --- CLAUDE.md | 1 + scripts/enrich_variable_sets.sh | 45 ++++++++++++++++--- .../terraform-cloud/example_payload.jsonc | 2 +- transformer/terraform-cloud/locals.tf | 33 +++++++++++--- transformer/terraform-cloud/summary.tmpl | 14 ++++++ .../terraform-cloud/terraform.tfvars.example | 13 ++++++ transformer/terraform-cloud/variables.tf | 30 +++++++++++++ 7 files changed, 126 insertions(+), 12 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 8624927..f185a21 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -43,6 +43,7 @@ The whole transformation lives in `locals.tf` — there are no `outputs.tf`/`mai - `project_id` → `CLIConfiguration.WorkflowGroup.name` = `tfc-<project-segment>` (matches the per-project filename and the group the importer creates/targets). - Sensitive variables (terraform + env) are skipped and recorded in the summary. - Overrides resolve once per workspace into `local.effective[<name>]` (workspaceOverrides > projectOverrides[<raw TFC project name>] > null = the `SGDefault*` value decides; each layer is null-filtered before `merge()`). Both maps are typed `any` with a field-name validation, because `map(object({... = optional(any)}))` refuses entries whose object-shaped fields differ. Object-shaped fields (`DeploymentPlatformConfig`, `RunnerConstraints`, `VCSTriggers`) are picked with the tuple idiom, not `? :`, since a partial override object does not unify with the derived one. `projectOverrides[<project>].workflowGroup` sets `CLIConfiguration.WorkflowGroup.name` for the whole project (default `tfc-<segment>`; `local.projectGroups`/`local.workflowGroups`); the summary records `projects` (name → segment/group/count), `workspaceProjects`, `workflowGroups` and `unknownProjectOverrides` (keys matching no TFC project). + - `stripCloudAuthVars` (default true) drops the cloud credential **env** vars the workflow's connector replaces: the family is the prefix of the effective `DeploymentPlatformConfig[0].kind` (`AWS`/`AZURE`/`GCP`), the patterns come from `cloudAuthVarPatterns` (per family; setting it replaces the whole map). Stripped even when sensitive, so no placeholder secret is created for them; recorded in `strippedCloudAuthVars` + `workspaceCloudKinds`. `scripts/enrich_variable_sets.sh` applies the same rule to variable-set env vars using the payload's connector kind (defaults duplicated there — keep in sync). - `local.resourceNames` sanitizes workspace names to a valid SG `ResourceName` (≤100 chars, `^[-a-zA-Z0-9_]+$`, collision-disambiguated). For normal TFC names this is a no-op; any actual rename is reported in the summary. This is the single place to adjust naming rules. - `resources.tf` — writes one `sg-payload.<project>.json` per project directly via `for_each` (no `mv`), plus `migration-summary.{md,json}`. When `exportStateFiles=true`, `null_resource.exportState` pulls each workspace's state **directly from the TFC/TFE API** (`GET /api/v2/workspaces/{id}/current-state-version` → `hosted-state-download-url`) via a `local-exec` `curl`/`jq` script — no `terraform init` or providers per workspace (avoids the plugin-cache concurrency bug and per-workspace provider downloads). The token is read at runtime from `~/.terraform.d/credentials.tfrc.json` (the `terraform login` file) or `TFE_TOKEN`, so it never enters TF state. Idempotent (keyed by workspace name/id; `forceStateRefresh` re-pulls), with per-workspace failures (no token / no state / download error) recorded in `state-export-failures.log` instead of aborting. `cmd_apply` ensures `jq`/`curl` are on PATH for the apply. - `summary.tmpl` — renders `local.summary` to `migration-summary.md`. diff --git a/scripts/enrich_variable_sets.sh b/scripts/enrich_variable_sets.sh index 8a653a2..5ef520c 100755 --- a/scripts/enrich_variable_sets.sh +++ b/scripts/enrich_variable_sets.sh @@ -112,18 +112,35 @@ sg_log "resolving $set_count variable set(s) across workspaces:" IGNORE_JSON="$(tfvars_get_json .ignoreVarPatterns)" [ "$IGNORE_JSON" = "null" ] && IGNORE_JSON='["^TFC_","^TFE_"]' +# Cloud credential env vars are stripped per workflow like the transformer does: +# the payload's DeploymentPlatformConfig[0].kind gives the family (AWS/AZURE/GCP), +# the patterns come from terraform.tfvars. Keep the defaults in sync with +# variables.tf (cloudAuthVarPatterns). jq (Oniguruma) and Terraform (RE2) agree +# on the anchored/prefix patterns used here. +CLOUD_AUTH_DEFAULTS='{"AWS":["^AWS_ACCESS_KEY_ID$","^AWS_SECRET_ACCESS_KEY$","^AWS_SESSION_TOKEN$","^AWS_PROFILE$","^AWS_ROLE_ARN$","^AWS_WEB_IDENTITY_TOKEN_FILE$","^AWS_SHARED_CREDENTIALS_FILE$","^AWS_CONFIG_FILE$"],"AZURE":["^ARM_CLIENT_ID$","^ARM_CLIENT_SECRET$","^ARM_TENANT_ID$","^ARM_SUBSCRIPTION_ID$","^ARM_USE_OIDC$","^ARM_OIDC_","^ARM_CLIENT_CERTIFICATE","^ARM_USE_MSI$","^ARM_MSI_ENDPOINT$"],"GCP":["^GOOGLE_CREDENTIALS$","^GOOGLE_APPLICATION_CREDENTIALS$","^GOOGLE_OAUTH_ACCESS_TOKEN$","^GOOGLE_IMPERSONATE_SERVICE_ACCOUNT$","^CLOUDSDK_AUTH_"]}' +CLOUD_JSON="$(tfvars_get_json .cloudAuthVarPatterns)" +[ "$CLOUD_JSON" = "null" ] && CLOUD_JSON="$CLOUD_AUTH_DEFAULTS" +[ "$(tfvars_get .stripCloudAuthVars true)" != "false" ] || CLOUD_JSON='{}' + +# workspace name -> cloud family, from the payloads (for the reports below). +"$JQ_BIN" -s '[.[][] | {key: ((.CLIConfiguration.TfStateFilePath // "") | sub(".*/"; "") | sub("\\.tfstate$"; "")), + value: ((.DeploymentPlatformConfig[0].kind // "") | split("_")[0])}] | from_entries' "$@" >"$WORK/cloud.json" + # Merge the effective set vars into each payload, then report counts. for f in "$@"; do before_tf="$("$JQ_BIN" '[.[].VCSConfig.iacInputData.data | length] | add // 0' "$f")" before_env="$("$JQ_BIN" '[.[].EnvironmentVariables | length] | add // 0' "$f")" out="$WORK/merged.json" - "$JQ_BIN" --slurpfile eff "$WORK/effective.json" --argjson ignore "$IGNORE_JSON" ' + "$JQ_BIN" --slurpfile eff "$WORK/effective.json" --argjson ignore "$IGNORE_JSON" --argjson cloud_patterns "$CLOUD_JSON" ' ($eff[0]) as $E | map( ((.CLIConfiguration.TfStateFilePath // "") | sub(".*/"; "") | sub("\\.tfstate$"; "")) as $wsName | ($E[$wsName] // [] | map(select(.key as $k | [$ignore[] | . as $p | select($k | test($p))] | length == 0))) as $all + | ((.DeploymentPlatformConfig[0].kind // "") | split("_")[0]) as $cloud + | ($cloud_patterns[$cloud] // []) as $cpats | ($all | map(select(.sensitive != true and .category == "terraform"))) as $tf - | ($all | map(select(.sensitive != true and .category == "env"))) as $env + | ($all | map(select(.sensitive != true and .category == "env" + and ((.key as $k | [$cpats[] | . as $p | select($k | test($p))] | length) == 0)))) as $env | .VCSConfig.iacInputData.data = ( reduce $tf[] as $v ((.VCSConfig.iacInputData.data // {}); ($v.value | (fromjson? // $v.value)) as $val @@ -147,10 +164,26 @@ for f in "$@"; do fi done -# Report sensitive set vars (cannot be migrated) and key conflicts. -"$JQ_BIN" -r ' - to_entries[] | .key as $ws | .value[] - | select(.sensitive == true) | " - \($ws): \(.category):\(.key) (set \(.set))" +# Report cloud credential set vars stripped (the connector provides them), +# then sensitive set vars (cannot be migrated; stripped ones excluded) and key +# conflicts. +"$JQ_BIN" -r --slurpfile cloud "$WORK/cloud.json" --argjson pats "$CLOUD_JSON" ' + ($cloud[0]) as $C + | to_entries[] | .key as $ws | ($pats[$C[$ws] // ""] // []) as $cpats + | .value[] | select(.category == "env") + | select(.key as $k | ([$cpats[] | . as $p | select($k | test($p))] | length) > 0) + | " - \($ws): env:\(.key) (set \(.set))" +' "$WORK/effective.json" | sort -u >"$WORK/cloudauth.txt" +if [ -s "$WORK/cloudauth.txt" ]; then + sg_log "cloud credential variable-set vars stripped (the workflow's cloud connector provides them):" + cat "$WORK/cloudauth.txt" >&2 +fi +"$JQ_BIN" -r --slurpfile cloud "$WORK/cloud.json" --argjson pats "$CLOUD_JSON" ' + ($cloud[0]) as $C + | to_entries[] | .key as $ws | ($pats[$C[$ws] // ""] // []) as $cpats + | .value[] | select(.sensitive == true) + | select(.category != "env" or ((.key as $k | [$cpats[] | . as $p | select($k | test($p))] | length) == 0)) + | " - \($ws): \(.category):\(.key) (set \(.set))" ' "$WORK/effective.json" | sort -u >"$WORK/sensitive.txt" if [ -s "$WORK/sensitive.txt" ]; then sg_warn "sensitive variable-set vars skipped (recreate as SG secrets):" diff --git a/transformer/terraform-cloud/example_payload.jsonc b/transformer/terraform-cloud/example_payload.jsonc index 977a67f..86b7281 100644 --- a/transformer/terraform-cloud/example_payload.jsonc +++ b/transformer/terraform-cloud/example_payload.jsonc @@ -8,7 +8,7 @@ }, "DeploymentPlatformConfig": [ { - "kind": "AWS_RBAC", // one of AWS_STATIC, AWS_RBAC, AWS_OIDC, AZURE_STATIC, AZURE_OIDC, AZURE_MANAGED_ID_OIDC, GCP_STATIC, GCP_OIDC + "kind": "AWS_RBAC", // one of AWS_STATIC, AWS_RBAC, AWS_OIDC, AZURE_STATIC, AZURE_OIDC, AZURE_MANAGED_ID_OIDC, GCP_STATIC, GCP_OIDC; its prefix decides which credential env vars are stripped (stripCloudAuthVars) "config": { "integrationId": "/integrations/aws-rbac", // value corresponds to /integrations/<your-integration-from-stackguardian> "profileName": "default" // AWS only: the profile name for AWS_RBAC diff --git a/transformer/terraform-cloud/locals.tf b/transformer/terraform-cloud/locals.tf index a5eeb98..ce07921 100644 --- a/transformer/terraform-cloud/locals.tf +++ b/transformer/terraform-cloud/locals.tf @@ -29,13 +29,16 @@ locals { # projectOverrides keys that name no project of the TFC org (typo guard). unknownProjectOverrides = sort([for k in keys(var.projectOverrides) : k if !contains(values(local.projectNames), k)]) - # Effective cloud connector per workflow. Picked via a tuple, not a - # conditional: an AWS and an Azure DeploymentPlatformConfig have different - # config shapes. + # Effective cloud connector per workflow, and the credential env variables it + # makes redundant: the family (AWS/AZURE/GCP) is the prefix of the connector + # kind, e.g. AZURE_OIDC -> AZURE. Picked via a tuple, not a conditional: an + # AWS and an Azure DeploymentPlatformConfig have different config shapes. deploymentPlatformConfig = { for name in local.workflowNames : name => try([for c in [local.effective[name].DeploymentPlatformConfig, var.SGDefaultDeploymentPlatformConfig] : c if c != null][0], var.SGDefaultDeploymentPlatformConfig) } + cloudPrefix = { for name in local.workflowNames : name => try(split("_", local.deploymentPlatformConfig[name][0].kind)[0], null) } + cloudAuthPatterns = { for name in local.workflowNames : name => var.stripCloudAuthVars ? try(var.cloudAuthVarPatterns[local.cloudPrefix[name]], []) : [] } # SG workflow-name (ResourceName) sanitization. Per the SG OpenAPI spec, # ResourceName must be 1-100 chars; SG's name convention is ^[-a-zA-Z0-9_]+$. @@ -62,13 +65,26 @@ locals { if anytrue([for p in var.ignoreVarPatterns : can(regex(p, v.name))])] } + # Cloud credential env variables the workflow's SG connector replaces + # (stripCloudAuthVars). Stripped whether sensitive or not: a sensitive one + # must not become a placeholder secret that fights the connector either. + strippedCloudAuthVars = { + for name, id in data.tfe_workspace_ids.data.ids : + name => [for v in data.tfe_variables.data[id].variables : "${v.category}:${v.name}" + if v.category == "env" + && !anytrue([for p in var.ignoreVarPatterns : can(regex(p, v.name))]) + && anytrue([for p in local.cloudAuthPatterns[name] : can(regex(p, v.name))])] + } + # TFC never returns values for sensitive variables, so they cannot be # migrated. Record them per workspace so the summary can flag them # (stripped variables excluded — nobody needs a secret stub for those). sensitiveVars = { for name, id in data.tfe_workspace_ids.data.ids : name => [for v in data.tfe_variables.data[id].variables : "${v.category}:${v.name}" - if v.sensitive && !anytrue([for p in var.ignoreVarPatterns : can(regex(p, v.name))])] + if v.sensitive + && !anytrue([for p in var.ignoreVarPatterns : can(regex(p, v.name))]) + && !(v.category == "env" && anytrue([for p in local.cloudAuthPatterns[name] : can(regex(p, v.name))]))] } # Workspaces whose terraform_version is not a pinned semver (e.g. "latest" or @@ -142,7 +158,9 @@ locals { EnvironmentVariables = concat( [for v in data.tfe_variables.data[wsId].variables : { "config" : { "textValue" : v.value, "varName" : v.name }, "kind" : "PLAIN_TEXT" } - if v.category == "env" && v.sensitive == false && !anytrue([for p in var.ignoreVarPatterns : can(regex(p, v.name))])], + if v.category == "env" && v.sensitive == false + && !anytrue([for p in var.ignoreVarPatterns : can(regex(p, v.name))]) + && !anytrue([for p in local.cloudAuthPatterns[wsName] : can(regex(p, v.name))])], local.effective[wsName].extraEnvironmentVariables != null ? local.effective[wsName].extraEnvironmentVariables : [] ) @@ -284,6 +302,11 @@ locals { skippedSensitiveVars = { for name, vars in local.sensitiveVars : name => vars if length(vars) > 0 } strippedVars = { for name, vars in local.strippedVars : name => vars if length(vars) > 0 } ignoreVarPatterns = var.ignoreVarPatterns + # Cloud credential env vars replaced by each workflow's connector. + stripCloudAuthVars = var.stripCloudAuthVars + cloudAuthVarPatterns = var.cloudAuthVarPatterns + workspaceCloudKinds = { for name in local.workflowNames : name => try(local.deploymentPlatformConfig[name][0].kind, null) } + strippedCloudAuthVars = { for name, vars in local.strippedCloudAuthVars : name => vars if length(vars) > 0 } # Version policy, so the later phases can explain what each workflow runs. terraformVersionSource = var.SGTerraformVersionSource terraformVersionDefault = var.SGDefaultTerraformVersion # null = the execution preset decides diff --git a/transformer/terraform-cloud/summary.tmpl b/transformer/terraform-cloud/summary.tmpl index ba257b2..687fb43 100644 --- a/transformer/terraform-cloud/summary.tmpl +++ b/transformer/terraform-cloud/summary.tmpl @@ -31,6 +31,20 @@ Not migrated because they only mean something inside Terraform Cloud (patterns: %{ endfor ~} %{ endif ~} +## Stripped cloud credential variables +Env variables replaced by the workflow's StackGuardian cloud connector (AWS_*/ARM_*/GOOGLE_* credential families, chosen by the connector kind). Set stripCloudAuthVars = false or edit cloudAuthVarPatterns to keep them. +%{ if !summary.stripCloudAuthVars ~} +- Disabled (stripCloudAuthVars = false). +%{ else ~} +%{ if length(summary.strippedCloudAuthVars) == 0 ~} +- None. +%{ else ~} +%{ for ws, vars in summary.strippedCloudAuthVars ~} +- ${ws} (${summary.workspaceCloudKinds[ws]}): ${join(", ", vars)} +%{ endfor ~} +%{ endif ~} +%{ endif ~} + ## Terraform version %{ if summary.terraformVersionSource == "preset" ~} No version is set on the migrated workflows (SGTerraformVersionSource = "preset"): StackGuardian applies the organisation's execution preset (Settings -> Runner groups -> Execution presets), or its platform default (managed Terraform 1.5.7) when none is configured. The workspaces ran these versions in TFC: diff --git a/transformer/terraform-cloud/terraform.tfvars.example b/transformer/terraform-cloud/terraform.tfvars.example index 27b1864..2e01d31 100644 --- a/transformer/terraform-cloud/terraform.tfvars.example +++ b/transformer/terraform-cloud/terraform.tfvars.example @@ -20,6 +20,19 @@ exportPath = "export" # name), e.g. TFC_WORKSPACE_NAME or TFC_AWS_RUN_ROLE_ARN. [] keeps everything. ignoreVarPatterns = ["^TFC_", "^TFE_"] +# Cloud credential env variables (AWS_ACCESS_KEY_ID, ARM_CLIENT_SECRET, +# GOOGLE_CREDENTIALS, ...) are replaced by the workflow's StackGuardian cloud +# connector and are not migrated; which family is stripped follows each +# workflow's DeploymentPlatformConfig kind. Set false to keep them. +stripCloudAuthVars = true +# The patterns per family (AWS / AZURE / GCP). Setting this replaces the whole +# map; the defaults are listed in variables.tf. +# cloudAuthVarPatterns = { +# AWS = ["^AWS_ACCESS_KEY_ID$", "^AWS_SECRET_ACCESS_KEY$", "^AWS_SESSION_TOKEN$"] +# AZURE = ["^ARM_CLIENT_ID$", "^ARM_CLIENT_SECRET$", "^ARM_TENANT_ID$", "^ARM_SUBSCRIPTION_ID$"] +# GCP = ["^GOOGLE_CREDENTIALS$", "^GOOGLE_APPLICATION_CREDENTIALS$"] +# } + # Add emails of the users who should approve the terraform plan, since approvalPreApply is set to true SGDefaultWfApprovers = [] diff --git a/transformer/terraform-cloud/variables.tf b/transformer/terraform-cloud/variables.tf index eb5ce56..3585ee9 100644 --- a/transformer/terraform-cloud/variables.tf +++ b/transformer/terraform-cloud/variables.tf @@ -45,6 +45,36 @@ variable "ignoreVarPatterns" { type = list(string) } +variable "stripCloudAuthVars" { + default = true + description = "Strip the cloud credential env variables (AWS_ACCESS_KEY_ID, ARM_CLIENT_SECRET, GOOGLE_CREDENTIALS, ...) that the workflow's StackGuardian cloud connector replaces. Which family is stripped follows each workflow's effective DeploymentPlatformConfig[0].kind (AWS_* -> AWS, AZURE_* -> AZURE, GCP_* -> GCP). Env variables only; terraform input variables are never touched. Stripped variables are listed in migration-summary.md (strippedCloudAuthVars) and a sensitive one is stripped as well, so no placeholder secret is created for it." + type = bool +} + +variable "cloudAuthVarPatterns" { + description = "Regexes (per cloud family AWS/AZURE/GCP, matched against the env variable name) stripped by stripCloudAuthVars. Setting this replaces the whole map, so keep the families you do not change. Also read by scripts/enrich_variable_sets.sh for variable-set variables." + type = map(list(string)) + default = { + AWS = [ + "^AWS_ACCESS_KEY_ID$", "^AWS_SECRET_ACCESS_KEY$", "^AWS_SESSION_TOKEN$", + "^AWS_PROFILE$", "^AWS_ROLE_ARN$", "^AWS_WEB_IDENTITY_TOKEN_FILE$", + "^AWS_SHARED_CREDENTIALS_FILE$", "^AWS_CONFIG_FILE$", + ] + AZURE = [ + "^ARM_CLIENT_ID$", "^ARM_CLIENT_SECRET$", "^ARM_TENANT_ID$", "^ARM_SUBSCRIPTION_ID$", + "^ARM_USE_OIDC$", "^ARM_OIDC_", "^ARM_CLIENT_CERTIFICATE", "^ARM_USE_MSI$", "^ARM_MSI_ENDPOINT$", + ] + GCP = [ + "^GOOGLE_CREDENTIALS$", "^GOOGLE_APPLICATION_CREDENTIALS$", "^GOOGLE_OAUTH_ACCESS_TOKEN$", + "^GOOGLE_IMPERSONATE_SERVICE_ACCOUNT$", "^CLOUDSDK_AUTH_", + ] + } + validation { + condition = alltrue([for k in keys(var.cloudAuthVarPatterns) : contains(["AWS", "AZURE", "GCP"], k)]) + error_message = "cloudAuthVarPatterns keys must be AWS, AZURE or GCP (the prefix of the DeploymentPlatformConfig kind)." + } +} + variable "SGDefaultWfApprovers" { default = [] description = "Add emails of the users who should approve the terraform plan, since approvalPreApply is set to true" From c5449127fa155ebed865759d272dccd66e454e52 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adis=20Halilovi=C4=87?= <adis.halilovic@stackguardian.io> Date: Mon, 14 Sep 2026 12:39:23 +0200 Subject: [PATCH 40/71] feat: reuse existing workflow groups, refuse moves and name collisions in the plan MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The importer takes each project's workflow group from the payload (CLIConfiguration.WorkflowGroup.name, written by the transformer from projectOverrides.<project>.workflowGroup or tfc-<project>); the legacy .sg/workflow-groups.json still applies on top, with a deprecation warning. An existing group is reused (STATUS reuse), a missing one is created unless --no-create-groups (missing!). The old rule that overridden groups must pre-exist is gone. StackGuardian cannot move workflows between groups, so the plan refuses a project whose workflows still live in the group recorded in .sg/state.json (or the default tfc-<project> group) when the target changed (STATUS moved!), and two projects mapped to one group with overlapping workflow names. Problems are listed as ✗ lines, the full plan is still shown (also with --dry-run), and the run stops before the confirmation. The per-workflow ACTION column now says skip for a file that state_import_done will skip, next to update and create. List endpoints are read through a paginated helper (the API pages at 50), and the migration summary shows the group each project maps to plus the new strippedCloudAuthVars / unknownProjectOverrides sections. --- scripts/lib/errors.sh | 8 +- scripts/lib/report.sh | 15 ++- scripts/lib/sg_api.sh | 29 ++++-- scripts/migrate.sh | 221 +++++++++++++++++++++++++++--------------- 4 files changed, 182 insertions(+), 91 deletions(-) diff --git a/scripts/lib/errors.sh b/scripts/lib/errors.sh index d6afccc..c4c400d 100644 --- a/scripts/lib/errors.sh +++ b/scripts/lib/errors.sh @@ -12,11 +12,11 @@ _SG_HINTS_SHOWN=" " # Table: "<regex> => <hint>" — keep hints to one line each. SG_API_HINTS=( 'above the highest managed version => Terraform version above SG'"'"'s managed ceiling (1.5.7, last MPL/FOSS release). The importer retries with SGDefaultTerraformVersion automatically; to keep the newer version use a private runner binary path or a custom runtime template (workspaceOverrides[<ws>].terraformVersion).' - 'integration.*(not found|does not exist|invalid)|(not found|does not exist|invalid).*integration => Connector/integration id not found in this org. Check SGDefaultVCSAuthIntegrationID and SGDefaultDeploymentPlatformConfig[].config.integrationId (or the workspaceOverrides equivalents); '"'"'preflight'"'"' lists the ids that exist.' - 'runner ?group.*(not found|does not exist|invalid)|(not found|does not exist|invalid).*runner => Runner group not found. Check SGDefaultRunnerConstraints.names / workspaceOverrides[<ws>].RunnerConstraints against the runner groups in the SG org.' + 'integration.*(not found|does not exist|invalid)|(not found|does not exist|invalid).*integration => Connector/integration id not found in this org. Check SGDefaultVCSAuthIntegrationID and SGDefaultDeploymentPlatformConfig[].config.integrationId (or the projectOverrides / workspaceOverrides equivalents); '"'"'preflight'"'"' lists the ids that exist.' + 'runner ?group.*(not found|does not exist|invalid)|(not found|does not exist|invalid).*runner => Runner group not found. Check SGDefaultRunnerConstraints.names / projectOverrides[<project>].RunnerConstraints / workspaceOverrides[<ws>].RunnerConstraints against the runner groups in the SG org.' '(repo|repository).*(not found|access|permission|denied|unable|could not|cannot)|(clone|checkout).*(fail|denied|unable) => The VCS connector cannot reach the repository. Check SGDefaultIACVCSRepoPrefix + the workspace repo path, and that the connector (SGDefaultVCSAuthIntegrationID) has access to that repository.' - 'approver => Approvers must be e-mail addresses of existing SG users. Check SGDefaultWfApprovers / workspaceOverrides[<ws>].Approvers.' - 'already exists => A resource with that name already exists. Existing workflows are updated by the importer; for workflow groups this is harmless.' + 'approver => Approvers must be e-mail addresses of existing SG users. Check SGDefaultWfApprovers / projectOverrides[<project>].Approvers / workspaceOverrides[<ws>].Approvers.' + 'already exists|not unique => A resource with that name already exists. Existing workflows are updated by the importer (PATCH); an existing workflow group is reused.' 'ResourceName => Invalid workflow name: 1-100 chars, letters/digits/-/_ only. The transformer sanitizes names (see renamedWorkspaces in migration-summary.md); adjust local.resourceNames if a rule is missing.' 'sourceConfigDestKind => Invalid VCS kind. SGDefaultSourceConfigDestKind must be one of GITHUB_COM, GITLAB_COM, BITBUCKET_ORG, AZURE_DEVOPS, GIT_OTHER.' '(unauthori[sz]ed|forbidden|invalid token|authentication) => SG_API_TOKEN was rejected or lacks permission for this org. Regenerate it under Org settings -> API keys and re-export SG_API_TOKEN.' diff --git a/scripts/lib/report.sh b/scripts/lib/report.sh index edadbf5..6f609d7 100644 --- a/scripts/lib/report.sh +++ b/scripts/lib/report.sh @@ -12,7 +12,10 @@ show_migration_summary() { sg_step "Migration summary" sg_row "TFC organisation" "$("$jqb" -r '.organization' "$f")" sg_row "Workspaces exported" "$("$jqb" -r '.workspaceCount' "$f")" - "$jqb" -r '.projectWorkspaceCounts | to_entries[] | " tfc-\(.key | ascii_downcase | gsub("[^a-z0-9-]+"; "-")): \(.value) workflow(s)"' "$f" >&2 + # Per project: the SG workflow group the transformer assigned (projects.*.workflowGroup; + # older summaries only have the counts, then the default tfc-<segment> is shown). + "$jqb" -r 'if (.projects // null) != null then .projects | to_entries[] | " \(.key) -> \(.value.workflowGroup): \(.value.workspaceCount) workflow(s)" + else .projectWorkspaceCounts | to_entries[] | " tfc-\(.key | ascii_downcase | gsub("[^a-z0-9-]+"; "-")): \(.value) workflow(s)" end' "$f" >&2 if [ "$(tfvars_get .exportStateFiles true)" != "false" ]; then states=0 for n in "$EXPORT_DIR"/states/*.tfstate; do [ -f "$n" ] && states=$((states + 1)); done @@ -41,6 +44,10 @@ show_migration_summary() { 'to_entries[] | "\(.key): \(.value | join(", "))"' "TFC never exposes their values; they become placeholder SG secrets after import" _summary_section "$f" '.strippedVars' "TFC-specific variables stripped" \ 'to_entries[] | "\(.key): \(.value | join(", "))"' "they only mean something inside Terraform Cloud (ignoreVarPatterns)" + _summary_section "$f" '.strippedCloudAuthVars // {}' "Cloud credential variables stripped" \ + 'to_entries[] | "\(.key): \(.value | join(", "))"' "the workflow's cloud connector provides them (stripCloudAuthVars)" + _summary_section "$f" '.unknownProjectOverrides // []' "projectOverrides key(s) matching no TFC project" \ + '.[]' "typo? their settings apply to nothing" _summary_section "$f" '.terraformVersionFallbacks' "Terraform version not pinned" \ 'to_entries[] | "\(.key): \"\(.value)\""' "$([ -z "$def" ] && echo "left to the execution preset" || echo "the fallback ${def#TERRAFORM-} is used")" _summary_section "$f" '.nonRemoteExecutionModes' "Non-remote execution mode" \ @@ -118,6 +125,7 @@ show_import_plan() { existing="$(sg_list_workflows "$grp")" printf '%s' "$existing" | "$JQ_BIN" -e 'type == "array"' >/dev/null 2>&1 || existing='[]' rows="$("$JQ_BIN" -r --argjson ex "$existing" --arg grp "$grp" --argjson ws "$(ws_filter_json)" \ + --argjson skip "${PLAN_SKIP_SEGS:-[]}" --arg seg "$seg" \ --slurpfile sum "${summary:-/dev/null}" ' ($sum[0] // {}) as $S | .[] @@ -125,7 +133,7 @@ show_import_plan() { | ((.CLIConfiguration.TfStateFilePath // "") | sub(".*/"; "") | sub("\\.tfstate$"; "")) as $wsName | .ResourceName as $n | [ $n, $grp, - (if ($ex | index($n)) != null then "update" else "create" end), + (if ($skip | index($seg)) != null then "skip" elif ($ex | index($n)) != null then "update" else "create" end), (.TerraformConfig.terraformVersion // "preset"), ((.RunnerConstraints // null) | if . == null then "preset" elif .type == "private" then "private" else "shared" end), (if (.VCSTriggers // null) != null then "yes" else "no" end), @@ -150,9 +158,10 @@ show_import_plan() { else tfv="${tfv#TERRAFORM-}"; fi [ "$runner" = "preset" ] && runner="preset${SG_PRESET_RUNNER_SHORT:+ ($SG_PRESET_RUNNER_SHORT)}" [ "$secrets" = "0" ] && secrets="-" - case "$action" in create) action="${C_GREEN}create ${C_RESET}" ;; update) action="${C_YELLOW}update ${C_RESET}" ;; esac + case "$action" in create) action="${C_GREEN}create ${C_RESET}" ;; update) action="${C_YELLOW}update ${C_RESET}" ;; skip) action="${C_DIM}skip ${C_RESET}" ;; esac printf " %-${wn}s %-${gn}s %s %-28s %-${rw}s %-8s %-5s %s\n" "$name" "$grp" "$action" "$tfv" "$runner" "$trig" "$vars" "$secrets" >&2 done <<<"$all_rows" echo >&2 + sg_dim "ACTION create = new workflow; update = exists in the group, PATCHed with the current payload; skip = file already imported with identical content (--fresh re-imports)" sg_dim "TERRAFORM '-> fallback' = pinned above SG's managed ceiling (1.5.7, last FOSS release); 'preset' = left to the org's execution preset at import; SECRETS = sensitive vars recreated as placeholder secrets" } diff --git a/scripts/lib/sg_api.sh b/scripts/lib/sg_api.sh index e908517..6d27353 100644 --- a/scripts/lib/sg_api.sh +++ b/scripts/lib/sg_api.sh @@ -83,15 +83,30 @@ wf_triggers_endpoint() { printf '%swebhooks/vcs_triggers/' "$(wf_url "$1" "$2")" # sg_workflow_exists <group> <wf> — exit 0 when the workflow exists. sg_workflow_exists() { [ "$(sg_http_code GET "$(wf_url "$1" "$2")")" = "200" ]; } -# sg_list_workflows <group> — ["wf-name", ...] in the group ([] on 404). -sg_list_workflows() { - local body out - if body="$(sg_api_get "$(sg_org_url)/wfgrps/$1/wfs/listall/" 2>/dev/null)"; then - out="$(printf '%s' "$body" | "$(sg_resolve jq sg_ensure_jq)" -c '[(if (.msg | type) == "array" then .msg elif (.data | type) == "array" then .data elif (.data.Workflows? | type) == "array" then .data.Workflows elif type == "array" then . else [] end)[] | (.ResourceName // .Id // empty)]' 2>/dev/null)" - fi - printf '%s' "${out:-[]}" +# _sg_listall <path-under-org> <jq-item-expr> — every item of a paginated +# listall endpoint (the API pages at 50 by default; lastevaluatedkey is the +# cursor) mapped through <jq-item-expr>, as a JSON array; [] on any error. +# Tolerates the response shapes seen so far (msg / data / data.Workflows / +# bare array). The expression is spliced into the jq program. +_sg_listall() { + local path="$1" expr="$2" key="" body page acc='[]' jqb + jqb="$(sg_resolve jq sg_ensure_jq)" + while :; do + body="$(sg_api_get "$(sg_org_url)/${path}?limit=100${key:+&lastevaluatedkey=$key}" 2>/dev/null)" || break + page="$(printf '%s' "$body" | "$jqb" -c '[(if (.msg | type) == "array" then .msg elif (.data | type) == "array" then .data elif (.data.Workflows? | type) == "array" then .data.Workflows elif type == "array" then . else [] end)[] | '"$expr"' | select(. != null and . != "")]' 2>/dev/null)" || page='[]' + acc="$("$jqb" -nc --argjson a "$acc" --argjson b "$page" '$a + $b')" + key="$(printf '%s' "$body" | "$jqb" -r '.lastevaluatedkey // empty' 2>/dev/null | "$jqb" -sRr @uri)" + [ -n "$key" ] || break + done + printf '%s' "$acc" } +# sg_list_workflows <group> — ["wf-name", ...] in the group ([] on 404). +sg_list_workflows() { _sg_listall "wfgrps/$1/wfs/listall/" '(.ResourceName // .Id)'; } + +# sg_list_wfgrps — ["group-name", ...] of the org. +sg_list_wfgrps() { _sg_listall "wfgrps/listall/" '(.ResourceName // .Id)'; } + # sg_patch_workflow <group> <wf> <json> — PATCH a workflow. sg_patch_workflow() { sg_api_patch "$(wf_url "$1" "$2")" "$3"; } diff --git a/scripts/migrate.sh b/scripts/migrate.sh index 471c1a1..cab5f62 100755 --- a/scripts/migrate.sh +++ b/scripts/migrate.sh @@ -101,14 +101,17 @@ Commands: update Pull the latest version of the migrator (git pull --ff-only) and rebuild the Docker image if the Dockerfile changed. Runs on the host. -Each TFC project maps to an SG workflow group named tfc-<project>, created via the -API if missing. Override a project's target group in .sg/workflow-groups.json -(\`{"<project-segment>": "<existing-group>"}\`); override groups are not auto-created. +Each TFC project is imported into the workflow group the transformer assigned to +it: projectOverrides.<project>.workflowGroup in terraform.tfvars, or tfc-<project>. +An existing group is reused, a missing one is created (--no-create-groups to +require it). Workflows are never moved: when a project's workflows already live +in another group the plan stops and says so. Options: --org NAME StackGuardian org for import (or set SG_ORG) --export-dir DIR Payload/state output dir (default: ./export) - --mapping FILE Optional project-segment -> group override map (default: .sg/workflow-groups.json) + --mapping FILE Deprecated: project-segment -> group override map (default: .sg/workflow-groups.json); + use projectOverrides.<project>.workflowGroup instead --concurrency N Max parallel jobs for convert/import (default: 4) --no-create-groups Do not create missing workflow groups; require them to exist --no-variable-sets Skip merging TFC Variable Set variables in the 'all' flow @@ -327,12 +330,122 @@ current_shell() { # group_for <segment> -> the SG workflow group for a project segment: an entry # from the optional override map, else the default tfc-<segment>. +# payload_group <file> — the workflow group the transformer wrote into the +# file's entries (CLIConfiguration.WorkflowGroup.name); "" when absent or mixed. +payload_group() { + "$JQ_BIN" -r '[.[] | .CLIConfiguration.WorkflowGroup.name // empty] | unique | if length == 1 then .[0] else "" end' "$1" 2>/dev/null +} + +# mapped_group <segment> — the legacy .sg/workflow-groups.json entry, if any. +mapped_group() { + [ -f "$MAPPING" ] || return 0 + "$JQ_BIN" -r --arg k "$1" '.[$k] // empty' "$MAPPING" 2>/dev/null || true +} + +# group_for <segment> — a project's target workflow group: the legacy mapping +# entry (deprecated), else the group in the payload (projectOverrides.<project> +# .workflowGroup or tfc-<project>, written by the transformer), else +# tfc-<segment>. Silent: also runs inside run_parallel subshells. group_for() { - local seg="$1" override="" - if [ -f "$MAPPING" ]; then - override="$("$JQ_BIN" -r --arg k "$seg" '.[$k] // empty' "$MAPPING" 2>/dev/null || true)" + local seg="$1" f="$EXPORT_DIR/sg-payload.$seg.json" g + g="$(mapped_group "$seg")" + [ -n "$g" ] || { [ -f "$f" ] && g="$(payload_group "$f")"; } + printf '%s' "${g:-tfc-$seg}" +} + +# plan_groups <payload>... — the workflow-group part of the import plan. Each +# file's target group is checked and shown as reuse (exists), create (missing, +# created before the import) or missing! (--no-create-groups). Two things block +# an import and are collected in PLAN_PROBLEMS: a project whose workflows +# already live in another group (StackGuardian cannot move workflows between +# groups), and two projects sharing a group with overlapping workflow names (a +# name is unique within a group). Sets PLAN_TO_CREATE, PLAN_N_CREATE, +# PLAN_TOTAL_WF; the caller shows the problems and stops after the full plan. +plan_groups() { + local f seg grp count code status prev still n_still names fw gw i legacy=0 line + local -a paths=("$@") files=() groups=() counts=() statuses=() + PLAN_TO_CREATE=" " + PLAN_N_CREATE=0 + PLAN_TOTAL_WF=0 + PLAN_PROBLEMS=() + for f in "${paths[@]}"; do + seg="$(seg_of "$f")" + names="$("$JQ_BIN" -c --argjson ws "$(ws_filter_json)" '[.[] | select(($ws | length) == 0 or (.ResourceName as $n | $ws | index($n) != null)) | .ResourceName]' "$f")" + count="$("$JQ_BIN" 'length' <<<"$names")" + PLAN_TOTAL_WF=$((PLAN_TOTAL_WF + count)) + grp="$(group_for "$seg")" + [ -n "$(mapped_group "$seg")" ] && legacy=$((legacy + 1)) + if [ -z "$(payload_group "$f")" ] && [ "$count" -gt 0 ]; then + sg_warn "$(basename "$f") carries no (or mixed) CLIConfiguration.WorkflowGroup.name — re-run 'apply'; using $grp" + fi + + code="$(wfgroup_http_code "$grp")" + case "$code" in + 200) status="${C_GREEN}reuse${C_RESET}" ;; + 404) + if [ "$CREATE_GROUPS" -eq 1 ]; then + status="${C_YELLOW}create${C_RESET}" + case "$PLAN_TO_CREATE" in *" $grp "*) ;; *) + PLAN_TO_CREATE="$PLAN_TO_CREATE$grp " + PLAN_N_CREATE=$((PLAN_N_CREATE + 1)) + ;; + esac + else + status="${C_RED}missing!${C_RESET}" + PLAN_PROBLEMS+=("workflow group '$grp' ($(basename "$f")) does not exist and --no-create-groups is set — create it in StackGuardian or drop the flag") + fi + ;; + 401 | 403) die "auth failed (HTTP $code) for org '$ORG' — check SG_API_TOKEN" ;; + 000) die "could not reach $SG_BASE_URL" ;; + *) die "unexpected HTTP $code checking group '$grp'" ;; + esac + + # Never move: if this project's workflows were imported into another group + # before (state), or would sit in the default tfc-<segment> group when the + # target is now a different one, and they still exist there, refuse. + prev="$(state_read | "$JQ_BIN" -r --arg s "$seg" '.import[$s].group // empty')" + [ -z "$prev" ] && [ "$grp" != "tfc-$seg" ] && prev="tfc-$seg" + if [ -n "$prev" ] && [ "$prev" != "$grp" ] && [ "$(wfgroup_http_code "$prev")" = "200" ]; then + still="$("$JQ_BIN" -nc --argjson ex "$(sg_list_workflows "$prev")" --argjson mine "$names" '[$mine[] | select(. as $n | $ex | index($n) != null)]')" + n_still="$("$JQ_BIN" 'length' <<<"$still")" + if [ "$n_still" -gt 0 ]; then + status="${C_RED}moved!${C_RESET}" + PLAN_PROBLEMS+=("$(basename "$f"): $n_still workflow(s) already live in group '$prev' but the target is now '$grp' — StackGuardian cannot move workflows between groups; keep '$prev' (projectOverrides.\"<project>\".workflowGroup) or delete them from '$prev' first: $("$JQ_BIN" -r 'join(", ")' <<<"$still")") + else + sg_dim "$(basename "$f"): previous group '$prev' no longer holds these workflows — importing into '$grp'" + fi + fi + files+=("$(basename "$f")") + groups+=("$grp") + counts+=("$count") + statuses+=("$status") + done + + # Two projects may share a group only when their workflow names do not overlap. + while IFS= read -r line; do + [ -n "$line" ] && PLAN_PROBLEMS+=("$line") + done < <(for ((i = 0; i < ${#paths[@]}; i++)); do + "$JQ_BIN" -c --arg seg "$(seg_of "${paths[i]}")" --arg grp "${groups[i]}" --argjson ws "$(ws_filter_json)" \ + '{seg: $seg, grp: $grp, names: [.[] | select(($ws | length) == 0 or (.ResourceName as $n | $ws | index($n) != null)) | .ResourceName]}' "${paths[i]}" + done | "$JQ_BIN" -sr ' + group_by(.grp)[] | select(length > 1) | .[0].grp as $g + | ([.[].names[]] | group_by(.) | map(select(length > 1) | .[0])) as $dups + | select(($dups | length) > 0) + | "workflow name(s) \($dups | join(", ")) appear in more than one project mapped to group \u0027\($g)\u0027 (\([.[].seg] | join(", "))) — a workflow name is unique within a group; give one of the projects its own workflowGroup"') + + fw="$(sg_maxlen 4 "${files[@]}")" + gw="$(sg_maxlen 14 "${groups[@]}")" + printf '%sImport plan%s (org: %s%s%s, %s)\n' "$C_BOLD" "$C_RESET" "$C_CYAN" "$ORG" "$C_RESET" "$SG_BASE_URL" >&2 + printf " %s%-${fw}s %-${gw}s %-9s %s%s\n" "$C_BOLD" "FILE" "WORKFLOW GROUP" "WORKFLOWS" "STATUS" "$C_RESET" >&2 + for ((i = 0; i < ${#files[@]}; i++)); do + printf " %-${fw}s %-${gw}s %-9s %s\n" "${files[i]}" "${groups[i]}" "${counts[i]}" "${statuses[i]}" >&2 + done + if [ "$legacy" -gt 0 ]; then + sg_warn "$(sg_rel "$MAPPING") overrides the group of $legacy project(s) — this file is deprecated; set projectOverrides.\"<project>\".workflowGroup in $(sg_rel "$TFVARS") instead (project names: workspaceProjects in $(sg_rel "$EXPORT_DIR")/migration-summary.json) and re-run 'apply'" fi - [ -n "$override" ] && echo "$override" || echo "tfc-$seg" + for line in ${PLAN_PROBLEMS[@]+"${PLAN_PROBLEMS[@]}"}; do + printf ' %s✗%s %s\n' "$C_RED$C_BOLD" "$C_RESET" "$line" >&2 + done } # do_set_triggers <payload> — for each workflow in the file with a non-null @@ -830,62 +943,26 @@ cmd_import() { payload_files [ "${#PF[@]}" -gt 0 ] || die "No payload files in $(sg_rel "$EXPORT_DIR")." - # Build the plan: resolve each project's group, check existence, and decide - # which groups need creating. Override groups (from the map) must already exist. - local fail=0 to_create=" " n_create=0 total_wf=0 f seg grp count override is_override code status fw gw q i - local -a files=() groups=() counts=() statuses=() + # The workflow-group part of the plan (reuse/create/missing!/moved!, name + # collisions); problems are shown here and stop the run after the full plan. + local q grp f seg + plan_groups "${PF[@]}" + + # Files already imported in full with identical content are skipped (the + # plan shows their workflows as "skip"); --fresh or a --workspace filter + # re-imports them. + local -a todo=() skip_segs=() + local skipped=0 import_rc=0 for f in "${PF[@]}"; do seg="$(seg_of "$f")" - count="$("$JQ_BIN" --argjson ws "$(ws_filter_json)" '[.[] | select(($ws | length) == 0 or (.ResourceName as $n | $ws | index($n) != null))] | length' "$f")" - total_wf=$((total_wf + count)) - override="" - [ -f "$MAPPING" ] && override="$("$JQ_BIN" -r --arg k "$seg" '.[$k] // empty' "$MAPPING" 2>/dev/null || true)" - if [ -n "$override" ]; then - grp="$override" - is_override=1 - else - grp="tfc-$seg" - is_override=0 + if [ "$FRESH" -eq 0 ] && [ "${#WS_FILTER[@]}" -eq 0 ] && state_import_done "$seg" "$(sg_sha_files "$f")"; then + skipped=$((skipped + 1)) + skip_segs+=("$seg") + continue fi - - code="$(wfgroup_http_code "$grp")" - case "$code" in - 200) status="${C_GREEN}exists${C_RESET}" ;; - 404) - if [ "$is_override" -eq 1 ]; then - status="${C_RED}missing!${C_RESET}" - fail=1 - elif [ "$CREATE_GROUPS" -eq 1 ]; then - status="${C_YELLOW}create${C_RESET}" - case "$to_create" in *" $grp "*) ;; *) - to_create="$to_create$grp " - n_create=$((n_create + 1)) - ;; - esac - else - status="${C_RED}missing!${C_RESET}" - fail=1 - fi - ;; - 401 | 403) die "auth failed (HTTP $code) for org '$ORG' — check SG_API_TOKEN" ;; - 000) die "could not reach $SG_BASE_URL" ;; - *) die "unexpected HTTP $code checking group '$grp'" ;; - esac - files+=("$(basename "$f")") - groups+=("$grp") - counts+=("$count") - statuses+=("$status") - done - fw="$(sg_maxlen 4 "${files[@]}")" - gw="$(sg_maxlen 14 "${groups[@]}")" - printf '%sImport plan%s (org: %s%s%s, %s)\n' "$C_BOLD" "$C_RESET" "$C_CYAN" "$ORG" "$C_RESET" "$SG_BASE_URL" >&2 - printf " %s%-${fw}s %-${gw}s %-9s %s%s\n" "$C_BOLD" "FILE" "WORKFLOW GROUP" "WORKFLOWS" "STATUS" "$C_RESET" >&2 - for ((i = 0; i < ${#files[@]}; i++)); do - printf " %-${fw}s %-${gw}s %-9s %s\n" "${files[i]}" "${groups[i]}" "${counts[i]}" "${statuses[i]}" >&2 + todo+=("$f") done - if [ "$fail" -ne 0 ]; then - die "some groups are missing (override groups are not auto-created; create them or remove the override)." - fi + PLAN_SKIP_SEGS="$([ "${#skip_segs[@]}" -gt 0 ] && names_json "${skip_segs[@]}" || echo '[]')" # Fallback for workflows the API rejects as above the managed ceiling: a fixed # SGDefaultTerraformVersion (missing key = 1.5.7), or an explicit null in @@ -900,14 +977,17 @@ cmd_import() { SG_PRESET_JSON="$(sg_execution_preset)" || SG_PRESET_JSON="" preset_labels show_import_plan "${PF[@]}" + if [ "${#PLAN_PROBLEMS[@]}" -gt 0 ]; then + die "${#PLAN_PROBLEMS[@]} problem(s) block the import (the ✗ lines above) — nothing was changed in $ORG" + fi if [ "$DRY_RUN" -eq 1 ]; then sg_success "dry run — nothing was created or changed" return 0 fi if [ "$ASSUME_YES" -ne 1 ]; then - q="Import $total_wf workflow(s) into $ORG" - [ "$n_create" -gt 0 ] && q="$q and create $n_create workflow group(s)" + q="Import $PLAN_TOTAL_WF workflow(s) into $ORG" + [ "$PLAN_N_CREATE" -gt 0 ] && q="$q and create $PLAN_N_CREATE workflow group(s)" sg_interactive || die "no terminal to confirm the import — re-run with -y to import without a prompt" if ! sg_confirm "$q?" N; then sg_warn "import cancelled — nothing was changed in $ORG" @@ -917,7 +997,7 @@ cmd_import() { fi # Create the missing tfc-* groups before importing into them. - for grp in $to_create; do + for grp in $PLAN_TO_CREATE; do sg_log "creating workflow group $grp" wfgroup_create "$grp" || die "failed to create workflow group $grp" done @@ -925,23 +1005,10 @@ cmd_import() { SGCLI_BIN="$(sg_resolve sg-cli sg_ensure_sgcli)" rm -f "$EXPORT_DIR/terraform-version-fallbacks.log" - # Resume: skip payload files already imported in full with identical content - # (a changed payload or a previous failure re-imports the whole file; - # existing workflows are updated via PATCH). - local -a todo=() - local skipped=0 import_rc=0 - for f in "${PF[@]}"; do - seg="$(seg_of "$f")" - if [ "$FRESH" -eq 0 ] && [ "${#WS_FILTER[@]}" -eq 0 ] && state_import_done "$seg" "$(sg_sha_files "$f")"; then - skipped=$((skipped + 1)) - continue - fi - todo+=("$f") - done [ "$skipped" -gt 0 ] && sg_log "skipping $skipped payload file(s) already imported and unchanged (--fresh to re-import)" if [ "${#todo[@]}" -gt 0 ]; then # More than one workflow to import: try a single one first (fail fast). - if [ "$total_wf" -gt 1 ]; then probe_import "${todo[0]}"; fi + if [ "$PLAN_TOTAL_WF" -gt 1 ]; then probe_import "${todo[0]}"; fi run_parallel do_import "$CONC" "importing ${#todo[@]} payload file(s) (retries: $RETRIES)" "${todo[@]}" || import_rc=1 for f in "${todo[@]}"; do seg="$(seg_of "$f")" From ad0c36af9db4842ea030e3390a0f6334b8e85dc7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adis=20Halilovi=C4=87?= <adis.halilovic@stackguardian.io> Date: Mon, 14 Sep 2026 12:39:24 +0200 Subject: [PATCH 41/71] feat: idempotent VCS triggers; say that existing secrets are kept The trigger endpoint upserts (a second POST answers 200 "VCS triggers updated"), so re-runs were already safe but re-posted every trigger. do_set_triggers keeps the sha of each posted body in state (triggers.<seg>.sha) and skips a workflow whose triggers are unchanged and already set; --fresh or an edited trigger re-posts. The call goes through sg_set_vcs_triggers, which also treats a 4xx "already exists" answer from older builds as success and reports other errors with the API hint table. Secret stubs were already reused, never overwritten; the log line and the checklist now say so. --- scripts/lib/checklist.sh | 6 +++--- scripts/lib/sg_api.sh | 19 +++++++++++++++++++ scripts/lib/state.sh | 9 ++++++++- scripts/migrate.sh | 28 +++++++++++++++++++--------- 4 files changed, 49 insertions(+), 13 deletions(-) diff --git a/scripts/lib/checklist.sh b/scripts/lib/checklist.sh index 9b9e1b0..431e7c7 100644 --- a/scripts/lib/checklist.sh +++ b/scripts/lib/checklist.sh @@ -85,7 +85,7 @@ create_secret_stubs() { state_update '.secrets[$n] = {workflow: $w, group: $g, var: $v, category: $c, workspace: $ws, at: $at}' \ --arg n "$name" --arg w "$wf" --arg g "$grp" --arg v "$var" --arg c "$cat" --arg ws "$ws" --arg at "$(state_now)" done < <("$JQ_BIN" -r '.skippedSensitiveVars | to_entries[] | .key as $ws | .value[] | "\($ws)\t\(.)"' "$summary") - sg_log "secret stubs: $created created, $reused already existed, $patched workflow(s) now reference them, $skipped skipped" + sg_log "secret stubs: $created created, $reused already existed (kept — a secret's value is never overwritten), $patched workflow(s) now reference them, $skipped skipped" return 0 } @@ -129,7 +129,7 @@ write_checklist() { i_nonremote="$("$JQ_BIN" -r '.nonRemoteExecutionModes | to_entries[] | "- [ ] `\(.key)` used `\(.value)` execution in TFC — its state may live outside TFC; verify the exported state is current"' "$summary")" i_renamed="$("$JQ_BIN" -r '.renamedWorkspaces | to_entries[] | "- `\(.key)` → `\(.value)`"' "$summary")" fi - i_failed="$(printf '%s' "$st" | "$JQ_BIN" -r '.import // {} | to_entries[] | .value.group as $g | .value.failed[]? | "- [ ] `\($g)/\(.)` — see the import output for the API error; fix terraform.tfvars (or workspaceOverrides) and re-run `./sg-migrate.sh import`"')" + i_failed="$(printf '%s' "$st" | "$JQ_BIN" -r '.import // {} | to_entries[] | .value.group as $g | .value.failed[]? | "- [ ] `\($g)/\(.)` — see the import output for the API error; fix terraform.tfvars (projectOverrides / workspaceOverrides) and re-run `./sg-migrate.sh import`"')" [ -s "$EXPORT_DIR/terraform-version-fallbacks.log" ] && i_fallback="$(sed 's/^/- [ ] /' "$EXPORT_DIR/terraform-version-fallbacks.log")" i_trig="$(printf '%s' "$st" | "$JQ_BIN" -r '.triggers // {} | to_entries[] | .value.group as $g | (.value.failed[]? | "- [ ] `\($g)/\(.)` — trigger registration failed; check the connector has admin/webhook rights on the repository, then re-run `./sg-migrate.sh triggers`"), (.value.missing[]? | "- [ ] `\($g)/\(.)` — workflow was not imported, so no trigger was registered")')" [ -s "$EXPORT_DIR/state-export-failures.log" ] && i_state="$(sed 's/^/- [ ] /' "$EXPORT_DIR/state-export-failures.log")" @@ -145,7 +145,7 @@ write_checklist() { printf '# Post-import checklist — StackGuardian org %s\n\n' "$ORG" printf 'Generated %s by stackguardian-migrator. Tick items as you complete them.\n\n' "$(state_now)" printf '## 1. Set the real values of the placeholder secrets\n\n' - printf 'TFC never exposes sensitive variable values, so each one was recreated as an SG secret with the value `CHANGE_ME` and referenced from its workflow as `${secret::<name>}`. Set the real values under [Org settings → Secrets](%s).\n\n' "$(secrets_ui_url)" + printf 'TFC never exposes sensitive variable values, so each one was recreated as an SG secret with the value `CHANGE_ME` and referenced from its workflow as `${secret::<name>}`. Set the real values under [Org settings → Secrets](%s). A secret that already existed was left untouched (its value is never overwritten by a re-run) — check it still holds the right value.\n\n' "$(secrets_ui_url)" _cl_block "$i_secrets" [ -n "$i_unstubbed" ] && printf '%s\n\n' "$i_unstubbed" printf '## 2. Workflows that failed to import\n\n' diff --git a/scripts/lib/sg_api.sh b/scripts/lib/sg_api.sh index 6d27353..d9d153f 100644 --- a/scripts/lib/sg_api.sh +++ b/scripts/lib/sg_api.sh @@ -80,6 +80,25 @@ wfgroup_create() { wf_url() { printf '%s/wfgrps/%s/wfs/%s/' "$(sg_org_url)" "$1" "$2"; } wf_triggers_endpoint() { printf '%swebhooks/vcs_triggers/' "$(wf_url "$1" "$2")"; } +# sg_set_vcs_triggers <group> <wf> <json> — POST the VCS triggers. The endpoint +# is an upsert: a second call answers 200 ("VCS triggers updated" / "Webhook +# already exists ..."), so re-runs are safe. Defensively also 0 on a 4xx whose +# body says the webhook exists (older builds); 22 on any other 4xx, 1 otherwise. +sg_set_vcs_triggers() { + local tmp rc=0 + tmp="$(mktemp)" + # Not a $(...) capture: SG_HTTP_CODE must survive into this shell. + sg_api_raw POST "$(wf_triggers_endpoint "$1" "$2")" "$3" >"$tmp" || rc=$? + if [ "$rc" -eq 22 ] && grep -Eiq 'already (exists|registered)|duplicate' "$tmp"; then + rc=0 + elif [ "$rc" -ne 0 ]; then + sg_err " HTTP $SG_HTTP_CODE from vcs_triggers ($1/$2): $(head -c 300 "$tmp")" + if [ "$rc" -eq 22 ] && declare -F explain_api_error >/dev/null; then explain_api_error "$(head -c 2000 "$tmp")"; fi + fi + rm -f "$tmp" + return "$rc" +} + # sg_workflow_exists <group> <wf> — exit 0 when the workflow exists. sg_workflow_exists() { [ "$(sg_http_code GET "$(wf_url "$1" "$2")")" = "200" ]; } diff --git a/scripts/lib/state.sh b/scripts/lib/state.sh index 567aabc..2cf8dc9 100644 --- a/scripts/lib/state.sh +++ b/scripts/lib/state.sh @@ -11,7 +11,14 @@ # "import": { "<seg>": {"at": iso, "payload_sha": sha, "group": g, # "imported": [...], "failed": [...], # "tf_fallback": [...], -# "state_uploaded": [...], "state_failed": [...]} } } +# "state_uploaded": [...], "state_failed": [...]} }, +# "triggers": { "<seg>": {"at": iso, "group": g, "set": [...], "unchanged": [...], +# "failed": [...], "missing": [...], +# "sha": {"<wf>": sha-of-posted-body}} } } +# +# import.<seg>.group is also the anchor of the "never move" check in the import +# plan: a project whose target group changed while its workflows still exist +# in the old group is refused (StackGuardian cannot move workflows). STATE_FILE="${SG_STATE_FILE:-$SG_REPO_ROOT/.sg/state.json}" diff --git a/scripts/migrate.sh b/scripts/migrate.sh index cab5f62..42e98c7 100755 --- a/scripts/migrate.sh +++ b/scripts/migrate.sh @@ -452,12 +452,16 @@ plan_groups() { # VCSTriggers block, register its VCS triggers via the dedicated webhooks # endpoint (the bulk create API silently drops VCSTriggers; this second pass is # what actually wires up the repo webhook). Sends {VCSConfig, VCSTriggers} taken -# straight from the (converted) payload. Per-workflow failures are surfaced but -# do not abort the rest of the file. +# straight from the (converted) payload. The endpoint upserts, so a re-run is +# safe; to avoid needless calls the sha of each body is kept in state +# (triggers.<seg>.sha) and an unchanged, already-set workflow is skipped +# (--fresh re-sends everything). Per-workflow failures are surfaced but do not +# abort the rest of the file. do_set_triggers() { - local f="$1" seg grp n i wf body rc=0 set=0 skip=0 ok=() failed=() missing=() + local f="$1" seg grp n i wf body sha prev rc=0 skip=0 ok=() failed=() missing=() unchanged=() shas='{}' seg="$(seg_of "$f")" grp="$(group_for "$seg")" + prev="$(state_read | "$JQ_BIN" -c --arg s "$seg" '.triggers[$s] // {}')" n="$("$JQ_BIN" 'length' "$f")" for ((i = 0; i < n; i++)); do if [ "$("$JQ_BIN" -r --argjson i "$i" '(.[$i].VCSTriggers // null) != null' "$f")" != "true" ]; then @@ -474,22 +478,28 @@ do_set_triggers() { continue fi body="$("$JQ_BIN" -c --argjson i "$i" '{VCSConfig: .[$i].VCSConfig, VCSTriggers: .[$i].VCSTriggers}' "$f")" - if SG_NO_RETRY_RC=22 sg_retry "$RETRIES" "$RETRY_BASE" -- \ - sg_api_post "$(wf_triggers_endpoint "$grp" "$wf")" "$body"; then - set=$((set + 1)) + sha="$(sg_sha "$body")" + if [ "$FRESH" -eq 0 ] && [ "$("$JQ_BIN" -r --arg w "$wf" --arg h "$sha" '((.sha // {})[$w] // "") == $h and ((.set // []) | index($w) != null)' <<<"$prev")" = "true" ]; then + unchanged+=("$wf") + shas="$("$JQ_BIN" -c --arg w "$wf" --arg h "$sha" '.[$w] = $h' <<<"$shas")" + continue + fi + if SG_NO_RETRY_RC=22 sg_retry "$RETRIES" "$RETRY_BASE" -- sg_set_vcs_triggers "$grp" "$wf" "$body"; then ok+=("$wf") + shas="$("$JQ_BIN" -c --arg w "$wf" --arg h "$sha" '.[$w] = $h' <<<"$shas")" else sg_warn " vcs triggers failed: $grp/$wf" failed+=("$wf") rc=1 fi done - sg_log "$(basename "$f"): set triggers on $set workflow(s) (skipped $skip without triggers)" - "$JQ_BIN" -nc --arg g "$grp" \ + sg_log "$(basename "$f"): triggers set on ${#ok[@]} workflow(s), ${#unchanged[@]} unchanged, skipped $skip without triggers" + "$JQ_BIN" -nc --arg g "$grp" --argjson sha "$shas" \ --argjson ok "$([ "${#ok[@]}" -gt 0 ] && names_json "${ok[@]}" || echo '[]')" \ + --argjson unchanged "$([ "${#unchanged[@]}" -gt 0 ] && names_json "${unchanged[@]}" || echo '[]')" \ --argjson failed "$([ "${#failed[@]}" -gt 0 ] && names_json "${failed[@]}" || echo '[]')" \ --argjson missing "$([ "${#missing[@]}" -gt 0 ] && names_json "${missing[@]}" || echo '[]')" \ - '{group: $g, set: $ok, failed: $failed, missing: $missing}' >"$EXPORT_DIR/.triggers-result.$seg.json" + '{group: $g, set: ($ok + $unchanged), unchanged: $unchanged, failed: $failed, missing: $missing, sha: $sha}' >"$EXPORT_DIR/.triggers-result.$seg.json" return "$rc" } From 3e35ac4cf2f6d97e946aff3b1cc93b63240c4b84 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adis=20Halilovi=C4=87?= <adis.halilovic@stackguardian.io> Date: Mon, 14 Sep 2026 12:45:27 +0200 Subject: [PATCH 42/71] feat: init asks per project; hand-written overrides survive a re-run MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit With more than one TFC project selected, the wizard's new step 3 asks whether the global connectors and the tfc-<project> groups apply to all projects. If not, it asks per project for the cloud connector, the VCS connector (kind and repo prefix follow, with the usual mismatch warning) and the workflow group — the default tfc-<project>, one of the org's existing groups (GET wfgrps/listall/) or a typed name — and writes the answers as projectOverrides. "same as the default" removes the key so the global value flows through; fields the wizard does not manage (Approvers, RunnerConstraints, ...) are kept. A re-run with existing entries defaults to reviewing them and pre-selects the previous picks. terraform.tfvars is rendered wholesale from the wizard's answers, so a hand-written workspaceOverrides (or projectOverrides) block used to be deleted by the next init. Both maps are now read before the write and re-rendered through a JSON-shaped HCL map renderer (_tfvars_map) that hcl2json round-trips; inner comments are the only thing not preserved. The policy step also asks about stripCloudAuthVars, and the review shows one row per project plus what was kept from the file. --- scripts/lib/tfvars.sh | 50 +++++++++--- scripts/lib/wizard.sh | 186 +++++++++++++++++++++++++++++++++++++++--- 2 files changed, 212 insertions(+), 24 deletions(-) diff --git a/scripts/lib/tfvars.sh b/scripts/lib/tfvars.sh index a0f52ea..803fd54 100644 --- a/scripts/lib/tfvars.sh +++ b/scripts/lib/tfvars.sh @@ -58,6 +58,26 @@ tfvars_valid() { # _tfvars_hcl <json> — pretty-print a JSON value so it reads like HCL in the file. _tfvars_hcl() { printf '%s' "$1" | "$(sg_resolve jq sg_ensure_jq)" --indent 2 '.' 2>/dev/null || printf '%s' "$1"; } +# _tfvars_map <json-object> — a name-keyed HCL map, one entry per key: +# "Project Name" = { +# workflowGroup = "platform-prod" +# DeploymentPlatformConfig = [{"kind":"AWS_RBAC","config":{...}}] +# } +# Attribute values are emitted as JSON, which HCL accepts and hcl2json +# round-trips (tfvars_valid); ${ and %{ inside strings are escaped like +# _tfvars_str does. "{}" for an empty or missing object. +_tfvars_map() { + local j="${1:-}" out + [ -n "$j" ] && [ "$j" != "null" ] || j='{}' + out="$(printf '%s' "$j" | "$(sg_resolve jq sg_ensure_jq)" -r ' + def hcl: tojson | gsub("\\$\\{"; "$${") | gsub("%\\{"; "%%{"); + if (. // {}) == {} then "{}" else + "{\n" + ([to_entries[] | " \(.key | tojson) = {\n" + + ([.value | to_entries[] | " \(.key) = \(.value | hcl)"] | join("\n")) + "\n }"] | join("\n")) + "\n}" + end' 2>/dev/null)" || out="" + printf '%s' "${out:-\{\}}" +} + # _tfvars_str <text> — a quoted HCL string literal (backslashes, quotes and # template sequences escaped, so any connector or org name round-trips). _tfvars_str() { @@ -76,8 +96,10 @@ _tfvars_str() { # W_TFORG W_TFHOST W_WSNAMES_JSON W_TAGS_JSON W_IGNORE_TAGS_JSON W_EXPORT_STATE # W_APPROVERS_JSON W_REPO_PREFIX W_VCS_INTEGRATION W_DPC_JSON W_RUNNER_JSON # W_DEST_KIND W_TF_SOURCE W_TF_VERSION W_TRIGGERS W_IGNORE_PATTERNS_JSON +# W_STRIP_CLOUD W_PROJECT_OVERRIDES_JSON W_WS_OVERRIDES_JSON # W_RUNNER_JSON and W_TF_VERSION may be the literal "null" (defer to the org's -# execution preset). +# execution preset). The two override maps are re-rendered from JSON, so a +# hand-written block survives a re-run of the wizard (its inner comments do not). tfvars_write() { local dest="$1" host_line="" tf_version_hcl if [ "${W_TF_VERSION:-null}" = "null" ]; then tf_version_hcl="null"; else tf_version_hcl="$(_tfvars_str "$W_TF_VERSION")"; fi @@ -110,6 +132,11 @@ exportPath = "export" # name), e.g. TFC_WORKSPACE_NAME or TFC_AWS_RUN_ROLE_ARN. [] keeps everything. ignoreVarPatterns = $W_IGNORE_PATTERNS_JSON +# Cloud credential env variables (ARM_*, AWS_ACCESS_KEY_ID, GOOGLE_CREDENTIALS, +# ...) are replaced by each workflow's cloud connector and are not migrated; +# the family follows the connector kind. See cloudAuthVarPatterns in variables.tf. +stripCloudAuthVars = ${W_STRIP_CLOUD:-true} + # Emails of the users who must approve plans (approvalPreApply is set for # workspaces without auto-apply) SGDefaultWfApprovers = $W_APPROVERS_JSON @@ -148,16 +175,17 @@ SGDefaultEnableVCSTriggers = $W_TRIGGERS # Re-pull state for every workspace on each apply (default: idempotent) forceStateRefresh = false -# Per-workspace overrides, keyed by workspace name. Any field set here wins over -# the SGDefault* value above, for that workspace only. See terraform.tfvars.example -# for every supported field. -# workspaceOverrides = { -# "prod-networking" = { -# RunnerConstraints = { "type" : "private", "names" : ["sg-runner"] } -# Approvers = ["lead@example.com"] -# terraformVersion = "TERRAFORM-1.5.7" -# } -# } +# Per-project settings, keyed by the TFC project name: connectors, runners, +# approvers and the workflow group (workflowGroup, default tfc-<project>) for +# every workspace of that project. Precedence: workspaceOverrides > +# projectOverrides > SGDefault*. See terraform.tfvars.example for every field. +projectOverrides = $(_tfvars_map "${W_PROJECT_OVERRIDES_JSON:-}") + +# Per-workspace overrides, keyed by workspace name; they win over the project +# and default values for that workspace only. Fields: DeploymentPlatformConfig, +# RunnerConstraints, Approvers, vcsAuthIntegrationID, vcsRepoPrefix, +# sourceConfigDestKind, terraformVersion, extraEnvironmentVariables, VCSTriggers. +workspaceOverrides = $(_tfvars_map "${W_WS_OVERRIDES_JSON:-}") TFVARS tfvars_invalidate } diff --git a/scripts/lib/wizard.sh b/scripts/lib/wizard.sh index 2398678..998b501 100644 --- a/scripts/lib/wizard.sh +++ b/scripts/lib/wizard.sh @@ -75,7 +75,7 @@ wizard_tfc() { local host token orgs n ws projects wsn prn scope tags alltags sel prov cnt prefix kind local jqb jqb="$(sg_resolve jq sg_ensure_jq)" - sg_step "1/4 Terraform Cloud / Enterprise" + sg_step "1/5 Terraform Cloud / Enterprise" host="$(sg_ask "TFC/TFE hostname" "$(_w_default .tfHostname app.terraform.io)")" || return 1 W_TFHOST="$host" # shellcheck disable=SC2034 # consumed by tfc_http (lib/tfc_api.sh) @@ -115,13 +115,14 @@ wizard_tfc() { W_TFC_VCS_KINDS="" W_TFC_REPO_PREFIX="" W_TFC_VCS_OTHER=0 + W_SEL_PROJECTS_JSON='[]' projects='[]' if [ "$W_TFC_DISCOVERY" -eq 1 ] && ws="$(tfc_list_workspaces "$W_TFORG" 2>/dev/null)"; then W_TFC_WS_JSON="$ws" wsn="$(printf '%s' "$ws" | "$jqb" 'length')" projects="$(tfc_list_projects "$W_TFORG" 2>/dev/null || echo '[]')" prn="$(printf '%s' "$projects" | "$jqb" 'length')" - sg_log "found $wsn workspace(s) in $prn project(s); each project becomes SG workflow group tfc-<project>" + sg_log "found $wsn workspace(s) in $prn project(s); each project becomes an SG workflow group (tfc-<project> unless you choose otherwise)" [ "$prn" -le 8 ] && [ "$wsn" -gt 0 ] && sg_dim "$(_w_project_counts "$ws" "$projects" 0)" alltags="$(printf '%s' "$ws" | "$jqb" -r '[.[].tags[]?] | unique | join(", ")')" else @@ -157,6 +158,12 @@ wizard_tfc() { W_WS_COUNT="$(printf '%s' "$sel" | "$jqb" 'length')" W_WS_ABOVE_CEILING="$(printf '%s' "$sel" | "$jqb" '[.[] | select((.terraform_version // "") | test("^[0-9]+\\.[0-9]+\\.[0-9]+$")) | select(((.terraform_version | split(".") | map(tonumber)) as $v | ($v[0] > 1) or ($v[0] == 1 and $v[1] > 5) or ($v[0] == 1 and $v[1] == 5 and $v[2] > 7)))] | length')" W_GROUPS="$(_w_project_counts "$sel" "$projects" 1)" + # The selected projects (raw name, payload segment, workspace count), most + # workspaces first — drives the per-project step and the review. + W_SEL_PROJECTS_JSON="$(printf '%s' "$sel" | "$jqb" -c --argjson pr "$projects" ' + ($pr | map({key: .id, value: .name}) | from_entries) as $names + | group_by(.project) | map({id: .[0].project, name: ($names[.[0].project] // .[0].project), count: length}) + | map(. + {segment: (.name | ascii_downcase | gsub("[^a-z0-9-]+"; "-"))}) | sort_by(-.count)' 2>/dev/null || echo '[]')" if [ "$scope" != "all" ]; then if [ "$W_WS_COUNT" -eq 0 ]; then sg_warn "no workspace matches that selection — 'apply' would export nothing" @@ -185,7 +192,7 @@ wizard_tfc() { wizard_sg() { local ints vcs cloud pick kind name runner groups jqb hint prov cnt prefix prev_kind prev_prefix line k jqb="$(sg_resolve jq sg_ensure_jq)" - sg_step "2/4 StackGuardian" + sg_step "2/5 StackGuardian" if [ -z "$ORG" ]; then ORG="$(sg_ask_required "StackGuardian organisation")" || return 1 else @@ -225,6 +232,9 @@ wizard_sg() { done <<<"$(printf '%s' "$ints" | "$jqb" -r '[.[] | select((.type // "") | IN("GITHUB_COM","GITHUB_APP_CUSTOM","GITLAB_COM","GITLAB_OAUTH_SSH","BITBUCKET_ORG","AZURE_DEVOPS","AZURE_DEVOPS_SP","GIT_OTHER"))] | sort_by(.name) | .[] | "\(.name)|\(.type)"')" vcs="$(printf '%s\n' ${first[@]+"${first[@]}"} ${rest[@]+"${rest[@]}"})" fi + # Kept for the per-project step (same lists, same connector table). + W_SG_INTS="${ints:-[]}" + W_VCS_LIST="$vcs" if [ -n "$vcs" ]; then pick="$(_w_select_lines "Which VCS connector should clone the repositories?" "$vcs")" || return 1 kind="$(sg_integration_type "$ints" "$pick")" @@ -266,6 +276,7 @@ wizard_sg() { cloud="" # Only kinds DeploymentPlatformConfig accepts (AZURE_DEVOPS* are VCS connectors). [ "$W_SG_DISCOVERY" -eq 1 ] && cloud="$(printf '%s' "$ints" | "$jqb" -r '[.[] | select((.type // "") | IN("AWS_STATIC","AWS_RBAC","AWS_OIDC","AZURE_STATIC","AZURE_OIDC","AZURE_MANAGED_ID_OIDC","GCP_STATIC","GCP_OIDC"))] | sort_by(.type, .name) | .[] | "\(.name)|\(.type)"')" + W_CLOUD_LIST="$cloud" if [ -n "$cloud" ]; then pick="$(_w_select_lines "Which cloud connector should the workflows deploy with?" "$cloud" "skip|decide later (leaves a placeholder to edit)")" || return 1 kind="$(sg_integration_type "$ints" "$pick")" @@ -337,9 +348,139 @@ wizard_sg() { fi } -# --- step 3: policy ------------------------------------------------------------ +# --- step 3: per-project settings ----------------------------------------------- +# Connectors and the workflow group per TFC project (projectOverrides in +# terraform.tfvars). Asked only when more than one project is selected; a +# re-run that already has entries defaults to reviewing them. "same as the +# default" removes the key, so the global value flows through; fields the +# wizard does not manage (Approvers, RunnerConstraints, ...) are kept as +# written. Every prompt is sg_select/sg_confirm/sg_ask, so -y and +# SG_ANSWERS_FILE keep working. + +# _w_first <lines> <value> — move the "value|..." item to the top (pre-select). +_w_first() { + local lines="$1" v="$2" line first="" rest="" + while IFS= read -r line; do + [ -n "$line" ] || continue + if [ "${line%%|*}" = "$v" ] && [ -z "$first" ]; then first="$line"; else rest="$rest${rest:+$'\n'}$line"; fi + done <<<"$lines" + printf '%s' "${first:+$first$'\n'}$rest" +} + +wizard_projects() { + local jqb n existing def i name segment count prev items pick kind grp groups entry dels desc cur + jqb="$(sg_resolve jq sg_ensure_jq)" + W_PROJECT_OVERRIDES_JSON="$(tfvars_get_json .projectOverrides)" + [ "$W_PROJECT_OVERRIDES_JSON" = "null" ] && W_PROJECT_OVERRIDES_JSON='{}' + W_PROJECT_ROWS="" + sg_step "3/5 Projects" + n="$(printf '%s' "${W_SEL_PROJECTS_JSON:-[]}" | "$jqb" 'length')" + existing="$(printf '%s' "$W_PROJECT_OVERRIDES_JSON" | "$jqb" 'length')" + if [ "$n" -eq 0 ]; then + sg_log "the TFC projects could not be listed — per-project settings can be added as projectOverrides in $(sg_rel "$TFVARS")" + [ "$existing" -gt 0 ] && sg_log "keeping the $existing projectOverrides entr(y/ies) already in the file" + return 0 + fi + for ((i = 0; i < n; i++)); do + IFS=$'\t' read -r name segment count <<<"$(printf '%s' "$W_SEL_PROJECTS_JSON" | "$jqb" -r --argjson i "$i" '.[$i] | [.name, .segment, .count] | @tsv')" + grp="$(printf '%s' "$W_PROJECT_OVERRIDES_JSON" | "$jqb" -r --arg p "$name" '.[$p].workflowGroup // empty')" + sg_dim " $name ($count workspace(s)) -> workflow group ${grp:-tfc-$segment}${grp:+ (from projectOverrides)}" + done + if [ "$n" -eq 1 ]; then + [ "$existing" -gt 0 ] && sg_log "keeping the $existing projectOverrides entr(y/ies) already in the file" + return 0 + fi + def=Y + [ "$existing" -gt 0 ] && def=N + if sg_confirm "Use these connectors and the tfc-<project> groups for all $n projects?" "$def"; then + [ "$existing" -gt 0 ] && sg_log "keeping the $existing projectOverrides entr(y/ies) already in the file" + return 0 + fi + groups="" + [ "$W_SG_DISCOVERY" -eq 1 ] && groups="$(sg_list_wfgrps 2>/dev/null | "$jqb" -r 'sort | .[]' 2>/dev/null || true)" + for ((i = 0; i < n; i++)); do + IFS=$'\t' read -r name segment count <<<"$(printf '%s' "$W_SEL_PROJECTS_JSON" | "$jqb" -r --argjson i "$i" '.[$i] | [.name, .segment, .count] | @tsv')" + sg_log "project '$name' ($count workspace(s))" + prev="$(printf '%s' "$W_PROJECT_OVERRIDES_JSON" | "$jqb" -c --arg p "$name" '.[$p] // {}')" + entry='{}' + dels='[]' + desc="" + + # Cloud connector: the default, or one of the org's cloud connectors. + cur="$(printf '%s' "$prev" | "$jqb" -r '.DeploymentPlatformConfig[0].config.integrationId // empty')" + cur="${cur#/integrations/}" + if [ -n "${W_CLOUD_LIST:-}" ]; then + items="global|same as the default (${W_CLOUD_DESC:-not set})"$'\n'"$W_CLOUD_LIST" + [ -n "$cur" ] && items="$(_w_first "$items" "$cur")" + pick="$(_w_select_lines "Cloud connector for '$name'?" "$items")" || return 1 + kind="$(sg_integration_type "$W_SG_INTS" "$pick")" + else + pick="$(sg_ask "Cloud connector for '$name' (name as in StackGuardian; empty = same as the default)" "$cur")" || return 1 + kind="" + [ -z "$pick" ] && pick="global" + fi + if [ "$pick" = "global" ]; then + dels="$(printf '%s' "$dels" | "$jqb" -c '. + ["DeploymentPlatformConfig"]')" + else + [ -n "$kind" ] || kind="$(sg_select "Connector kind of '$pick'" AWS_RBAC AWS_STATIC AWS_OIDC AZURE_STATIC AZURE_OIDC AZURE_MANAGED_ID_OIDC GCP_STATIC GCP_OIDC)" || return 1 + entry="$(printf '%s' "$entry" | "$jqb" -c --arg k "$kind" --arg i "/integrations/${pick#/integrations/}" '.DeploymentPlatformConfig = [{kind: $k, config: {integrationId: $i}}]')" + desc="cloud: ${pick#/integrations/} ($kind)" + fi + + # VCS connector: the default, or another connector (kind and repo prefix follow). + cur="$(printf '%s' "$prev" | "$jqb" -r '.vcsAuthIntegrationID // empty')" + cur="${cur#/integrations/}" + if [ -n "${W_VCS_LIST:-}" ]; then + items="global|same as the default (${W_VCS_INTEGRATION#/integrations/})"$'\n'"$W_VCS_LIST" + [ -n "$cur" ] && items="$(_w_first "$items" "$cur")" + pick="$(_w_select_lines "VCS connector for '$name'?" "$items")" || return 1 + kind="$(sg_vcs_kind_of "$(sg_integration_type "$W_SG_INTS" "$pick")")" + else + pick="$(sg_ask "VCS connector for '$name' (name as in StackGuardian; empty = same as the default)" "$cur")" || return 1 + kind="" + [ -z "$pick" ] && pick="global" + fi + if [ "$pick" = "global" ]; then + dels="$(printf '%s' "$dels" | "$jqb" -c '. + ["vcsAuthIntegrationID", "sourceConfigDestKind", "vcsRepoPrefix"]')" + else + [ -n "$kind" ] || kind="$(sg_select "VCS provider kind of '$pick'" GITHUB_COM GITLAB_COM BITBUCKET_ORG AZURE_DEVOPS GIT_OTHER)" || return 1 + entry="$(printf '%s' "$entry" | "$jqb" -c --arg i "/integrations/${pick#/integrations/}" '.vcsAuthIntegrationID = $i')" + if [ "$kind" != "$W_DEST_KIND" ]; then + entry="$(printf '%s' "$entry" | "$jqb" -c --arg k "$kind" --arg p "$(_w_repo_prefix_for "$kind")" '.sourceConfigDestKind = $k | .vcsRepoPrefix = $p')" + [ -n "${W_TFC_VCS_KINDS:-}" ] && ! _w_tfc_kind_matches "$kind" && sg_warn "the TFC workspaces are connected to $(tfc_vcs_label_for "${W_TFC_VCS%% *}") but '$pick' is a $kind connector" + else + dels="$(printf '%s' "$dels" | "$jqb" -c '. + ["sourceConfigDestKind", "vcsRepoPrefix"]')" + fi + desc="$desc${desc:+ · }VCS: ${pick#/integrations/} ($kind)" + fi + + # Workflow group: the default tfc-<project>, an existing group, or a new name. + cur="$(printf '%s' "$prev" | "$jqb" -r '.workflowGroup // empty')" + items="default|tfc-$segment (created if it does not exist)" + [ -n "$groups" ] && items="$items"$'\n'"$(printf '%s\n' "$groups" | grep -v -x -F "tfc-$segment" | sed 's/$/|existing workflow group/')" + items="$items"$'\n'"new|another name (typed; created if it does not exist)" + if [ -n "$cur" ] && [ "$cur" != "tfc-$segment" ]; then + if printf '%s\n' "$groups" | grep -q -x -F "$cur"; then items="$(_w_first "$items" "$cur")"; else items="$cur|from the current tfvars (created if it does not exist)"$'\n'"$items"; fi + fi + grp="$(_w_select_lines "Workflow group for '$name'?" "$items")" || return 1 + [ "$grp" = "new" ] && { grp="$(sg_ask_required "Workflow group name for '$name'")" || return 1; } + if [ "$grp" = "default" ] || [ "$grp" = "tfc-$segment" ]; then + dels="$(printf '%s' "$dels" | "$jqb" -c '. + ["workflowGroup"]')" + else + entry="$(printf '%s' "$entry" | "$jqb" -c --arg g "$grp" '.workflowGroup = $g')" + desc="$desc${desc:+ · }group: $grp" + fi + + # Merge: unmanaged fields of the existing entry stay; "global" picks delete. + W_PROJECT_OVERRIDES_JSON="$(printf '%s' "$W_PROJECT_OVERRIDES_JSON" | "$jqb" -c --arg p "$name" --argjson e "$entry" --argjson d "$dels" \ + '.[$p] = (((.[$p] // {}) | delpaths([$d[] | [.]])) + $e) | if .[$p] == {} then del(.[$p]) else . end')" + W_PROJECT_ROWS="$W_PROJECT_ROWS${W_PROJECT_ROWS:+$'\n'}$name|${desc:-defaults}" + done +} + +# --- step 4: policy ------------------------------------------------------------ wizard_policy() { - sg_step "3/4 Workflow defaults" + sg_step "4/5 Workflow defaults" # Approvers, repo prefix and the fallback Terraform version are plain values # with sensible defaults — edit them in terraform.tfvars if needed. W_APPROVERS_JSON="$(tfvars_get_json .SGDefaultWfApprovers)" @@ -378,12 +519,18 @@ wizard_policy() { else W_IGNORE_PATTERNS_JSON='[]' fi + sg_dim "Cloud credential env variables (ARM_CLIENT_SECRET, AWS_ACCESS_KEY_ID, GOOGLE_CREDENTIALS, ...) are provided by the workflow's cloud connector in StackGuardian." + if sg_confirm "Strip the cloud credential variables the cloud connector replaces?" "$([ "$(_w_default .stripCloudAuthVars true)" = "false" ] && echo N || echo Y)"; then + W_STRIP_CLOUD=true + else + W_STRIP_CLOUD=false + fi } -# --- step 4: review + write ---------------------------------------------------- +# --- step 5: review + write ---------------------------------------------------- wizard_review() { - local scope tf yn_state yn_trig strip - sg_step "4/4 Review" + local scope tf yn_state yn_trig strip line k + sg_step "5/5 Review" sg_row "TFC" "$W_TFHOST / $W_TFORG" case "${W_SCOPE:-all}" in tags) scope="workspaces tagged $(_w_csv "$W_TAGS_JSON")" ;; @@ -395,7 +542,15 @@ wizard_review() { if [ "${W_SCOPE:-all}" = "all" ]; then scope="$scope ($W_WS_COUNT)"; else scope="$scope — $W_WS_COUNT of $W_WS_TOTAL match"; fi fi sg_row "Workspaces" "$scope" - [ -n "${W_GROUPS:-}" ] && sg_row "Workflow groups" "$W_GROUPS" + if [ -n "${W_PROJECT_ROWS:-}" ]; then + while IFS='|' read -r k line; do [ -n "$k" ] && sg_row "Project '$k'" "$line"; done <<<"$W_PROJECT_ROWS" + elif [ -n "${W_GROUPS:-}" ]; then + sg_row "Workflow groups" "$W_GROUPS" + fi + k="$(printf '%s' "${W_PROJECT_OVERRIDES_JSON:-{\}}" | "$(sg_resolve jq sg_ensure_jq)" 'length' 2>/dev/null || echo 0)" + [ -z "${W_PROJECT_ROWS:-}" ] && [ "$k" -gt 0 ] && sg_row "Project settings" "$k projectOverrides entr(y/ies) kept from the current file" + k="$(printf '%s' "${W_WS_OVERRIDES_JSON:-{\}}" | "$(sg_resolve jq sg_ensure_jq)" 'length' 2>/dev/null || echo 0)" + [ "$k" -gt 0 ] && sg_row "Workspace overrides" "$k workspaceOverrides entr(y/ies) kept from the current file" sg_row "StackGuardian org" "$ORG" sg_row "VCS connector" "${W_VCS_INTEGRATION#/integrations/} ($W_DEST_KIND) — repositories under $W_REPO_PREFIX" sg_row "Cloud connector" "$W_CLOUD_DESC" @@ -404,8 +559,9 @@ wizard_review() { [ "$W_TRIGGERS" = "true" ] && yn_trig="yes, from each workspace's TFC settings" || yn_trig=no sg_row "State export" "$yn_state" sg_row "VCS triggers" "$yn_trig" - [ "$W_IGNORE_PATTERNS_JSON" = "[]" ] && strip="none" || strip="TFC_*, TFE_*" - sg_row "Strip variables" "$strip" + [ "$W_IGNORE_PATTERNS_JSON" = "[]" ] && strip="" || strip="TFC_*, TFE_*" + [ "${W_STRIP_CLOUD:-true}" = "true" ] && strip="$strip${strip:+; }cloud credentials of the connector's kind (ARM_*, AWS_ACCESS_KEY_ID, ...)" + sg_row "Strip variables" "${strip:-none}" if [ "${W_TF_SOURCE:-carry}" = "preset" ]; then tf="from the org's execution preset${W_PRESET_TFVER:+ (now: $W_PRESET_TFVER)}" else @@ -418,7 +574,7 @@ wizard_review() { fi sg_row "Terraform version" "$tf" [ "${W_DPC_PLACEHOLDER:-0}" -eq 1 ] && sg_warn "cloud connector left as a placeholder — edit SGDefaultDeploymentPlatformConfig in $(sg_rel "$TFVARS") before 'apply'" - [ "${W_TFC_VCS_OTHER:-0}" -gt 0 ] && sg_warn "$W_TFC_VCS_OTHER workspace(s) use a different VCS provider than the default above — give them their own connector/prefix via workspaceOverrides in $(sg_rel "$TFVARS")" + [ "${W_TFC_VCS_OTHER:-0}" -gt 0 ] && sg_warn "$W_TFC_VCS_OTHER workspace(s) use a different VCS provider than the default above — give them their own connector/prefix via projectOverrides or workspaceOverrides in $(sg_rel "$TFVARS")" sg_dim "approvers and the repo URL prefix can be edited in $(sg_rel "$TFVARS")" sg_confirm "Write $(sg_rel "$TFVARS")?" Y } @@ -426,7 +582,11 @@ wizard_review() { # wizard_run — the whole flow; returns non-zero when aborted. wizard_run() { local kept="" - wizard_tfc && wizard_sg && wizard_policy || { sg_err "init aborted"; return 1; } + # Hand-written override blocks survive the rewrite (read before writing: the + # tfvars cache is invalidated by tfvars_write). + W_WS_OVERRIDES_JSON="$(tfvars_get_json .workspaceOverrides)" + [ "$W_WS_OVERRIDES_JSON" = "null" ] && W_WS_OVERRIDES_JSON='{}' + wizard_tfc && wizard_sg && wizard_projects && wizard_policy || { sg_err "init aborted"; return 1; } wizard_review || { sg_log "nothing written"; return 1; } if [ -f "$TFVARS" ]; then cp "$TFVARS" "$TFVARS.bak" From 09bc5e7d09b278c9de4142c3e8e9c468c51f74fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adis=20Halilovi=C4=87?= <adis.halilovic@stackguardian.io> Date: Mon, 14 Sep 2026 12:45:27 +0200 Subject: [PATCH 43/71] feat: preflight covers projectOverrides and the workflow-group policy Connector ids, private runner groups and DeploymentPlatformConfig kinds inside projectOverrides are checked like the global and per-workspace ones; a project's VCS connector is compared with the VCS kind that project ends up with; projectOverrides keys that match no TFC project are flagged as a typo (the real project names are listed). The import context states the workflow-group policy (reuse when it exists, create otherwise, or must-exist with --no-create-groups) and warns early when a project's group changed since its last import, which the plan would refuse as a move. One line reports whether cloud credential variables are stripped. --- scripts/lib/preflight.sh | 54 ++++++++++++++++++++++++++++++++++++++-- 1 file changed, 52 insertions(+), 2 deletions(-) diff --git a/scripts/lib/preflight.sh b/scripts/lib/preflight.sh index ecf29b2..2674d82 100644 --- a/scripts/lib/preflight.sh +++ b/scripts/lib/preflight.sh @@ -56,6 +56,7 @@ preflight_tfc() { fi PF_TFC_WORKSPACES="$body" PF_TFC_SELECTED="$sel" + PF_TFC_PROJECTS="$(tfc_list_projects "$org" 2>/dev/null || echo '[]')" else pf_warn "could not list workspaces for '$org' (HTTP $TFC_HTTP_CODE)" fi @@ -114,6 +115,9 @@ preflight_sg() { done < <(tfvars_json | "$jqb" -r ' [ (.SGDefaultVCSAuthIntegrationID | select(. != null and . != "") | ["VCS", .]), ((.SGDefaultDeploymentPlatformConfig // [])[]?.config.integrationId | select(. != null and . != "") | ["cloud", .]), + ((.projectOverrides // {}) | to_entries[]? | .value + | ((.vcsAuthIntegrationID | select(. != null and . != "") | ["project VCS", .]), + ((.DeploymentPlatformConfig // [])[]?.config.integrationId | select(. != null and . != "") | ["project cloud", .]))), ((.workspaceOverrides // {}) | to_entries[]? | .value | ((.vcsAuthIntegrationID | select(. != null and . != "") | ["override VCS", .]), ((.DeploymentPlatformConfig // [])[]?.config.integrationId | select(. != null and . != "") | ["override cloud", .]))) ] @@ -122,9 +126,10 @@ preflight_sg() { # Every private runner group must exist. while IFS= read -r rg; do [ -n "$rg" ] || continue - if sg_runnergroup_exists "$rg"; then pf_ok "runner group '$rg' exists"; else pf_fail "runner group '$rg' not found in org '$ORG' — check SGDefaultRunnerConstraints / workspaceOverrides"; fi + if sg_runnergroup_exists "$rg"; then pf_ok "runner group '$rg' exists"; else pf_fail "runner group '$rg' not found in org '$ORG' — check SGDefaultRunnerConstraints / projectOverrides / workspaceOverrides"; fi done < <(tfvars_json | "$jqb" -r ' [ (.SGDefaultRunnerConstraints // {} | select(.type == "private") | .names[]?), + ((.projectOverrides // {}) | to_entries[]? | .value.RunnerConstraints // {} | select(.type == "private") | .names[]?), ((.workspaceOverrides // {}) | to_entries[]? | .value.RunnerConstraints // {} | select(.type == "private") | .names[]?) ] | unique | .[]') if tfvars_is_null SGDefaultRunnerConstraints; then @@ -210,13 +215,34 @@ preflight_config() { pf_warn "the TFC workspaces are connected to $(tfc_vcs_label_for "$prov") but the VCS kind is $v" fi + # Per-project VCS connector vs. the kind that project ends up with. + local pconn pkind pconn_kind + while IFS=$'\t' read -r line pconn pkind; do + [ -n "$line" ] || continue + [ -n "$pkind" ] || pkind="$v" + pconn_kind="" + [ -n "${PF_SG_INTS:-}" ] && pconn_kind="$(sg_vcs_kind_of "$(sg_integration_type "$PF_SG_INTS" "$pconn")")" + if [ -n "$pconn_kind" ] && [ "$pconn_kind" != "$pkind" ]; then + pf_fail "projectOverrides['$line']: VCS kind $pkind does not match connector ${pconn#/integrations/} ($pconn_kind) — set projectOverrides[\"$line\"].sourceConfigDestKind = \"$pconn_kind\"" + elif [ -n "$pconn_kind" ]; then + pf_ok "projectOverrides['$line']: VCS kind $pkind matches connector ${pconn#/integrations/}" + fi + done < <(tfvars_json | "$jqb" -r '(.projectOverrides // {}) | to_entries[]? | select((.value.vcsAuthIntegrationID // "") != "") | [.key, .value.vcsAuthIntegrationID, (.value.sourceConfigDestKind // "")] | @tsv') + while IFS= read -r v; do [ -n "$v" ] || continue case "$v" in AWS_STATIC | AWS_RBAC | AWS_OIDC | AZURE_STATIC | AZURE_OIDC | AZURE_MANAGED_ID_OIDC | GCP_STATIC | GCP_OIDC) pf_ok "cloud connector kind $v" ;; *) pf_fail "cloud connector kind '$v' (DeploymentPlatformConfig) is not one of AWS_STATIC, AWS_RBAC, AWS_OIDC, AZURE_STATIC, AZURE_OIDC, AZURE_MANAGED_ID_OIDC, GCP_STATIC, GCP_OIDC — a VCS connector was picked as the cloud connector?" ;; esac - done < <(tfvars_json | "$jqb" -r '[ (.SGDefaultDeploymentPlatformConfig // [])[]?.kind, ((.workspaceOverrides // {}) | to_entries[]? | .value.DeploymentPlatformConfig // [] | .[]?.kind) ] | map(select(. != null)) | unique | .[]') + done < <(tfvars_json | "$jqb" -r '[ (.SGDefaultDeploymentPlatformConfig // [])[]?.kind, ((.projectOverrides // {}) | to_entries[]? | .value.DeploymentPlatformConfig // [] | .[]?.kind), ((.workspaceOverrides // {}) | to_entries[]? | .value.DeploymentPlatformConfig // [] | .[]?.kind) ] | map(select(. != null)) | unique | .[]') + + # Cloud credential env vars: stripped per connector kind, or kept. + if [ "$(tfvars_get .stripCloudAuthVars true)" != "false" ]; then + pf_ok "cloud credential variables (ARM_*, AWS_ACCESS_KEY_ID, GOOGLE_CREDENTIALS, ... per connector kind) are stripped — the connector provides them (stripCloudAuthVars)" + else + pf_ok "cloud credential variables are kept (stripCloudAuthVars = false)" + fi # Terraform version policy, and what the execution preset would supply where # tfvars leaves the decision to it. @@ -271,6 +297,16 @@ preflight_config() { fi done < <(tfvars_json | "$jqb" -r '(.workspaceOverrides // {}) | keys[]') fi + if [ -n "${PF_TFC_PROJECTS:-}" ] && [ "$PF_TFC_PROJECTS" != "[]" ]; then + while IFS= read -r v; do + [ -n "$v" ] || continue + if printf '%s' "$PF_TFC_PROJECTS" | "$jqb" -e --arg n "$v" '[.[].name] | index($n) != null' >/dev/null; then + pf_ok "projectOverrides['$v'] matches a TFC project" + else + pf_warn "projectOverrides['$v'] matches no TFC project in the org (typo? projects: $(printf '%s' "$PF_TFC_PROJECTS" | "$jqb" -r '[.[].name] | join(", ")')) — its settings would apply to nothing" + fi + done < <(tfvars_json | "$jqb" -r '(.projectOverrides // {}) | keys[]') + fi return 0 } @@ -283,6 +319,19 @@ preflight_import_inputs() { pf_fail "no payload files in $(sg_rel "$EXPORT_DIR") — run '$PROG apply' first" fi if sg_resolve sg-cli sg_ensure_sgcli >/dev/null 2>&1; then pf_ok "sg-cli available"; else pf_fail "sg-cli not found and could not be downloaded"; fi + # Workflow groups: reused when they exist, created otherwise; the API-backed + # reuse/create/move/collision check is the import plan's. Warn early when a + # project's group changed since its last import (a move would be refused). + local f seg grp prev + if [ "${CREATE_GROUPS:-1}" -eq 1 ]; then pf_ok "workflow groups: reused when they exist, created otherwise"; else pf_ok "workflow groups: must already exist (--no-create-groups)"; fi + for f in ${PF[@]+"${PF[@]}"}; do + seg="$(seg_of "$f")" + grp="$(group_for "$seg")" + prev="$(state_read | "$(sg_resolve jq sg_ensure_jq)" -r --arg s "$seg" '.import[$s].group // empty')" + if [ -n "$prev" ] && [ "$prev" != "$grp" ]; then + pf_warn "$(basename "$f"): last imported into '$prev', the target is now '$grp' — the import plan refuses a move unless '$prev' no longer holds these workflows" + fi + done return 0 } @@ -299,6 +348,7 @@ preflight_run() { PF_WARN=0 PF_TFC_WORKSPACES="" PF_TFC_SELECTED="" + PF_TFC_PROJECTS="" PF_SG_INTS="" PF_PRESET="{}" PF_PRESET_READ=0 From 974588f3e69b1924f70a66c478820204e418be23 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adis=20Halilovi=C4=87?= <adis.halilovic@stackguardian.io> Date: Mon, 14 Sep 2026 12:46:46 +0200 Subject: [PATCH 44/71] docs: per-project settings, group reuse, cloud credential stripping, update semantics README: how init picks connectors and the group per project, the projectOverrides precedence, that hand-written override blocks survive init, the workflow-group rules (reuse/create, never moved, no name collisions across projects sharing a group, mapping file deprecated), the cloud credential variables that are stripped per connector kind, and what a re-run updates (payload change -> update, unchanged -> skip, triggers only when changed, secrets never overwritten). CLAUDE.md follows the code: plan_groups, group_for, the trigger sha, _tfvars_map, wizard_projects, the preflight collectors and the state layout. --- CLAUDE.md | 6 +++--- README.md | 18 ++++++++++-------- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index f185a21..14503a9 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -24,11 +24,11 @@ The migration is a user-driven pipeline, not a single program: The five phases are wrapped by an orchestrator so users don't run them by hand: - `sg-migrate.sh` (host entrypoint, repo root) — runs `scripts/migrate.sh` **inside the Docker image** (`Dockerfile`), bind-mounting the repo at `/app` and forwarding `SG_API_TOKEN`/`SG_ORG`/`SG_BASE_URL`/`TFE_TOKEN`/`TF_TOKEN_*`. For TFC auth it prefers a long-lived `TFE_TOKEN` (env); otherwise it mounts `~/.terraform.d/credentials.tfrc.json` read-only. Runs natively instead when `--native`/`--local` is passed, `SG_NATIVE=1` is set, the command is `clean`/`completion`/help (or no command is given), or Docker is absent. Exports `SG_PROG` so `migrate.sh` shows `./sg-migrate.sh` in its usage/hints. `update` is host-only too (`cmd_update` in the wrapper, never passed to `migrate.sh`): refuses a non-git checkout, a detached HEAD or dirty tracked files, then `git pull --ff-only` and rebuilds the image only when the `Dockerfile` changed between the old and new HEAD — this is how customers pick up fixes from a plain clone (untracked config/output survive). `migrate.sh` lists `update` in its usage/`SG_COMMANDS` and rejects it with a host-only hint. `.gitattributes` forces LF so a Git-for-Windows clone does not CRLF the scripts. -- `scripts/migrate.sh` — the actual orchestrator (sources `scripts/lib/*.sh`, see below). Subcommands `init|preflight|apply|enrich|convert|validate|import|triggers|checklist|all|clean|completion` (no command prints the help menu; `enrich` runs in `all` unless `--no-variable-sets`). Flags beyond the basics: `--dry-run` (import: plan only), `--fresh` (ignore run state), `--project SEG` / `--workspace NAME` (repeatable filters; apply maps `--workspace` to `-var workspacenames=[...]`, import works on a jq-filtered temp copy), `--skip-preflight`, `--no-secret-stubs`. Resolves each tool via `sg_resolve` (PATH first — the image installs them — else `sg_ensure_*` cache). `apply`/`all` first run `require_tfc_auth`, which resolves the token the tfe provider will use (`TFE_TOKEN`, `TF_TOKEN_<host>`, or the `terraform login` file for `tfHostname`) and verifies it with `GET /api/v2/account/details`, failing fast on a missing or rejected (expired) credential. Runs `convert` and `import` in parallel (`--concurrency`, default 4), raises `terraform apply -parallelism`, and retries `sg-cli` imports with backoff (`sg_retry`, `SG_RETRIES`). Import resolves each project's workflow group, checks existence via the SG API, shows a plan (`exists`/`create`), prompts (skip with `-y`), **creates the missing `tfc-<project>` groups via the API**, then imports. Imports go through `import_bulk`: workflows with Terraform variables via `sg-cli workflow create --bulk`, workflows **without** via a direct `POST .../wfs/` (`sg_create_workflow` + `sg_upload_tfstate` in `sg_api.sh`) because sg-cli drops an empty `iacInputData.data` and the API rejects the workflow — a workaround (`TODO(sg-cli)` markers) until sg-cli ships with sg-sdk-go >= v1.5.7, which sends the empty object; the direct path prints sg-cli-style `Failed to create <name>: <code>: <body>` lines so the parsing below covers both. State files are the migrator's job on both paths: `import_bulk` tracks sg-cli's per-workflow upload lines (`State file uploaded successfully` / `Failed to upload state file for <wf>`), re-uploads the failures via `upload_state` → `sg_upload_tfstate` (adds the `x-ms-blob-type: BlockBlob` header Azure Blob requires; sg-cli omits it and also treats anything but a literal `HTTP/1.1 200 OK` as failure — second `TODO(sg-cli)`), and writes `[state] uploaded|failed|none <wf>` markers into the parsed output; `do_import` turns them into `state_uploaded`/`state_failed` in the result, a workflow without its state fails the file (and `state_import_done` re-tries it next run). Existing workflows come back from sg-cli as `Failed to create <wf>: 409: Workflow ID not unique` (its update branch matches the outdated text `Workflow name not unique` — third `TODO(sg-cli)`); `import_bulk` collects those, PATCHes them via `sg_update_workflow`, re-uploads their state and drops the failure line, and `sg_create_workflow` does the same fallback on the direct path (prints `updated`). Before the parallel import, `probe_import` imports one workflow alone (first selected of the first file) and dies unless the create and the state upload both succeeded — the fail-fast for environment problems. sg-cli exits 0 even when individual workflows fail, so `do_import` parses its output: a workflow rejected as above SG's managed Terraform ceiling (1.5.7, last MPL/FOSS release) is re-imported with `SGDefaultTerraformVersion` (read from `terraform.tfvars`; an explicit `null` there means the `terraformVersion` key is deleted instead, so the org's execution preset decides — `tfvars_is_null` tells an explicit null from a missing key), the payload patched in place, and the case logged to `export/terraform-version-fallbacks.log` plus a printed notice; any other per-workflow failure fails the run. `cmd_import` reads the org's execution preset (`sg_execution_preset`, `GET /orgs/{org}/` → `Settings.workflowDefaults`) and `preset_labels` (report.sh) turns it into the `preset (1.5.7)` / `preset (private:rg)` cells of the plan and the `SG_TF_FALLBACK_LABEL` used in messages. The trigger pass skips workflows that do not exist in SG, and `sg_api_post` returns 22 on 4xx so `sg_retry` (via `SG_NO_RETRY_RC`) does not retry definitive errors. `clean` removes local artifacts (`export/`, TF state, tool cache); `clean --all` also removes config. `completion bash|zsh` prints a completion script (`cmd_completion`; commands/options come from `SG_COMMANDS`/`SG_OPTIONS`, keep them in sync with the parser and the host flags); like `clean` it always runs natively. Output is concise by default — `apply` captures terraform's init/plan output and shows only progress + the final summary (or the full log on failure, `TF_IN_AUTOMATION=1`/`-no-color`); `-v`/`--verbose` (`SG_VERBOSE`) streams everything and un-gates the convert detail lines. Colored logging via `sg_step`/`sg_log`/`sg_success`/`sg_warn`/`sg_err`/`sg_row`; paths shown relative via `sg_rel` (all auto-off when not a TTY / `NO_COLOR`). Long steps show a live progress line (`sg_run_quiet` for terraform init/apply, `run_parallel <fn> <max> <label> <items>` for convert/import/triggers — animated only when `SG_ANIMATE=1`, i.e. colors are on; plain log lines otherwise) and report their duration (`sg_fmt_secs`). Phases start with `phase_begin` — inside `all` the header is numbered (`Phase 2/5: ...`, `PHASE_TOTAL` set in `main`) and `phase_took` gives the elapsed time; `run_parallel` prints a job's output as-is when it is a single line and adds the `── <file> ──` header only for longer or failed output (always with `-v`). The import confirmation goes through `sg_confirm` (so `SG_ANSWERS_FILE` works; a declined prompt is a neutral "cancelled" line, not an error) and `import`/`all` end with `finish_line` (outcome, total time, open checklist items from `CHECKLIST_OPEN`). A Ctrl-C is trapped to clear the progress line and print a resume hint. -- `scripts/lib/` — sourced libraries (a `lib/` gitignore rule is negated for this dir): `prompt.sh` (`sg_ask`/`sg_confirm`/`sg_select`; tty or `SG_ANSWERS_FILE` for tests; `SG_NONINTERACTIVE=1`/`-y`/no TTY → defaults), `tfvars.sh` (`tfvars_get` cached hcl2json reads, `tfvars_has`/`tfvars_is_null` to tell an explicit `null` from a missing key, `tfvars_write` renders the wizard's `W_*` vars — `W_TF_SOURCE`, and `W_TF_VERSION`/`W_RUNNER_JSON` may be the literal `null`; plain strings go through `_tfvars_str`, so quotes/backslashes in names stay valid HCL), `tfc_api.sh` (`tfc_http`/`tfc_get_all` paginated, `tfc_list_{orgs,projects,workspaces}` — workspaces include `vcs_provider`/`vcs_url`/`vcs_identifier` —, `tfc_select_workspaces` (the module's name-glob / include-tags(all) / exclude-tags filter, shared by wizard and preflight), `tfc_vcs_summary`/`tfc_vcs_kind_for`/`tfc_vcs_label_for`, `tfc_token` with provider precedence, `require_tfc_auth`), `sg_api.sh` (`_sg_api` 0/22/1 contract + `SG_HTTP_CODE`, `sg_list_integrations`, `sg_integration_type`, `sg_vcs_kind_of` (connector type → `sourceConfigDestKind`), `sg_execution_preset` + `sg_preset_{desc,runner_desc,version_desc,runner_provided}` (the org's execution preset and how to describe it), `sg_*_exists`, `sg_list_workflows`, `sg_create_secret`, `sg_patch_workflow`, `wfgroup_*`), `wizard.sh` (`wizard_run`: 4 steps TFC → SG → defaults → review; the TFC step keeps the workspace list and derives, via `tfc_vcs_summary`, which VCS provider the selected workspaces use and the repositories' URL prefix — the SG step lists matching connectors first and takes the repo URL prefix from TFC (never from a previous tfvars written for another provider kind), warns on a kind mismatch, reads the org's execution preset and offers it as the runner choice; the defaults step asks whether to carry TFC versions or use the preset (`SGTerraformVersionSource`) and, for carry, which fallback (`SGDefaultTerraformVersion` or the preset = null); the review shows human-readable rows plus the `tfc-<project>` groups; `_w_select_lines` feeds menus line-by-line so names with spaces/globs survive), `preflight.sh` (`preflight_run <apply|import|all>`, once per process; ✓ lines are plain language and name the connector's role/kind, ✗/! lines name the tfvars field to fix; besides existence it checks consistency — VCS kind vs. the connector's type (fail), repo URL prefix vs. where the TFC repositories live or the kind's well-known host (warn); it also reports the org's execution preset next to the version/runner policy and warns when `carry` meets a preset that mounts a runner-provided binary), `state.sh` (`.sg/state.json`: `phases.<name>.input_sha`, `import.<seg>.{imported,failed,tf_fallback,state_uploaded,state_failed}`, `triggers.<seg>`, `secrets.<name>`; `run_phase` in migrate.sh skips unchanged phases), `report.sh` (`show_migration_summary` after apply — incl. a state-files row —, `show_import_plan` per-workflow table with columns sized to their content), `errors.sh` (`explain_api_error` regex→hint table, `SG_API_HINTS`), `checklist.sh` (`create_secret_stubs` → SG secret `tfc-<wf>-<VAR>` = `CHANGE_ME`, referenced as `${secret::<name>}` via PATCH + payload patch; `write_checklist` → `export/post-import-checklist.md`, printed to the terminal as one ✓/! status line per section plus the imported groups with UI links — the file has the details; section 5 covers both the TFC state export and the SG state upload (`state_failed` from the import results); sets `CHECKLIST_OPEN`). Parallel jobs (`run_parallel`) run in subshells, so `do_import`/`do_set_triggers` write `.import-result.<seg>.json` / `.triggers-result.<seg>.json` into the export dir and the caller merges them into state. +- `scripts/migrate.sh` — the actual orchestrator (sources `scripts/lib/*.sh`, see below). Subcommands `init|preflight|apply|enrich|convert|validate|import|triggers|checklist|all|clean|completion` (no command prints the help menu; `enrich` runs in `all` unless `--no-variable-sets`). Flags beyond the basics: `--dry-run` (import: plan only), `--fresh` (ignore run state), `--project SEG` / `--workspace NAME` (repeatable filters; apply maps `--workspace` to `-var workspacenames=[...]`, import works on a jq-filtered temp copy), `--skip-preflight`, `--no-secret-stubs`. Resolves each tool via `sg_resolve` (PATH first — the image installs them — else `sg_ensure_*` cache). `apply`/`all` first run `require_tfc_auth`, which resolves the token the tfe provider will use (`TFE_TOKEN`, `TF_TOKEN_<host>`, or the `terraform login` file for `tfHostname`) and verifies it with `GET /api/v2/account/details`, failing fast on a missing or rejected (expired) credential. Runs `convert` and `import` in parallel (`--concurrency`, default 4), raises `terraform apply -parallelism`, and retries `sg-cli` imports with backoff (`sg_retry`, `SG_RETRIES`). Import resolves each project's workflow group with `group_for` (legacy `.sg/workflow-groups.json` entry → the payload's `CLIConfiguration.WorkflowGroup.name` via `payload_group` → `tfc-<seg>`) and runs `plan_groups`: STATUS `reuse` (exists) / `create` (missing; created after the confirmation) / `missing!` (`--no-create-groups`) / `moved!`; it collects `PLAN_PROBLEMS` — a project whose workflows still live in the group recorded in `state.json` (or the default group) when the target changed (SG cannot move workflows), and two projects sharing a group with overlapping names — prints them as ✗ lines, still shows the per-workflow plan (also under `--dry-run`) and then dies. The skip set (`state_import_done`) is computed before the plan so the ACTION column can say `skip`/`update`/`create`. Groups are never PATCHed. Imports go through `import_bulk`: workflows with Terraform variables via `sg-cli workflow create --bulk`, workflows **without** via a direct `POST .../wfs/` (`sg_create_workflow` + `sg_upload_tfstate` in `sg_api.sh`) because sg-cli drops an empty `iacInputData.data` and the API rejects the workflow — a workaround (`TODO(sg-cli)` markers) until sg-cli ships with sg-sdk-go >= v1.5.7, which sends the empty object; the direct path prints sg-cli-style `Failed to create <name>: <code>: <body>` lines so the parsing below covers both. State files are the migrator's job on both paths: `import_bulk` tracks sg-cli's per-workflow upload lines (`State file uploaded successfully` / `Failed to upload state file for <wf>`), re-uploads the failures via `upload_state` → `sg_upload_tfstate` (adds the `x-ms-blob-type: BlockBlob` header Azure Blob requires; sg-cli omits it and also treats anything but a literal `HTTP/1.1 200 OK` as failure — second `TODO(sg-cli)`), and writes `[state] uploaded|failed|none <wf>` markers into the parsed output; `do_import` turns them into `state_uploaded`/`state_failed` in the result, a workflow without its state fails the file (and `state_import_done` re-tries it next run). Existing workflows come back from sg-cli as `Failed to create <wf>: 409: Workflow ID not unique` (its update branch matches the outdated text `Workflow name not unique` — third `TODO(sg-cli)`); `import_bulk` collects those, PATCHes them via `sg_update_workflow`, re-uploads their state and drops the failure line, and `sg_create_workflow` does the same fallback on the direct path (prints `updated`). Before the parallel import, `probe_import` imports one workflow alone (first selected of the first file) and dies unless the create and the state upload both succeeded — the fail-fast for environment problems. sg-cli exits 0 even when individual workflows fail, so `do_import` parses its output: a workflow rejected as above SG's managed Terraform ceiling (1.5.7, last MPL/FOSS release) is re-imported with `SGDefaultTerraformVersion` (read from `terraform.tfvars`; an explicit `null` there means the `terraformVersion` key is deleted instead, so the org's execution preset decides — `tfvars_is_null` tells an explicit null from a missing key), the payload patched in place, and the case logged to `export/terraform-version-fallbacks.log` plus a printed notice; any other per-workflow failure fails the run. `cmd_import` reads the org's execution preset (`sg_execution_preset`, `GET /orgs/{org}/` → `Settings.workflowDefaults`) and `preset_labels` (report.sh) turns it into the `preset (1.5.7)` / `preset (private:rg)` cells of the plan and the `SG_TF_FALLBACK_LABEL` used in messages. The trigger pass (`do_set_triggers` → `sg_set_vcs_triggers`) skips workflows that do not exist in SG, keeps the sha of each posted `{VCSConfig, VCSTriggers}` body in `triggers.<seg>.sha` and does not re-POST an unchanged, already-set workflow (`--fresh` does; the endpoint upserts anyway, a 4xx "already exists" from older builds counts as success), and `sg_api_post` returns 22 on 4xx so `sg_retry` (via `SG_NO_RETRY_RC`) does not retry definitive errors. `clean` removes local artifacts (`export/`, TF state, tool cache); `clean --all` also removes config. `completion bash|zsh` prints a completion script (`cmd_completion`; commands/options come from `SG_COMMANDS`/`SG_OPTIONS`, keep them in sync with the parser and the host flags); like `clean` it always runs natively. Output is concise by default — `apply` captures terraform's init/plan output and shows only progress + the final summary (or the full log on failure, `TF_IN_AUTOMATION=1`/`-no-color`); `-v`/`--verbose` (`SG_VERBOSE`) streams everything and un-gates the convert detail lines. Colored logging via `sg_step`/`sg_log`/`sg_success`/`sg_warn`/`sg_err`/`sg_row`; paths shown relative via `sg_rel` (all auto-off when not a TTY / `NO_COLOR`). Long steps show a live progress line (`sg_run_quiet` for terraform init/apply, `run_parallel <fn> <max> <label> <items>` for convert/import/triggers — animated only when `SG_ANIMATE=1`, i.e. colors are on; plain log lines otherwise) and report their duration (`sg_fmt_secs`). Phases start with `phase_begin` — inside `all` the header is numbered (`Phase 2/5: ...`, `PHASE_TOTAL` set in `main`) and `phase_took` gives the elapsed time; `run_parallel` prints a job's output as-is when it is a single line and adds the `── <file> ──` header only for longer or failed output (always with `-v`). The import confirmation goes through `sg_confirm` (so `SG_ANSWERS_FILE` works; a declined prompt is a neutral "cancelled" line, not an error) and `import`/`all` end with `finish_line` (outcome, total time, open checklist items from `CHECKLIST_OPEN`). A Ctrl-C is trapped to clear the progress line and print a resume hint. +- `scripts/lib/` — sourced libraries (a `lib/` gitignore rule is negated for this dir): `prompt.sh` (`sg_ask`/`sg_confirm`/`sg_select`; tty or `SG_ANSWERS_FILE` for tests; `SG_NONINTERACTIVE=1`/`-y`/no TTY → defaults), `tfvars.sh` (`tfvars_get` cached hcl2json reads, `tfvars_has`/`tfvars_is_null` to tell an explicit `null` from a missing key, `tfvars_write` renders the wizard's `W_*` vars — `W_TF_SOURCE`, and `W_TF_VERSION`/`W_RUNNER_JSON` may be the literal `null`; plain strings go through `_tfvars_str`, so quotes/backslashes in names stay valid HCL; `projectOverrides`/`workspaceOverrides` are rendered from `W_PROJECT_OVERRIDES_JSON`/`W_WS_OVERRIDES_JSON` by `_tfvars_map` — `"key" = { attr = <json> }`, which hcl2json round-trips — so hand-written blocks survive a re-run of `init` (`wizard_run` reads them before writing; inner comments are lost)), `tfc_api.sh` (`tfc_http`/`tfc_get_all` paginated, `tfc_list_{orgs,projects,workspaces}` — workspaces include `vcs_provider`/`vcs_url`/`vcs_identifier` —, `tfc_select_workspaces` (the module's name-glob / include-tags(all) / exclude-tags filter, shared by wizard and preflight), `tfc_vcs_summary`/`tfc_vcs_kind_for`/`tfc_vcs_label_for`, `tfc_token` with provider precedence, `require_tfc_auth`), `sg_api.sh` (`_sg_api` 0/22/1 contract + `SG_HTTP_CODE`, `_sg_listall` paginated listall reader (the API pages at 50) behind `sg_list_workflows`/`sg_list_wfgrps`, `sg_set_vcs_triggers`, `sg_list_integrations`, `sg_integration_type`, `sg_vcs_kind_of` (connector type → `sourceConfigDestKind`), `sg_execution_preset` + `sg_preset_{desc,runner_desc,version_desc,runner_provided}` (the org's execution preset and how to describe it), `sg_*_exists`, `sg_list_workflows`, `sg_create_secret`, `sg_patch_workflow`, `wfgroup_*`), `wizard.sh` (`wizard_run`: 5 steps TFC → SG → projects → defaults → review; `wizard_projects` (with >1 selected project, from `W_SEL_PROJECTS_JSON`) asks whether the global connectors and `tfc-<project>` groups apply to all, else per project the cloud connector, VCS connector (kind + prefix follow) and group (default / existing via `sg_list_wfgrps` / typed), writing `projectOverrides`; "same as the default" deletes the key, unmanaged fields are kept, previous picks come first (`_w_first`); the defaults step also asks `stripCloudAuthVars`; the TFC step keeps the workspace list and derives, via `tfc_vcs_summary`, which VCS provider the selected workspaces use and the repositories' URL prefix — the SG step lists matching connectors first and takes the repo URL prefix from TFC (never from a previous tfvars written for another provider kind), warns on a kind mismatch, reads the org's execution preset and offers it as the runner choice; the defaults step asks whether to carry TFC versions or use the preset (`SGTerraformVersionSource`) and, for carry, which fallback (`SGDefaultTerraformVersion` or the preset = null); the review shows human-readable rows plus the `tfc-<project>` groups; `_w_select_lines` feeds menus line-by-line so names with spaces/globs survive), `preflight.sh` (`preflight_run <apply|import|all>`, once per process; ✓ lines are plain language and name the connector's role/kind, ✗/! lines name the tfvars field to fix; connector ids, runner groups and DPC kinds are collected from `SGDefault*`, `projectOverrides` and `workspaceOverrides`; besides existence it checks consistency — VCS kind vs. the connector's type, globally and per `projectOverrides` entry (fail), repo URL prefix vs. where the TFC repositories live or the kind's well-known host (warn), `projectOverrides` keys vs. the TFC project names (warn); the import context states the group policy and warns when a project's group changed since its last import; it also reports the org's execution preset next to the version/runner policy and warns when `carry` meets a preset that mounts a runner-provided binary), `state.sh` (`.sg/state.json`: `phases.<name>.input_sha`, `import.<seg>.{group,imported,failed,tf_fallback,state_uploaded,state_failed}` (`group` anchors the never-move check), `triggers.<seg>.{set,unchanged,failed,missing,sha}`, `secrets.<name>`; `run_phase` in migrate.sh skips unchanged phases), `report.sh` (`show_migration_summary` after apply — incl. a state-files row —, `show_import_plan` per-workflow table with columns sized to their content), `errors.sh` (`explain_api_error` regex→hint table, `SG_API_HINTS`), `checklist.sh` (`create_secret_stubs` → SG secret `tfc-<wf>-<VAR>` = `CHANGE_ME`, referenced as `${secret::<name>}` via PATCH + payload patch; `write_checklist` → `export/post-import-checklist.md`, printed to the terminal as one ✓/! status line per section plus the imported groups with UI links — the file has the details; section 5 covers both the TFC state export and the SG state upload (`state_failed` from the import results); sets `CHECKLIST_OPEN`). Parallel jobs (`run_parallel`) run in subshells, so `do_import`/`do_set_triggers` write `.import-result.<seg>.json` / `.triggers-result.<seg>.json` into the export dir and the caller merges them into state. - `scripts/tools.sh` — sourced by the other scripts (sets `SG_REPO_ROOT` to its parent dir). Provides `sg_resolve` (PATH-or-cache), `sg_ensure_{jq,hcl2json,yajsv,sgcli}` (pinned downloads into `.sg/cached/`; `sg-cli` is the Go binary), `sg_retry`, the color vars + log helpers. The Docker image bundles these tools at build time (`yajsv` built from source so arm64 works), so in-container runs never download. - **Variable sets** — `scripts/enrich_variable_sets.sh <tfc-org> <payload...>` (run by `cmd_enrich`, which reads `tfOrg`/`tfHostname` from `terraform.tfvars` via `hcl2json` — variable sets live in the **TFC** org, not `SG_ORG`). Lists all sets + vars via the TFC API (Bearer token from the `terraform login` creds file or `TFE_TOKEN`), computes per-workspace effective vars (scope: global/project/workspace; precedence: priority sets > workspace vars > non-priority sets), and merges them into each workflow (matched by workspace name parsed from `TfStateFilePath`). Sensitive set vars and key conflicts are reported. Skipped by `--no-variable-sets`. -- **Workflow groups** — each TFC project maps to an SG workflow group `tfc-<project-segment>`, created via `POST /api/v1/orgs/{org}/wfgrps/` (auth `Authorization: apikey <token>`) if missing. (Both the API calls and sg-cli honor the undocumented `SG_BASE_URL` for non-prod targets.) Groups are addressed by name, so no generated ID is tracked. `.sg/workflow-groups.json` (gitignored, optional) overrides the target group per segment (`{"<segment>": "<existing-group>"}`); override groups are not auto-created. `--no-create-groups` requires all groups to pre-exist. +- **Workflow groups** — each TFC project maps to the SG workflow group the transformer wrote into its payload (`projectOverrides[<project>].workflowGroup`, default `tfc-<project-segment>`); an existing group is reused, a missing one created via `POST /api/v1/orgs/{org}/wfgrps/` (auth `Authorization: apikey <token>`) unless `--no-create-groups`. (Both the API calls and sg-cli honor the undocumented `SG_BASE_URL` for non-prod targets.) Groups are addressed by name, so no generated ID is tracked, and they are never PATCHed; workflows are never moved between groups (see `plan_groups`). `.sg/workflow-groups.json` (gitignored) is a deprecated per-segment override that still applies on top, with a warning. ### The transformer (`transformer/terraform-cloud/`) diff --git a/README.md b/README.md index 97ca78d..0af6a70 100644 --- a/README.md +++ b/README.md @@ -24,11 +24,11 @@ export TFE_TOKEN=<TFC/TFE token> # long-lived API token (User/Team/Org token export SG_API_TOKEN=<your SG token> export SG_ORG=<your SG org> -./sg-migrate.sh init # guided setup: picks TFC org/workspaces, SG connectors, runners -> terraform.tfvars +./sg-migrate.sh init # guided setup: TFC org/workspaces, SG connectors, runners, per-project settings -> terraform.tfvars ./sg-migrate.sh all # preflight -> apply -> enrich -> convert -> validate -> import (shows a plan, asks before importing) ``` -That's it — no IDs to look up and no workflow-group mapping to fill in. `init` lists what the tokens can see (TFC organisations and workspaces, SG VCS/cloud connectors and runner groups, the org's execution preset) and writes `terraform.tfvars` from your picks — it also reads which VCS provider your TFC workspaces are connected to, lists the matching connectors first and takes the repository URL prefix from TFC. `all` verifies every reference **before** running terraform (preflight — including that the VCS connector, the VCS kind and the repo URL prefix agree with each other and with the TFC repositories), prints a migration summary after the export, shows a per-workflow import plan (create/update, Terraform version, runner, triggers, secrets), and ends with a **post-import checklist** of what still needs a human. Phases are numbered, long steps show a live progress line, and every phase reports how long it took. Each TFC project is imported into an SG workflow group named `tfc-<project>`, **created automatically via the API** if it doesn't exist. +That's it — no IDs to look up and no workflow-group mapping to fill in. `init` lists what the tokens can see (TFC organisations and workspaces, SG VCS/cloud connectors and runner groups, the org's execution preset) and writes `terraform.tfvars` from your picks — it also reads which VCS provider your TFC workspaces are connected to, lists the matching connectors first and takes the repository URL prefix from TFC. With more than one TFC project it asks whether the same connectors and groups apply to every project, or lets you pick a cloud connector, a VCS connector and the workflow group per project (see *Per-project settings* below). `all` verifies every reference **before** running terraform (preflight — including that the VCS connector, the VCS kind and the repo URL prefix agree with each other and with the TFC repositories), prints a migration summary after the export, shows a per-workflow import plan (create/update, Terraform version, runner, triggers, secrets), and ends with a **post-import checklist** of what still needs a human. Phases are numbered, long steps show a live progress line, and every phase reports how long it took. Each TFC project is imported into the SG workflow group assigned to it — `tfc-<project>` by default, or the group you picked for that project — **reusing the group when it exists and creating it otherwise**. Workflows are never moved between groups: when a project's workflows already live in another group, the plan stops and says so. - **Updating.** Clone the repo (don't fork it or download the release zip) and run `./sg-migrate.sh update` to pull the latest version; it fast-forwards the checkout and rebuilds the Docker image only if the `Dockerfile` changed. Your `terraform.tfvars`, `export/` and `.sg/` are never tracked, so they survive every update. To stay on a fixed release instead, `git checkout v1.2.2` (then `git checkout master` to follow the latest again). - Single phase: `./sg-migrate.sh preflight|apply|enrich|convert|validate|import|triggers|checklist`. Running `./sg-migrate.sh` with no command prints the help menu. @@ -38,9 +38,9 @@ That's it — no IDs to look up and no workflow-group mapping to fill in. `init` - **Sensitive variables** (which TFC never exposes) are recreated as SG secrets with the value `CHANGE_ME` and referenced from the workflows as `${secret::<name>}`; the checklist lists each one to fill in. Opt out with `--no-secret-stubs`. - TFC **Variable Set** variables are merged into the payloads automatically (the `enrich` phase, via the TFC API); skip it with `--no-variable-sets`. - `./sg-migrate.sh clean` removes local working artifacts (`export/`, Terraform state, run state, tool cache) for a fresh start; add `--all` to also remove config. `clean` (like `update` and `completion`) always runs locally. -- **Override** a project's target group (to reuse an existing group) in `.sg/workflow-groups.json`: `{"<project-segment>": "<existing-group>"}`. Override groups must already exist (they're not auto-created). +- **Per-project settings.** `projectOverrides` in `terraform.tfvars` (keyed by the TFC project name, written by `init` or by hand) sets the cloud connector, VCS connector, runners, approvers, Terraform version and the target workflow group (`workflowGroup`) for every workspace of a project; precedence is `workspaceOverrides` > `projectOverrides` > `SGDefault*`. Re-running `init` keeps hand-written `projectOverrides`/`workspaceOverrides` blocks (comments inside them are not preserved). The older `.sg/workflow-groups.json` mapping still works but is deprecated. - Output is concise by default (terraform's plan/init noise is hidden; shown on error). Add `-v`/`--verbose` for full output. Known API errors come with a hint naming the `terraform.tfvars` field to fix. -- Flags: `-y` skip the import prompt (CI; also makes `init` non-interactive), `--concurrency N` parallel jobs, `--org NAME`, `--no-create-groups` require groups to pre-exist, `--skip-preflight`, `--build` rebuild the image, `--native`/`--local` force a local run even when Docker is available. +- Flags: `-y` skip the import prompt (CI; also makes `init` non-interactive), `--concurrency N` parallel jobs, `--org NAME`, `--no-create-groups` require groups to pre-exist, `--skip-preflight`, `--build` rebuild the image, `--native`/`--local` force a local run even when Docker is available, `--mapping FILE` (deprecated group override map, see *Per-project settings*). - Tuning via env: `SG_RETRIES`, `SG_TF_PARALLELISM`, `SG_NATIVE=1`, `SG_UI_URL` (base URL for the checklist's links, default `https://app.stackguardian.io`). - Tab completion for the current shell session: `source <(./sg-migrate.sh completion)` (bash/zsh detected; or pass `bash`/`zsh`). `init` prints this line. - TFC auth: set `TFE_TOKEN` (recommended — a long-lived token avoids re-running `terraform login`); otherwise the `terraform login` credentials file is mounted read-only into the container. Tokens are only ever read from the environment; `init` never writes them to disk. @@ -50,7 +50,7 @@ The manual, step-by-step flow below remains supported for fine-grained control a ## Prerequisites - An organization on [StackGuardian Platform](https://app.stackguardian.io) -- Optionally, pre-configure VCS, cloud integrations or private runners to use when importing into StackGuardian Platform. To run every workflow on a private runner group, set `SGDefaultRunnerConstraints = { type = "private", names = ["<runner-group>"] }` in `terraform.tfvars` (per-workspace exceptions via `workspaceOverrides[<name>].RunnerConstraints`). +- Optionally, pre-configure VCS, cloud integrations or private runners to use when importing into StackGuardian Platform. To run every workflow on a private runner group, set `SGDefaultRunnerConstraints = { type = "private", names = ["<runner-group>"] }` in `terraform.tfvars` (per-project exceptions via `projectOverrides[<project>].RunnerConstraints`, per-workspace via `workspaceOverrides[<name>].RunnerConstraints`). - Terraform - [sg-cli](https://github.com/StackGuardian/sg-cli) @@ -176,17 +176,19 @@ To update workflows with different details, re-run the sg-cli command with the m ## Notes and limitations -- **Workflow groups.** Each TFC project imports into an SG workflow group `tfc-<project>`, created via the API if missing (disable with `--no-create-groups`). Override the target group per project in `.sg/workflow-groups.json`; override groups must already exist. +- **Workflow groups.** Each TFC project imports into the workflow group the transformer assigned to it: `projectOverrides[<project>].workflowGroup`, or `tfc-<project>`. The import plan shows `reuse` for a group that exists and `create` for one that will be created (`--no-create-groups` requires every group to exist). StackGuardian cannot move workflows between groups and the migrator never updates a group: when a project's workflows already live in another group (the one recorded in `.sg/state.json`, or the default `tfc-<project>` group) the plan shows `moved!` and stops — keep the old group or delete the workflows there first. Two projects may share a group only when their workflow names do not overlap; the plan refuses a collision. `.sg/workflow-groups.json` is still honoured as a deprecated override. +- **Per-project settings.** `projectOverrides` (keyed by the TFC project name) carries the same fields as `workspaceOverrides` plus `workflowGroup`, for every workspace of that project; precedence `workspaceOverrides` > `projectOverrides` > `SGDefault*`. `init` fills it in when you answer no to "use these connectors and the tfc-<project> groups for all projects", preflight checks every connector, runner group and kind in it and warns about keys that match no TFC project, and the migration summary lists which group each project maps to. - **Variable Sets are migrated** (the `enrich` phase) — global, project-, and workspace-scoped sets are resolved per workspace with TFC precedence (priority sets override workspace vars; otherwise workspace vars win). **Sensitive** set variables can't be read from the API, so they're skipped and reported — recreate them as StackGuardian secrets. - **TFC-specific variables are stripped.** Variables whose name matches `ignoreVarPatterns` (default `^TFC_`, `^TFE_`, e.g. `TFC_WORKSPACE_NAME` or the `TFC_AWS_*` dynamic-credential settings) only mean something inside Terraform Cloud and are not migrated, from workspaces or variable sets. They are listed in the migration summary; set `ignoreVarPatterns = []` to keep them. +- **Cloud credential variables are stripped.** A workflow's StackGuardian cloud connector provides the credentials, so the env variables the TFC workspace used for them (`ARM_CLIENT_ID`/`ARM_CLIENT_SECRET`/`ARM_TENANT_ID`/`ARM_SUBSCRIPTION_ID`/`ARM_USE_OIDC`..., `AWS_ACCESS_KEY_ID`/`AWS_SECRET_ACCESS_KEY`/`AWS_SESSION_TOKEN`/`AWS_PROFILE`/`AWS_ROLE_ARN`..., `GOOGLE_CREDENTIALS`/`GOOGLE_APPLICATION_CREDENTIALS`/`CLOUDSDK_AUTH_*`) are not migrated. Which family is stripped follows each workflow's effective connector kind (`AWS_*`, `AZURE_*`, `GCP_*`), from workspaces and variable sets alike; a sensitive one is stripped too, so no placeholder secret is created for it. They are listed in the migration summary; `stripCloudAuthVars = false` keeps them, `cloudAuthVarPatterns` changes the regexes per family. Terraform input variables are never touched. - **Sensitive variables become placeholder secrets.** TFC never returns sensitive values via the API. The export omits them (listed in `migration-summary.md`); after import the orchestrator creates an SG secret `tfc-<workflow>-<VAR>` with the value `CHANGE_ME` for each, references it from the workflow (`${secret::<name>}`, as an environment variable or IaC input) and lists it in `export/post-import-checklist.md`. Set the real values in the SG UI. `--no-secret-stubs` leaves SG secrets untouched. - **Terraform version fallback (FOSS ceiling).** With `SGTerraformVersionSource = "carry"` (the default) pinned versions are carried over as-is and tried first at import, so a custom runtime image or private runner that ships that binary keeps working; workspaces set to `latest` or a version constraint use `SGDefaultTerraformVersion` at export time. StackGuardian's _managed_ runtimes only go up to **1.5.7**, the last MPL-licensed (FOSS) Terraform release; newer versions are BSL-licensed and are not bundled. When the API rejects a workflow for that reason, the importer **automatically re-imports it with `SGDefaultTerraformVersion`** (or, when that is `null`, without any version so the execution preset decides — see the next bullet), patches the payload file to match, prints a notice, and records each case in `export/terraform-version-fallbacks.log`. Those workflows run a different Terraform than they did in TFC, so check compatibility before the first run. To keep a newer version, set `workspaceOverrides[<name>].terraformVersion` to a binary path mounted from a private runner (or use a custom runtime container template) and re-import. - **Execution presets.** StackGuardian org admins can define an execution preset (Settings → Runner groups → Execution presets): default runner constraints plus a Terraform and an OpenTofu configuration that the API applies to any new workflow whose payload does not carry those fields. To lean on it, set `SGTerraformVersionSource = "preset"` (no version is sent at all) and/or `SGDefaultRunnerConstraints = null` (no runner constraints are sent); `SGDefaultTerraformVersion = null` keeps carrying TFC pins but hands the unpinned and rejected ones to the preset. `init` offers these choices with the org's current preset shown inline, preflight prints what the preset would supply, and the import plan marks such cells as `preset (…)`. The preset's custom runtime image or runner-provided binary is inherited in every mode, since the migrator never sets those keys. - **Workspaces without variables are imported directly.** sg-cli (up to v2.2.1) drops an empty `iacInputData.data` from the request, and the API then rejects the workflow with `VCSConfig.iacInputData.data: This field is required.` The importer therefore creates workflows that have no Terraform variables straight through the API (same create, same state upload) and uses sg-cli for the rest. This is a workaround until sg-cli picks up sg-sdk-go v1.5.7, which fixes the dropped key. - **Terraform state is uploaded by the migrator, and checked.** sg-cli's own upload sends a PUT that Azure-backed StackGuardian environments reject (missing `x-ms-blob-type`) and it reports success only on a literal `HTTP/1.1 200 OK`, so the importer re-uploads every state file sg-cli reports as failed and records per workflow whether the store accepted it. A workflow without its state counts as a failed import: it is listed in the checklist (`state: N workflow(s) are in SG without their state`) and the file is retried on the next `import`. -- **Re-runs update existing workflows.** sg-cli answers `409 Workflow ID not unique` for a workflow that already exists instead of updating it (its update path waits for a message the API no longer sends), so the importer updates such workflows itself via PATCH and re-uploads their state. Re-running `import` (or `import --fresh`) is therefore safe and idempotent. +- **Re-runs update existing workflows.** sg-cli answers `409 Workflow ID not unique` for a workflow that already exists instead of updating it (its update path waits for a message the API no longer sends), so the importer updates such workflows itself via PATCH and re-uploads their state. A change in `terraform.tfvars` (connector, runner, approvers, version) reaches existing workflows through `apply` + `import`: the regenerated payload differs, the plan shows `update`, and the workflow is PATCHed; an unchanged file shows `skip`. VCS triggers are re-registered only when their configuration changed (the API upserts them). Secrets that already exist are never overwritten. Re-running `import` (or `import --fresh`) is therefore safe and idempotent. - **Fail fast.** When more than one workflow is to be imported, the importer first imports a single one (the first selected workflow of the first payload file) and requires both the create and its state upload to succeed before the rest is imported in parallel. An environment problem then costs one workflow, not all of them; the probe workflow is simply updated again with its file. - **State export is idempotent.** Re-running `terraform apply` only pulls state for workspaces not yet exported. Set `forceStateRefresh = true` to re-pull everything. Workspaces not using `remote` execution may export incomplete state — see the summary. - **Workflow naming.** `ResourceName` mirrors the TFC workspace name, sanitized to StackGuardian's rules (1-100 chars, `[-a-zA-Z0-9_]`); any rename is listed in the summary and the checklist. -- **Preflight.** `apply`, `import` and `all` first verify the TFC and SG tokens, the TFC org and workspace selection, and that every connector, secret and runner group referenced in `terraform.tfvars` exists. Fix what it reports (or re-run `init`); `--skip-preflight` bypasses it. +- **Preflight.** `apply`, `import` and `all` first verify the TFC and SG tokens, the TFC org and workspace selection, and that every connector, secret and runner group referenced in `terraform.tfvars` (globally, per project and per workspace) exists and agrees with the VCS kinds; `projectOverrides`/`workspaceOverrides` keys are checked against the TFC projects and workspaces. Fix what it reports (or re-run `init`); `--skip-preflight` bypasses it. - **Post-import checklist.** `export/post-import-checklist.md` collects the secrets to fill in, failed imports, Terraform version fallbacks, failed VCS triggers, missing state exports and renames, with links into the SG UI. The terminal shows one status line per section (plus links to the imported workflow groups); the file has the details. From b9eb8be17f339ae0bce808df1b3b3d990bd1ea90 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adis=20Halilovi=C4=87?= <adis.halilovic@stackguardian.io> Date: Mon, 14 Sep 2026 15:12:19 +0200 Subject: [PATCH 45/71] feat: init writes a commented, ready-to-use entry per project and per workspace After the real projectOverrides / workspaceOverrides maps, terraform.tfvars now carries one commented entry for every selected project and workspace that has no entry yet, pre-filled with what it gets today: the picked connectors, runners and approvers (a workspace inherits its project's override when one exists), the default tfc-<project> group, and for a workspace the Terraform version it runs in TFC. Fine-tuning becomes "move the entry up and change a value" instead of typing field names; each entry is annotated with its workspace count or project and version. _tfvars_map aligns attribute names and takes per-key notes. --- scripts/lib/tfvars.sh | 39 ++++++++++++++++++++++++++++++++------- scripts/lib/wizard.sh | 43 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 75 insertions(+), 7 deletions(-) diff --git a/scripts/lib/tfvars.sh b/scripts/lib/tfvars.sh index 803fd54..7dbef6a 100644 --- a/scripts/lib/tfvars.sh +++ b/scripts/lib/tfvars.sh @@ -58,26 +58,40 @@ tfvars_valid() { # _tfvars_hcl <json> — pretty-print a JSON value so it reads like HCL in the file. _tfvars_hcl() { printf '%s' "$1" | "$(sg_resolve jq sg_ensure_jq)" --indent 2 '.' 2>/dev/null || printf '%s' "$1"; } -# _tfvars_map <json-object> — a name-keyed HCL map, one entry per key: -# "Project Name" = { -# workflowGroup = "platform-prod" +# _tfvars_map <json-object> [notes-json] — a name-keyed HCL map, one entry +# per key, attribute names aligned: +# "Project Name" = { # <note for that key, when given> +# workflowGroup = "platform-prod" # DeploymentPlatformConfig = [{"kind":"AWS_RBAC","config":{...}}] # } # Attribute values are emitted as JSON, which HCL accepts and hcl2json # round-trips (tfvars_valid); ${ and %{ inside strings are escaped like # _tfvars_str does. "{}" for an empty or missing object. _tfvars_map() { - local j="${1:-}" out + local j="${1:-}" notes="${2:-}" out [ -n "$j" ] && [ "$j" != "null" ] || j='{}' - out="$(printf '%s' "$j" | "$(sg_resolve jq sg_ensure_jq)" -r ' + [ -n "$notes" ] && [ "$notes" != "null" ] || notes='{}' + out="$(printf '%s' "$j" | "$(sg_resolve jq sg_ensure_jq)" -r --argjson notes "$notes" ' def hcl: tojson | gsub("\\$\\{"; "$${") | gsub("%\\{"; "%%{"); + def pad($w): . + (" " * ($w - length)); if (. // {}) == {} then "{}" else - "{\n" + ([to_entries[] | " \(.key | tojson) = {\n" - + ([.value | to_entries[] | " \(.key) = \(.value | hcl)"] | join("\n")) + "\n }"] | join("\n")) + "\n}" + "{\n" + ([to_entries[] | .key as $k | (.value | keys | map(length) | max // 0) as $w + | " \($k | tojson) = {" + (if ($notes[$k] // "") != "" then " # \($notes[$k])" else "" end) + "\n" + + ([.value | to_entries[] | " \(.key | pad($w)) = \(.value | hcl)"] | join("\n")) + "\n }"] | join("\n")) + "\n}" end' 2>/dev/null)" || out="" printf '%s' "${out:-\{\}}" } +# _tfvars_map_commented <json-object> [notes-json] — the map's entries (without +# the outer braces), every line commented out: ready to be moved into the real +# map above. Empty output for an empty map. +_tfvars_map_commented() { + local out + out="$(_tfvars_map "$@")" + [ "$out" != "{}" ] || return 0 + printf '%s\n' "$out" | sed '1d;$d' | sed 's/^/# /' +} + # _tfvars_str <text> — a quoted HCL string literal (backslashes, quotes and # template sequences escaped, so any connector or org name round-trips). _tfvars_str() { @@ -97,9 +111,12 @@ _tfvars_str() { # W_APPROVERS_JSON W_REPO_PREFIX W_VCS_INTEGRATION W_DPC_JSON W_RUNNER_JSON # W_DEST_KIND W_TF_SOURCE W_TF_VERSION W_TRIGGERS W_IGNORE_PATTERNS_JSON # W_STRIP_CLOUD W_PROJECT_OVERRIDES_JSON W_WS_OVERRIDES_JSON +# W_PROJECT_TEMPLATE_JSON/_NOTES W_WS_TEMPLATE_JSON/_NOTES (commented examples) # W_RUNNER_JSON and W_TF_VERSION may be the literal "null" (defer to the org's # execution preset). The two override maps are re-rendered from JSON, so a # hand-written block survives a re-run of the wizard (its inner comments do not). +# The template maps are written as comments: one ready-to-uncomment entry per +# selected project / workspace, pre-filled with the effective values. tfvars_write() { local dest="$1" host_line="" tf_version_hcl if [ "${W_TF_VERSION:-null}" = "null" ]; then tf_version_hcl="null"; else tf_version_hcl="$(_tfvars_str "$W_TF_VERSION")"; fi @@ -180,12 +197,20 @@ forceStateRefresh = false # every workspace of that project. Precedence: workspaceOverrides > # projectOverrides > SGDefault*. See terraform.tfvars.example for every field. projectOverrides = $(_tfvars_map "${W_PROJECT_OVERRIDES_JSON:-}") +$(if [ -n "${W_PROJECT_TEMPLATE_JSON:-}" ] && [ "$W_PROJECT_TEMPLATE_JSON" != "{}" ]; then + printf '\n# Ready to use: one entry per selected project, pre-filled with the values\n# chosen above. Move a project into projectOverrides = { } and change what should differ.\n' + _tfvars_map_commented "$W_PROJECT_TEMPLATE_JSON" "${W_PROJECT_TEMPLATE_NOTES:-}" + fi) # Per-workspace overrides, keyed by workspace name; they win over the project # and default values for that workspace only. Fields: DeploymentPlatformConfig, # RunnerConstraints, Approvers, vcsAuthIntegrationID, vcsRepoPrefix, # sourceConfigDestKind, terraformVersion, extraEnvironmentVariables, VCSTriggers. workspaceOverrides = $(_tfvars_map "${W_WS_OVERRIDES_JSON:-}") +$(if [ -n "${W_WS_TEMPLATE_JSON:-}" ] && [ "$W_WS_TEMPLATE_JSON" != "{}" ]; then + printf '\n# Ready to use: one entry per selected workspace, pre-filled with what it gets\n# today (terraformVersion = what it runs in TFC). Move a workspace into\n# workspaceOverrides = { } and change what should differ.\n' + _tfvars_map_commented "$W_WS_TEMPLATE_JSON" "${W_WS_TEMPLATE_NOTES:-}" + fi) TFVARS tfvars_invalidate } diff --git a/scripts/lib/wizard.sh b/scripts/lib/wizard.sh index 998b501..212b775 100644 --- a/scripts/lib/wizard.sh +++ b/scripts/lib/wizard.sh @@ -116,6 +116,7 @@ wizard_tfc() { W_TFC_REPO_PREFIX="" W_TFC_VCS_OTHER=0 W_SEL_PROJECTS_JSON='[]' + W_SEL_WS_JSON='[]' projects='[]' if [ "$W_TFC_DISCOVERY" -eq 1 ] && ws="$(tfc_list_workspaces "$W_TFORG" 2>/dev/null)"; then W_TFC_WS_JSON="$ws" @@ -158,6 +159,7 @@ wizard_tfc() { W_WS_COUNT="$(printf '%s' "$sel" | "$jqb" 'length')" W_WS_ABOVE_CEILING="$(printf '%s' "$sel" | "$jqb" '[.[] | select((.terraform_version // "") | test("^[0-9]+\\.[0-9]+\\.[0-9]+$")) | select(((.terraform_version | split(".") | map(tonumber)) as $v | ($v[0] > 1) or ($v[0] == 1 and $v[1] > 5) or ($v[0] == 1 and $v[1] == 5 and $v[2] > 7)))] | length')" W_GROUPS="$(_w_project_counts "$sel" "$projects" 1)" + W_SEL_WS_JSON="$sel" # The selected projects (raw name, payload segment, workspace count), most # workspaces first — drives the per-project step and the review. W_SEL_PROJECTS_JSON="$(printf '%s' "$sel" | "$jqb" -c --argjson pr "$projects" ' @@ -575,10 +577,50 @@ wizard_review() { sg_row "Terraform version" "$tf" [ "${W_DPC_PLACEHOLDER:-0}" -eq 1 ] && sg_warn "cloud connector left as a placeholder — edit SGDefaultDeploymentPlatformConfig in $(sg_rel "$TFVARS") before 'apply'" [ "${W_TFC_VCS_OTHER:-0}" -gt 0 ] && sg_warn "$W_TFC_VCS_OTHER workspace(s) use a different VCS provider than the default above — give them their own connector/prefix via projectOverrides or workspaceOverrides in $(sg_rel "$TFVARS")" + k="$(printf '%s' "${W_WS_TEMPLATE_JSON:-{\}}" | "$(sg_resolve jq sg_ensure_jq)" 'length' 2>/dev/null || echo 0)" + [ "$k" -gt 0 ] && sg_dim "the file also gets a commented, ready-to-uncomment entry per project and per workspace ($k) for later fine-tuning" sg_dim "approvers and the repo URL prefix can be edited in $(sg_rel "$TFVARS")" sg_confirm "Write $(sg_rel "$TFVARS")?" Y } +# wizard_templates — commented, ready-to-uncomment projectOverrides / +# workspaceOverrides entries for every selected project and workspace that has +# no entry yet, pre-filled with the effective values (the global picks; a +# workspace's terraformVersion is what it runs in TFC). Rendered as comments +# by tfvars_write, so users edit a copy instead of typing field names. +wizard_templates() { + local jqb runner tfv + jqb="$(sg_resolve jq sg_ensure_jq)" + runner="${W_RUNNER_JSON:-null}" + [ "$runner" = "null" ] && runner='{"type":"shared"}' + tfv="${W_TF_VERSION:-null}" + [ "$tfv" = "null" ] && tfv="TERRAFORM-1.5.7" + W_PROJECT_TEMPLATE_JSON="$(printf '%s' "${W_SEL_PROJECTS_JSON:-[]}" | "$jqb" -c --argjson have "${W_PROJECT_OVERRIDES_JSON:-{\}}" \ + --argjson dpc "${W_DPC_JSON:-[]}" --arg vcs "${W_VCS_INTEGRATION:-}" --argjson runner "$runner" --argjson appr "${W_APPROVERS_JSON:-[]}" ' + map(select(.name as $n | ($have | has($n)) | not)) + | map({key: .name, value: {workflowGroup: ("tfc-" + .segment), DeploymentPlatformConfig: $dpc, vcsAuthIntegrationID: $vcs, RunnerConstraints: $runner, Approvers: $appr}}) + | from_entries' 2>/dev/null || echo '{}')" + W_PROJECT_TEMPLATE_NOTES="$(printf '%s' "${W_SEL_PROJECTS_JSON:-[]}" | "$jqb" -c 'map({key: .name, value: "\(.count) workspace(s)"}) | from_entries' 2>/dev/null || echo '{}')" + # A workspace's template shows what it gets today: its project's override + # where one exists, else the global pick. + W_WS_TEMPLATE_JSON="$(printf '%s' "${W_SEL_WS_JSON:-[]}" | "$jqb" -c --argjson have "${W_WS_OVERRIDES_JSON:-{\}}" --argjson projects "${W_PROJECT_OVERRIDES_JSON:-{\}}" --argjson pr "${W_SEL_PROJECTS_JSON:-[]}" \ + --argjson dpc "${W_DPC_JSON:-[]}" --arg vcs "${W_VCS_INTEGRATION:-}" --argjson runner "$runner" --argjson appr "${W_APPROVERS_JSON:-[]}" --arg tfv "$tfv" ' + ($pr | map({key: .id, value: .name}) | from_entries) as $names + | sort_by(.name) + | map(select(.name as $n | ($have | has($n)) | not)) + | map(($projects[$names[.project] // ""] // {}) as $p + | {key: .name, value: { + DeploymentPlatformConfig: ($p.DeploymentPlatformConfig // $dpc), + vcsAuthIntegrationID: ($p.vcsAuthIntegrationID // $vcs), + RunnerConstraints: ($p.RunnerConstraints // $runner), + Approvers: ($p.Approvers // $appr), + terraformVersion: (if ((.terraform_version // "") | test("^[0-9]+\\.[0-9]+\\.[0-9]+$")) then "TERRAFORM-" + .terraform_version else ($p.terraformVersion // $tfv) end)}}) + | from_entries' 2>/dev/null || echo '{}')" + W_WS_TEMPLATE_NOTES="$(printf '%s' "${W_SEL_WS_JSON:-[]}" | "$jqb" -c --argjson pr "${W_SEL_PROJECTS_JSON:-[]}" ' + ($pr | map({key: .id, value: .name}) | from_entries) as $names + | map({key: .name, value: ("project " + ($names[.project] // .project // "?") + (if (.terraform_version // "") != "" then ", Terraform " + .terraform_version else "" end))}) | from_entries' 2>/dev/null || echo '{}')" +} + # wizard_run — the whole flow; returns non-zero when aborted. wizard_run() { local kept="" @@ -587,6 +629,7 @@ wizard_run() { W_WS_OVERRIDES_JSON="$(tfvars_get_json .workspaceOverrides)" [ "$W_WS_OVERRIDES_JSON" = "null" ] && W_WS_OVERRIDES_JSON='{}' wizard_tfc && wizard_sg && wizard_projects && wizard_policy || { sg_err "init aborted"; return 1; } + wizard_templates wizard_review || { sg_log "nothing written"; return 1; } if [ -f "$TFVARS" ]; then cp "$TFVARS" "$TFVARS.bak" From 2e430b2d1eddc13498908103c5dbcb053c1161d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adis=20Halilovi=C4=87?= <adis.halilovic@stackguardian.io> Date: Mon, 14 Sep 2026 15:48:42 +0200 Subject: [PATCH 46/71] fix: --workspace globs select the same workflows in every phase apply already passed the patterns to terraform, but the plan, the import subset, the probe, the triggers and the secret stubs compared exact names, so 'import --workspace team-*' imported nothing. One matcher (bash case / jq ws_selected) now serves all of them; '*' alone means no filter so the unchanged-file skip still applies, and a filter matching nothing is an error instead of an empty success. --- scripts/lib/report.sh | 6 ++-- scripts/migrate.sh | 64 ++++++++++++++++++++++++++++++++----------- 2 files changed, 51 insertions(+), 19 deletions(-) diff --git a/scripts/lib/report.sh b/scripts/lib/report.sh index 6f609d7..6966787 100644 --- a/scripts/lib/report.sh +++ b/scripts/lib/report.sh @@ -124,12 +124,12 @@ show_import_plan() { grp="$(group_for "$seg")" existing="$(sg_list_workflows "$grp")" printf '%s' "$existing" | "$JQ_BIN" -e 'type == "array"' >/dev/null 2>&1 || existing='[]' - rows="$("$JQ_BIN" -r --argjson ex "$existing" --arg grp "$grp" --argjson ws "$(ws_filter_json)" \ + rows="$("$JQ_BIN" -r --argjson ex "$existing" --arg grp "$grp" --argjson inc "$(ws_filter_json)" \ --argjson skip "${PLAN_SKIP_SEGS:-[]}" --arg seg "$seg" \ - --slurpfile sum "${summary:-/dev/null}" ' + --slurpfile sum "${summary:-/dev/null}" "$WS_SCOPE_JQ"' ($sum[0] // {}) as $S | .[] - | select(($ws | length) == 0 or (.ResourceName as $n | $ws | index($n) != null)) + | select(.ResourceName | ws_selected) | ((.CLIConfiguration.TfStateFilePath // "") | sub(".*/"; "") | sub("\\.tfstate$"; "")) as $wsName | .ResourceName as $n | [ $n, $grp, diff --git a/scripts/migrate.sh b/scripts/migrate.sh index 42e98c7..01c5caf 100755 --- a/scripts/migrate.sh +++ b/scripts/migrate.sh @@ -121,7 +121,8 @@ Options: --no-secret-stubs Do not create placeholder SG secrets for sensitive variables --fresh Ignore the saved run state: redo every phase and re-import everything --project SEG Only handle this TFC project (repeatable; matches sg-payload.<SEG>.json) - --workspace NAME Only handle this workspace (repeatable; apply exports only it) + --workspace GLOB Only handle matching workspaces (repeatable; "team-*", "*" = all; + apply exports only them, later phases select the same ones) --all With 'clean': also remove config (terraform.tfvars, mapping, .sg) -v, --verbose Show full terraform/tool output (default: concise) -y, --yes Skip the import confirmation prompt @@ -263,17 +264,45 @@ payload_files() { fi } -# ws_filter_json — the --workspace names as a JSON array (empty array = no filter). +# --- run scope ---------------------------------------------------------------- +# terraform.tfvars holds the widest scope (workspacenames, tags, ...); the CLI +# narrows one run, and every phase applies the same selection: the --workspace +# globs go to terraform for the export and are matched against the payload +# entries (ResourceName) afterwards, so 'import --workspace "team-*"' picks the +# workflows 'apply --workspace "team-*"' exported. + +# ws_narrowed — exit 0 when --workspace restricts the run ("*" alone does not, +# so a CI run over everything keeps the unchanged-file skip). +ws_narrowed() { + local p + for p in ${WS_FILTER[@]+"${WS_FILTER[@]}"}; do [ "$p" = "*" ] || return 0; done + return 1 +} + +# ws_filter_json — the --workspace globs as a JSON array ([] = no filter). ws_filter_json() { - if [ "${#WS_FILTER[@]}" -eq 0 ]; then echo '[]'; else names_json "${WS_FILTER[@]}"; fi + JQ_BIN="${JQ_BIN:-$(sg_resolve jq sg_ensure_jq)}" + if ws_narrowed; then names_json "${WS_FILTER[@]}"; else echo '[]'; fi } -# ws_selected <name> — exit 0 when no --workspace filter is set or it lists <name>. +# ws_selected <name> — exit 0 when <name> is in the run's scope. ws_selected() { - [ "${#WS_FILTER[@]}" -eq 0 ] && return 0 - case " ${WS_FILTER[*]} " in *" $1 "*) return 0 ;; *) return 1 ;; esac + local p + ws_narrowed || return 0 + for p in "${WS_FILTER[@]}"; do + # shellcheck disable=SC2254 # unquoted on purpose: $p is a glob + case "$1" in $p) return 0 ;; esac + done + return 1 } +# WS_SCOPE_JQ — the same test for jq programs over payload entries. Callers +# pass --argjson inc "$(ws_filter_json)" and use 'select(.ResourceName | ws_selected)'. +WS_SCOPE_JQ=' + def ws_glob($p): "^" + ($p | gsub("(?<c>[.+^$(){}|\\[\\]\\\\])"; "\\\(.c)") | gsub("\\*"; ".*") | gsub("\\?"; ".")) + "$"; + def ws_selected: . as $n | (($inc | length) == 0 or any($inc[]; . as $p | $n | test(ws_glob($p)))); +' + seg_of() { local b b="$(basename "$1")" @@ -370,7 +399,7 @@ plan_groups() { PLAN_PROBLEMS=() for f in "${paths[@]}"; do seg="$(seg_of "$f")" - names="$("$JQ_BIN" -c --argjson ws "$(ws_filter_json)" '[.[] | select(($ws | length) == 0 or (.ResourceName as $n | $ws | index($n) != null)) | .ResourceName]' "$f")" + names="$("$JQ_BIN" -c --argjson inc "$(ws_filter_json)" "$WS_SCOPE_JQ"'[.[] | select(.ResourceName | ws_selected) | .ResourceName]' "$f")" count="$("$JQ_BIN" 'length' <<<"$names")" PLAN_TOTAL_WF=$((PLAN_TOTAL_WF + count)) grp="$(group_for "$seg")" @@ -425,8 +454,8 @@ plan_groups() { while IFS= read -r line; do [ -n "$line" ] && PLAN_PROBLEMS+=("$line") done < <(for ((i = 0; i < ${#paths[@]}; i++)); do - "$JQ_BIN" -c --arg seg "$(seg_of "${paths[i]}")" --arg grp "${groups[i]}" --argjson ws "$(ws_filter_json)" \ - '{seg: $seg, grp: $grp, names: [.[] | select(($ws | length) == 0 or (.ResourceName as $n | $ws | index($n) != null)) | .ResourceName]}' "${paths[i]}" + "$JQ_BIN" -c --arg seg "$(seg_of "${paths[i]}")" --arg grp "${groups[i]}" --argjson inc "$(ws_filter_json)" \ + "$WS_SCOPE_JQ"'{seg: $seg, grp: $grp, names: [.[] | select(.ResourceName | ws_selected) | .ResourceName]}' "${paths[i]}" done | "$JQ_BIN" -sr ' group_by(.grp)[] | select(length > 1) | .[0].grp as $g | ([.[].names[]] | group_by(.) | map(select(length > 1) | .[0])) as $dups @@ -575,10 +604,12 @@ cmd_apply() { local tflog rc=0 local -a tfvar_args=() TF_PATH="$(dirname "$(sg_resolve jq sg_ensure_jq)"):$PATH" + # --workspace replaces the tfvars workspacenames for this run (globs included, + # "*" = every workspace); the module applies the tag filters on top. if [ "${#WS_FILTER[@]}" -gt 0 ]; then JQ_BIN="${JQ_BIN:-$(sg_resolve jq sg_ensure_jq)}" - tfvar_args=(-var "workspacenames=$(ws_filter_json)") - sg_log "limiting apply to workspace(s): ${WS_FILTER[*]}" + tfvar_args=(-var "workspacenames=$(names_json "${WS_FILTER[@]}")") + sg_log "exporting workspace(s): ${WS_FILTER[*]}" fi if [ "$VERBOSE" -eq 1 ]; then @@ -796,9 +827,9 @@ do_import() { grp="$(group_for "$seg")" # With --workspace, import only the selected workflows (a filtered copy). work="$f" - if [ "${#WS_FILTER[@]}" -gt 0 ]; then + if ws_narrowed; then work="$(mktemp "$EXPORT_DIR/.subset.$seg.XXXXXX")" - "$JQ_BIN" --argjson names "$(ws_filter_json)" 'map(select(.ResourceName as $n | $names | index($n) != null))' "$f" >"$work" + "$JQ_BIN" --argjson inc "$(ws_filter_json)" "$WS_SCOPE_JQ"'map(select(.ResourceName | ws_selected))' "$f" >"$work" if [ "$("$JQ_BIN" 'length' "$work")" -eq 0 ]; then sg_log "$(basename "$f"): no selected workflows — skipped" rm -f "$work" @@ -897,7 +928,7 @@ probe_import() { local f="$1" seg grp name dir probe res seg="$(seg_of "$f")" grp="$(group_for "$seg")" - name="$("$JQ_BIN" -r --argjson ws "$(ws_filter_json)" 'first(.[] | select(($ws | length) == 0 or (.ResourceName as $n | $ws | index($n) != null)) | .ResourceName) // empty' "$f")" + name="$("$JQ_BIN" -r --argjson inc "$(ws_filter_json)" "$WS_SCOPE_JQ"'first(.[] | select(.ResourceName | ws_selected) | .ResourceName) // empty' "$f")" [ -n "$name" ] || return 0 dir="$(mktemp -d "$EXPORT_DIR/.probe.XXXXXX")" probe="$dir/$(basename "$f")" @@ -957,6 +988,7 @@ cmd_import() { # collisions); problems are shown here and stop the run after the full plan. local q grp f seg plan_groups "${PF[@]}" + [ "$PLAN_TOTAL_WF" -gt 0 ] || die "no workflow in $(sg_rel "$EXPORT_DIR")/ matches the --workspace filter (${WS_FILTER[*]-}) — check the glob, or re-run 'apply' with it" # Files already imported in full with identical content are skipped (the # plan shows their workflows as "skip"); --fresh or a --workspace filter @@ -965,7 +997,7 @@ cmd_import() { local skipped=0 import_rc=0 for f in "${PF[@]}"; do seg="$(seg_of "$f")" - if [ "$FRESH" -eq 0 ] && [ "${#WS_FILTER[@]}" -eq 0 ] && state_import_done "$seg" "$(sg_sha_files "$f")"; then + if [ "$FRESH" -eq 0 ] && ! ws_narrowed && state_import_done "$seg" "$(sg_sha_files "$f")"; then skipped=$((skipped + 1)) skip_segs+=("$seg") continue @@ -1140,7 +1172,7 @@ _sg_migrate() { '--no-secret-stubs[Do not create placeholder SG secrets for sensitive vars]' \\ '--fresh[Ignore saved run state: redo every phase]' \\ '*--project[Only this TFC project segment]:segment' \\ - '*--workspace[Only this workspace]:name' \\ + '*--workspace[Only matching workspaces (glob)]:glob' \\ '--all[With clean: also remove config]' \\ '(-v --verbose)'{-v,--verbose}'[Show full terraform/tool output]' \\ '(-y --yes)'{-y,--yes}'[Skip the import confirmation prompt]' \\ From ae572ccf42afd1be8fc327e7ef8e7ca1b9395f38 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adis=20Halilovi=C4=87?= <adis.halilovic@stackguardian.io> Date: Mon, 14 Sep 2026 15:57:28 +0200 Subject: [PATCH 47/71] feat: exclude workspaces by name (tfWorkspaceIgnoreNames, --exclude-workspace) The provider only excludes by tag, so the module filters the selected workspaces itself (local.selectedWorkspaces) and everything downstream, state export included, follows. terraform.tfvars keeps the permanent list, --exclude-workspace adds to it for one run, and the run scope helpers move to lib/scope.sh so apply, preflight, the plan, the import subset, the probe and the triggers use one definition. Excluded workspaces are listed in the migration summary; init keeps the list on a re-run and the review shows it. --- scripts/lib/preflight.sh | 15 ++- scripts/lib/report.sh | 3 +- scripts/lib/scope.sh | 113 ++++++++++++++++++ scripts/lib/tfc_api.sh | 14 ++- scripts/lib/tfvars.sh | 6 +- scripts/lib/wizard.sh | 10 +- scripts/migrate.sh | 91 +++++--------- transformer/terraform-cloud/locals.tf | 32 +++-- transformer/terraform-cloud/resources.tf | 2 +- transformer/terraform-cloud/summary.tmpl | 3 + .../terraform-cloud/terraform.tfvars.example | 4 + transformer/terraform-cloud/variables.tf | 6 + 12 files changed, 215 insertions(+), 84 deletions(-) create mode 100644 scripts/lib/scope.sh diff --git a/scripts/lib/preflight.sh b/scripts/lib/preflight.sh index 2674d82..4669bef 100644 --- a/scripts/lib/preflight.sh +++ b/scripts/lib/preflight.sh @@ -45,14 +45,23 @@ preflight_tfc() { esac return 0 fi - # Workspace selection: mirror the module's filters (names glob, include/exclude tags). + # Workspace selection: mirror the module's filters (names glob, include/exclude + # tags, exclude names) with the CLI scope applied (lib/scope.sh, when loaded), + # so the count is the one the apply that follows will export. if body="$(tfc_list_workspaces "$org" 2>/dev/null)"; then - sel="$(tfc_select_workspaces "$body" "$(tfvars_get_json .workspacenames)" "$(tfvars_get_json .tfWorkspaceTags)" "$(tfvars_get_json .tfWorkspaceIgnoreTags)")" + local names ignore_names + names="$(tfvars_get_json .workspacenames)" + ignore_names="$(tfvars_get_json .tfWorkspaceIgnoreNames)" + if declare -F ws_exclude_json >/dev/null; then + [ "${#WS_FILTER[@]}" -gt 0 ] && names="$(names_json "${WS_FILTER[@]}")" + ignore_names="$(ws_exclude_json)" + fi + sel="$(tfc_select_workspaces "$body" "$names" "$(tfvars_get_json .tfWorkspaceTags)" "$(tfvars_get_json .tfWorkspaceIgnoreTags)" "$ignore_names")" n="$(printf '%s' "$sel" | "$jqb" 'length')" if [ "$n" -gt 0 ]; then pf_ok "$n workspace(s) match the selection (of $(printf '%s' "$body" | "$jqb" 'length') in the org)" else - pf_warn "no workspace matches workspacenames/tfWorkspaceTags/tfWorkspaceIgnoreTags — apply would export nothing" + pf_warn "no workspace matches workspacenames/tfWorkspaceTags/tfWorkspaceIgnoreTags/tfWorkspaceIgnoreNames${WS_FILTER[*]:+ and the --workspace/--exclude-workspace flags} — apply would export nothing" fi PF_TFC_WORKSPACES="$body" PF_TFC_SELECTED="$sel" diff --git a/scripts/lib/report.sh b/scripts/lib/report.sh index 6966787..2c9e371 100644 --- a/scripts/lib/report.sh +++ b/scripts/lib/report.sh @@ -119,12 +119,13 @@ show_import_plan() { local -a names=() groups=() summary="$EXPORT_DIR/migration-summary.json" [ -f "$summary" ] || summary="" + ws_jq_args for f in "$@"; do seg="$(seg_of "$f")" grp="$(group_for "$seg")" existing="$(sg_list_workflows "$grp")" printf '%s' "$existing" | "$JQ_BIN" -e 'type == "array"' >/dev/null 2>&1 || existing='[]' - rows="$("$JQ_BIN" -r --argjson ex "$existing" --arg grp "$grp" --argjson inc "$(ws_filter_json)" \ + rows="$("$JQ_BIN" -r --argjson ex "$existing" --arg grp "$grp" "${WS_JQ_ARGS[@]}" \ --argjson skip "${PLAN_SKIP_SEGS:-[]}" --arg seg "$seg" \ --slurpfile sum "${summary:-/dev/null}" "$WS_SCOPE_JQ"' ($sum[0] // {}) as $S diff --git a/scripts/lib/scope.sh b/scripts/lib/scope.sh new file mode 100644 index 0000000..4e7173a --- /dev/null +++ b/scripts/lib/scope.sh @@ -0,0 +1,113 @@ +#!/bin/bash +# Run scope for the migrator (sourced; needs tools.sh, lib/tfvars.sh, jq). +# +# terraform.tfvars holds the widest scope (workspacenames, tfWorkspaceIgnoreNames, +# the tag filters) and the CLI narrows one run: +# --workspace GLOB replaces workspacenames for the run ("*" = all) +# --exclude-workspace GLOB adds to tfWorkspaceIgnoreNames for the run +# Every phase applies the same selection: apply hands the lists to terraform, +# the later phases match them against the payload entries (ResourceName), so +# 'import --workspace "team-*"' picks the workflows 'apply --workspace "team-*"' +# exported. Globs support * and ?. + +PROJECT_FILTER=() +WS_FILTER=() +WS_EXCLUDE=() + +# names_json <name>... — the arguments as a JSON array of strings. +names_json() { printf '%s\n' "$@" | "$JQ_BIN" -R . | "$JQ_BIN" -s .; } + +_scope_jq() { JQ_BIN="${JQ_BIN:-$(sg_resolve jq sg_ensure_jq)}"; } + +# ws_narrowed — exit 0 when a CLI flag restricts the run ("*" alone does not, +# so a CI run over everything keeps the unchanged-file skip). +ws_narrowed() { + local p + [ "${#WS_EXCLUDE[@]}" -gt 0 ] && return 0 + for p in ${WS_FILTER[@]+"${WS_FILTER[@]}"}; do [ "$p" = "*" ] || return 0; done + return 1 +} + +# ws_filter_json — the --workspace globs as a JSON array ([] = no filter). +ws_filter_json() { + local p + _scope_jq + for p in ${WS_FILTER[@]+"${WS_FILTER[@]}"}; do + [ "$p" = "*" ] || { names_json "${WS_FILTER[@]}"; return; } + done + echo '[]' +} + +# ws_exclude_json — tfWorkspaceIgnoreNames from terraform.tfvars plus the +# --exclude-workspace globs, as a JSON array (memoized per process). +ws_exclude_json() { + local tf + if [ -z "${_WS_EXCLUDE_JSON:-}" ]; then + _scope_jq + tf="$(tfvars_get_json .tfWorkspaceIgnoreNames)" + [ "$tf" = "null" ] && tf='[]' + if [ "${#WS_EXCLUDE[@]}" -eq 0 ]; then + _WS_EXCLUDE_JSON="$("$JQ_BIN" -c . <<<"$tf")" + else + _WS_EXCLUDE_JSON="$("$JQ_BIN" -nc --argjson a "$tf" --argjson b "$(names_json "${WS_EXCLUDE[@]}")" '$a + $b | unique')" + fi + fi + printf '%s' "$_WS_EXCLUDE_JSON" +} + +# ws_selected <name> — exit 0 when <name> is in the run's scope: not excluded, +# and matched by a --workspace glob when one is set. +ws_selected() { + local p + if [ -z "${_WS_EXCLUDE_LIST+x}" ]; then + _WS_EXCLUDE_LIST=() + while IFS= read -r p; do [ -n "$p" ] && _WS_EXCLUDE_LIST+=("$p"); done < <(ws_exclude_json | "$JQ_BIN" -r '.[]') + fi + for p in ${_WS_EXCLUDE_LIST[@]+"${_WS_EXCLUDE_LIST[@]}"}; do + # shellcheck disable=SC2254 # unquoted on purpose: $p is a glob + case "$1" in $p) return 1 ;; esac + done + ws_narrowed || return 0 + for p in ${WS_FILTER[@]+"${WS_FILTER[@]}"}; do + [ "$p" = "*" ] && return 0 + # shellcheck disable=SC2254 + case "$1" in $p) return 0 ;; esac + done + [ "${#WS_FILTER[@]}" -eq 0 ] +} + +# WS_SCOPE_JQ — the same test for jq programs over payload entries. Prepend it +# to the program and pass "${WS_JQ_ARGS[@]}" (after ws_jq_args), then use +# 'select(.ResourceName | ws_selected)'. +WS_SCOPE_JQ=' + def ws_glob($p): "^" + ($p | gsub("(?<c>[.+^$(){}|\\[\\]\\\\])"; "\\\(.c)") | gsub("\\*"; ".*") | gsub("\\?"; ".")) + "$"; + def ws_selected: . as $n + | (($inc | length) == 0 or any($inc[]; . as $p | $n | test(ws_glob($p)))) + and (($exc | length) == 0 or (any($exc[]; . as $p | $n | test(ws_glob($p))) | not)); +' +WS_JQ_ARGS=() +# ws_jq_args — fill WS_JQ_ARGS with the --argjson pairs WS_SCOPE_JQ expects. +ws_jq_args() { WS_JQ_ARGS=(--argjson inc "$(ws_filter_json)" --argjson exc "$(ws_exclude_json)"); } + +# scope_tfvar_args — fill SCOPE_TFVAR_ARGS with the -var flags that hand the +# CLI scope to terraform apply (nothing when no flag is set). +SCOPE_TFVAR_ARGS=() +scope_tfvar_args() { + SCOPE_TFVAR_ARGS=() + _scope_jq + [ "${#WS_FILTER[@]}" -gt 0 ] && SCOPE_TFVAR_ARGS+=(-var "workspacenames=$(names_json "${WS_FILTER[@]}")") + [ "${#WS_EXCLUDE[@]}" -gt 0 ] && SCOPE_TFVAR_ARGS+=(-var "tfWorkspaceIgnoreNames=$(ws_exclude_json)") + return 0 +} + +# scope_describe — one line for the log: what the CLI flags select. +scope_describe() { + local out="" + [ "${#WS_FILTER[@]}" -gt 0 ] && out="workspaces ${WS_FILTER[*]}" + [ "${#WS_EXCLUDE[@]}" -gt 0 ] && out="${out:+$out, }excluding ${WS_EXCLUDE[*]}" + printf '%s' "$out" +} + +# scope_sha_input — the CLI scope as a string for the apply phase hash, so a +# run with a different selection re-runs the export. +scope_sha_input() { printf '%s|%s' "$(ws_filter_json)" "$(ws_exclude_json)"; } diff --git a/scripts/lib/tfc_api.sh b/scripts/lib/tfc_api.sh index ba0bc3c..6d9f771 100644 --- a/scripts/lib/tfc_api.sh +++ b/scripts/lib/tfc_api.sh @@ -87,17 +87,19 @@ tfc_list_workspaces() { vcs_provider: (.attributes."vcs-repo"."service-provider" // ""), vcs_url: (.attributes."vcs-repo"."repository-http-url" // ""), vcs_identifier: (.attributes."vcs-repo".identifier // "")}]' } -# tfc_select_workspaces <workspaces-json> <names-json> <tags-json> <ignore-json> -# — the subset the transformer exports, mirroring tfe_workspace_ids: name globs -# (["*"] = all), include tags (a workspace must carry all of them), exclude -# tags (any of them drops the workspace). null/[] disables a filter. +# tfc_select_workspaces <workspaces-json> <names-json> <tags-json> <ignore-tags-json> [ignore-names-json] +# — the subset the transformer exports, mirroring tfe_workspace_ids plus the +# module's tfWorkspaceIgnoreNames: name globs (["*"] = all), include tags (a +# workspace must carry all of them), exclude tags (any of them drops the +# workspace), exclude name globs. null/[] disables a filter. tfc_select_workspaces() { - printf '%s' "$1" | "$(sg_resolve jq sg_ensure_jq)" -c --argjson names "${2:-null}" --argjson tags "${3:-null}" --argjson ignore "${4:-null}" ' - def glob($p): ("^" + ($p | gsub("\\*"; ".*")) + "$"); + printf '%s' "$1" | "$(sg_resolve jq sg_ensure_jq)" -c --argjson names "${2:-null}" --argjson tags "${3:-null}" --argjson ignore "${4:-null}" --argjson ignoreNames "${5:-null}" ' + def glob($p): ("^" + ($p | gsub("(?<c>[.+^$(){}|\\[\\]\\\\])"; "\\\(.c)") | gsub("\\*"; ".*") | gsub("\\?"; ".")) + "$"); [ .[] | select(($names == null) or ($names == ["*"]) or ([$names[] as $p | (.name | test(glob($p)))] | any)) | select(($tags == null) or (($tags | length) == 0) or ([$tags[] as $t | ([.tags[]?] | index($t) != null)] | all)) | select(($ignore == null) or (($ignore | length) == 0) or (([.tags[]?] | map(select(. as $t | $ignore | index($t) != null)) | length) == 0)) + | select(($ignoreNames == null) or (($ignoreNames | length) == 0) or ([$ignoreNames[] as $p | (.name | test(glob($p)))] | any | not)) ]' } diff --git a/scripts/lib/tfvars.sh b/scripts/lib/tfvars.sh index 7dbef6a..937d7b1 100644 --- a/scripts/lib/tfvars.sh +++ b/scripts/lib/tfvars.sh @@ -107,7 +107,7 @@ _tfvars_str() { # wizard (lists/objects are passed as compact JSON, which HCL accepts). Keeps # the same order and comments as terraform.tfvars.example so the file stays # hand-editable afterwards. -# W_TFORG W_TFHOST W_WSNAMES_JSON W_TAGS_JSON W_IGNORE_TAGS_JSON W_EXPORT_STATE +# W_TFORG W_TFHOST W_WSNAMES_JSON W_TAGS_JSON W_IGNORE_TAGS_JSON W_IGNORE_NAMES_JSON W_EXPORT_STATE # W_APPROVERS_JSON W_REPO_PREFIX W_VCS_INTEGRATION W_DPC_JSON W_RUNNER_JSON # W_DEST_KIND W_TF_SOURCE W_TF_VERSION W_TRIGGERS W_IGNORE_PATTERNS_JSON # W_STRIP_CLOUD W_PROJECT_OVERRIDES_JSON W_WS_OVERRIDES_JSON @@ -142,6 +142,10 @@ tfWorkspaceTags = $W_TAGS_JSON # Exclude workspaces carrying these tags (null = none). Excludes win over includes. tfWorkspaceIgnoreTags = $W_IGNORE_TAGS_JSON +# Exclude workspaces by name (globs, e.g. ["sandbox-*", "*-scratch"]); applied after +# the filters above. sg-migrate.sh --exclude-workspace adds to this list for one run. +tfWorkspaceIgnoreNames = ${W_IGNORE_NAMES_JSON:-[]} + # Directory to export Terraform files to exportPath = "export" diff --git a/scripts/lib/wizard.sh b/scripts/lib/wizard.sh index 212b775..4a0313d 100644 --- a/scripts/lib/wizard.sh +++ b/scripts/lib/wizard.sh @@ -154,7 +154,7 @@ wizard_tfc() { # What the selection looks like (drives the review and the SG step's hints). if [ -n "$W_TFC_WS_JSON" ]; then - sel="$(tfc_select_workspaces "$W_TFC_WS_JSON" "$W_WSNAMES_JSON" "$W_TAGS_JSON" "$W_IGNORE_TAGS_JSON")" + sel="$(tfc_select_workspaces "$W_TFC_WS_JSON" "$W_WSNAMES_JSON" "$W_TAGS_JSON" "$W_IGNORE_TAGS_JSON" "${W_IGNORE_NAMES_JSON:-[]}")" W_WS_TOTAL="$wsn" W_WS_COUNT="$(printf '%s' "$sel" | "$jqb" 'length')" W_WS_ABOVE_CEILING="$(printf '%s' "$sel" | "$jqb" '[.[] | select((.terraform_version // "") | test("^[0-9]+\\.[0-9]+\\.[0-9]+$")) | select(((.terraform_version | split(".") | map(tonumber)) as $v | ($v[0] > 1) or ($v[0] == 1 and $v[1] > 5) or ($v[0] == 1 and $v[1] == 5 and $v[2] > 7)))] | length')" @@ -540,8 +540,11 @@ wizard_review() { names) scope="workspaces named $(_w_csv "$W_WSNAMES_JSON")" ;; *) scope="all workspaces" ;; esac + if [ "${W_IGNORE_NAMES_JSON:-[]}" != "[]" ]; then + scope="$scope, except $(_w_csv "$W_IGNORE_NAMES_JSON") (tfWorkspaceIgnoreNames)" + fi if [ -n "${W_WS_COUNT:-}" ]; then - if [ "${W_SCOPE:-all}" = "all" ]; then scope="$scope ($W_WS_COUNT)"; else scope="$scope — $W_WS_COUNT of $W_WS_TOTAL match"; fi + if [ "${W_SCOPE:-all}" = "all" ] && [ "${W_IGNORE_NAMES_JSON:-[]}" = "[]" ]; then scope="$scope ($W_WS_COUNT)"; else scope="$scope — $W_WS_COUNT of $W_WS_TOTAL match"; fi fi sg_row "Workspaces" "$scope" if [ -n "${W_PROJECT_ROWS:-}" ]; then @@ -628,6 +631,9 @@ wizard_run() { # tfvars cache is invalidated by tfvars_write). W_WS_OVERRIDES_JSON="$(tfvars_get_json .workspaceOverrides)" [ "$W_WS_OVERRIDES_JSON" = "null" ] && W_WS_OVERRIDES_JSON='{}' + # The name exclude list is a hand-edit / CI knob: kept as is, never asked. + W_IGNORE_NAMES_JSON="$(tfvars_get_json .tfWorkspaceIgnoreNames)" + [ "$W_IGNORE_NAMES_JSON" = "null" ] && W_IGNORE_NAMES_JSON='[]' wizard_tfc && wizard_sg && wizard_projects && wizard_policy || { sg_err "init aborted"; return 1; } wizard_templates wizard_review || { sg_log "nothing written"; return 1; } diff --git a/scripts/migrate.sh b/scripts/migrate.sh index 01c5caf..f21a762 100755 --- a/scripts/migrate.sh +++ b/scripts/migrate.sh @@ -13,6 +13,8 @@ source "$SCRIPT_DIR/tools.sh" source "$SCRIPT_DIR/lib/prompt.sh" # shellcheck source=lib/tfvars.sh source "$SCRIPT_DIR/lib/tfvars.sh" +# shellcheck source=lib/scope.sh +source "$SCRIPT_DIR/lib/scope.sh" # shellcheck source=lib/tfc_api.sh source "$SCRIPT_DIR/lib/tfc_api.sh" # shellcheck source=lib/sg_api.sh @@ -53,8 +55,6 @@ PREFLIGHT_DONE=0 FRESH=0 DRY_RUN=0 SECRET_STUBS=1 -PROJECT_FILTER=() -WS_FILTER=() VERBOSE="${SG_VERBOSE:-0}" CONC="${SG_CONCURRENCY:-4}" TF_PARALLELISM="${SG_TF_PARALLELISM:-20}" @@ -122,7 +122,9 @@ Options: --fresh Ignore the saved run state: redo every phase and re-import everything --project SEG Only handle this TFC project (repeatable; matches sg-payload.<SEG>.json) --workspace GLOB Only handle matching workspaces (repeatable; "team-*", "*" = all; - apply exports only them, later phases select the same ones) + replaces workspacenames for this run, every phase selects the same ones) + --exclude-workspace GLOB + Leave matching workspaces out (repeatable; adds to tfWorkspaceIgnoreNames) --all With 'clean': also remove config (terraform.tfvars, mapping, .sg) -v, --verbose Show full terraform/tool output (default: concise) -y, --yes Skip the import confirmation prompt @@ -264,45 +266,6 @@ payload_files() { fi } -# --- run scope ---------------------------------------------------------------- -# terraform.tfvars holds the widest scope (workspacenames, tags, ...); the CLI -# narrows one run, and every phase applies the same selection: the --workspace -# globs go to terraform for the export and are matched against the payload -# entries (ResourceName) afterwards, so 'import --workspace "team-*"' picks the -# workflows 'apply --workspace "team-*"' exported. - -# ws_narrowed — exit 0 when --workspace restricts the run ("*" alone does not, -# so a CI run over everything keeps the unchanged-file skip). -ws_narrowed() { - local p - for p in ${WS_FILTER[@]+"${WS_FILTER[@]}"}; do [ "$p" = "*" ] || return 0; done - return 1 -} - -# ws_filter_json — the --workspace globs as a JSON array ([] = no filter). -ws_filter_json() { - JQ_BIN="${JQ_BIN:-$(sg_resolve jq sg_ensure_jq)}" - if ws_narrowed; then names_json "${WS_FILTER[@]}"; else echo '[]'; fi -} - -# ws_selected <name> — exit 0 when <name> is in the run's scope. -ws_selected() { - local p - ws_narrowed || return 0 - for p in "${WS_FILTER[@]}"; do - # shellcheck disable=SC2254 # unquoted on purpose: $p is a glob - case "$1" in $p) return 0 ;; esac - done - return 1 -} - -# WS_SCOPE_JQ — the same test for jq programs over payload entries. Callers -# pass --argjson inc "$(ws_filter_json)" and use 'select(.ResourceName | ws_selected)'. -WS_SCOPE_JQ=' - def ws_glob($p): "^" + ($p | gsub("(?<c>[.+^$(){}|\\[\\]\\\\])"; "\\\(.c)") | gsub("\\*"; ".*") | gsub("\\?"; ".")) + "$"; - def ws_selected: . as $n | (($inc | length) == 0 or any($inc[]; . as $p | $n | test(ws_glob($p)))); -' - seg_of() { local b b="$(basename "$1")" @@ -397,9 +360,10 @@ plan_groups() { PLAN_N_CREATE=0 PLAN_TOTAL_WF=0 PLAN_PROBLEMS=() + ws_jq_args for f in "${paths[@]}"; do seg="$(seg_of "$f")" - names="$("$JQ_BIN" -c --argjson inc "$(ws_filter_json)" "$WS_SCOPE_JQ"'[.[] | select(.ResourceName | ws_selected) | .ResourceName]' "$f")" + names="$("$JQ_BIN" -c "${WS_JQ_ARGS[@]}" "$WS_SCOPE_JQ"'[.[] | select(.ResourceName | ws_selected) | .ResourceName]' "$f")" count="$("$JQ_BIN" 'length' <<<"$names")" PLAN_TOTAL_WF=$((PLAN_TOTAL_WF + count)) grp="$(group_for "$seg")" @@ -454,7 +418,7 @@ plan_groups() { while IFS= read -r line; do [ -n "$line" ] && PLAN_PROBLEMS+=("$line") done < <(for ((i = 0; i < ${#paths[@]}; i++)); do - "$JQ_BIN" -c --arg seg "$(seg_of "${paths[i]}")" --arg grp "${groups[i]}" --argjson inc "$(ws_filter_json)" \ + "$JQ_BIN" -c --arg seg "$(seg_of "${paths[i]}")" --arg grp "${groups[i]}" "${WS_JQ_ARGS[@]}" \ "$WS_SCOPE_JQ"'{seg: $seg, grp: $grp, names: [.[] | select(.ResourceName | ws_selected) | .ResourceName]}' "${paths[i]}" done | "$JQ_BIN" -sr ' group_by(.grp)[] | select(length > 1) | .[0].grp as $g @@ -604,13 +568,12 @@ cmd_apply() { local tflog rc=0 local -a tfvar_args=() TF_PATH="$(dirname "$(sg_resolve jq sg_ensure_jq)"):$PATH" - # --workspace replaces the tfvars workspacenames for this run (globs included, - # "*" = every workspace); the module applies the tag filters on top. - if [ "${#WS_FILTER[@]}" -gt 0 ]; then - JQ_BIN="${JQ_BIN:-$(sg_resolve jq sg_ensure_jq)}" - tfvar_args=(-var "workspacenames=$(names_json "${WS_FILTER[@]}")") - sg_log "exporting workspace(s): ${WS_FILTER[*]}" - fi + # The CLI scope goes to terraform as -var flags (lib/scope.sh): --workspace + # replaces workspacenames for this run, --exclude-workspace adds to + # tfWorkspaceIgnoreNames; the module applies the tag filters on top. + scope_tfvar_args + tfvar_args=(${SCOPE_TFVAR_ARGS[@]+"${SCOPE_TFVAR_ARGS[@]}"}) + [ "${#tfvar_args[@]}" -gt 0 ] && sg_log "run scope: $(scope_describe)" if [ "$VERBOSE" -eq 1 ]; then # shellcheck disable=SC2119 @@ -813,9 +776,6 @@ import_bulk() { # Terraform release; newer versions are BSL and are not shipped. TF_CEILING_RE='Failed to create ([^:]+): 400: .*above the highest managed version \(([0-9.]+)\)' -# names_json <name...> — JSON array of the given names (for jq --argjson). -names_json() { printf '%s\n' "$@" | "$JQ_BIN" -R . | "$JQ_BIN" -s .; } - # do_import <payload> — bulk-import one file. Workflows rejected because their # Terraform version is above the SG ceiling are re-imported with # SG_DEFAULT_TF_VERSION (the payload file is patched in place so re-runs and @@ -825,11 +785,13 @@ do_import() { local f="$1" seg grp out rc=0 ceiling="" failed=() fb=() st_ok=() st_failed=() name line tmp names work all_names patch seg="$(seg_of "$f")" grp="$(group_for "$seg")" - # With --workspace, import only the selected workflows (a filtered copy). + # With --workspace / --exclude-workspace, import only the selected workflows + # (a filtered copy). work="$f" if ws_narrowed; then + ws_jq_args work="$(mktemp "$EXPORT_DIR/.subset.$seg.XXXXXX")" - "$JQ_BIN" --argjson inc "$(ws_filter_json)" "$WS_SCOPE_JQ"'map(select(.ResourceName | ws_selected))' "$f" >"$work" + "$JQ_BIN" "${WS_JQ_ARGS[@]}" "$WS_SCOPE_JQ"'map(select(.ResourceName | ws_selected))' "$f" >"$work" if [ "$("$JQ_BIN" 'length' "$work")" -eq 0 ]; then sg_log "$(basename "$f"): no selected workflows — skipped" rm -f "$work" @@ -928,7 +890,8 @@ probe_import() { local f="$1" seg grp name dir probe res seg="$(seg_of "$f")" grp="$(group_for "$seg")" - name="$("$JQ_BIN" -r --argjson inc "$(ws_filter_json)" "$WS_SCOPE_JQ"'first(.[] | select(.ResourceName | ws_selected) | .ResourceName) // empty' "$f")" + ws_jq_args + name="$("$JQ_BIN" -r "${WS_JQ_ARGS[@]}" "$WS_SCOPE_JQ"'first(.[] | select(.ResourceName | ws_selected) | .ResourceName) // empty' "$f")" [ -n "$name" ] || return 0 dir="$(mktemp -d "$EXPORT_DIR/.probe.XXXXXX")" probe="$dir/$(basename "$f")" @@ -988,7 +951,7 @@ cmd_import() { # collisions); problems are shown here and stop the run after the full plan. local q grp f seg plan_groups "${PF[@]}" - [ "$PLAN_TOTAL_WF" -gt 0 ] || die "no workflow in $(sg_rel "$EXPORT_DIR")/ matches the --workspace filter (${WS_FILTER[*]-}) — check the glob, or re-run 'apply' with it" + [ "$PLAN_TOTAL_WF" -gt 0 ] || die "no workflow in $(sg_rel "$EXPORT_DIR")/ is in scope ($(scope_describe)) — check the globs, or re-run 'apply' with them" # Files already imported in full with identical content are skipped (the # plan shows their workflows as "skip"); --fresh or a --workspace filter @@ -1095,7 +1058,7 @@ finish_line() { # Single source of truth for shell completion (keep in sync with the parser below # and the host-only flags in sg-migrate.sh). SG_COMMANDS="init preflight apply enrich convert validate import triggers checklist all clean completion update" -SG_OPTIONS="--org --export-dir --mapping --concurrency --no-create-groups --no-variable-sets --no-vcs-triggers --skip-preflight --dry-run --no-secret-stubs --fresh --project --workspace --all -v --verbose -y --yes -h --help --native --local --build" +SG_OPTIONS="--org --export-dir --mapping --concurrency --no-create-groups --no-variable-sets --no-vcs-triggers --skip-preflight --dry-run --no-secret-stubs --fresh --project --workspace --exclude-workspace --all -v --verbose -y --yes -h --help --native --local --build" # cmd_completion <bash|zsh> — print a completion script for sg-migrate.sh / # migrate.sh to stdout. Both shells fall back to the basename when the command @@ -1119,7 +1082,7 @@ _sg_migrate() { case "\$prev" in --export-dir) COMPREPLY=(\$(compgen -d -- "\$cur")); return ;; --mapping) COMPREPLY=(\$(compgen -f -- "\$cur")); return ;; - --org | --concurrency | --project | --workspace) COMPREPLY=(); return ;; + --org | --concurrency | --project | --workspace | --exclude-workspace) COMPREPLY=(); return ;; completion) COMPREPLY=(\$(compgen -W "bash zsh" -- "\$cur")); return ;; esac for w in "\${COMP_WORDS[@]:1:COMP_CWORD-1}"; do @@ -1173,6 +1136,7 @@ _sg_migrate() { '--fresh[Ignore saved run state: redo every phase]' \\ '*--project[Only this TFC project segment]:segment' \\ '*--workspace[Only matching workspaces (glob)]:glob' \\ + '*--exclude-workspace[Leave matching workspaces out (glob)]:glob' \\ '--all[With clean: also remove config]' \\ '(-v --verbose)'{-v,--verbose}'[Show full terraform/tool output]' \\ '(-y --yes)'{-y,--yes}'[Skip the import confirmation prompt]' \\ @@ -1239,6 +1203,11 @@ main() { shift ;; --workspace=*) WS_FILTER+=("${1#*=}") ;; + --exclude-workspace) + WS_EXCLUDE+=("$2") + shift + ;; + --exclude-workspace=*) WS_EXCLUDE+=("${1#*=}") ;; -v | --verbose) VERBOSE=1 ;; --all) PURGE=1 ;; -h | --help) @@ -1305,7 +1274,7 @@ main() { PHASE_TOTAL=4 [ "$ENRICH_VARSETS" -eq 1 ] && PHASE_TOTAL=5 local apply_sha - apply_sha="$(sg_sha "$(sg_sha_files "$TFVARS")|$(ws_filter_json)")" + apply_sha="$(sg_sha "$(sg_sha_files "$TFVARS")|$(scope_sha_input)")" run_phase apply "$apply_sha" cmd_apply if [ "$ENRICH_VARSETS" -eq 1 ]; then run_phase enrich "$apply_sha" cmd_enrich; fi run_phase convert "$(payload_sha)" cmd_convert diff --git a/transformer/terraform-cloud/locals.tf b/transformer/terraform-cloud/locals.tf index ce07921..291e126 100644 --- a/transformer/terraform-cloud/locals.tf +++ b/transformer/terraform-cloud/locals.tf @@ -1,7 +1,18 @@ locals { - # data.tfe_workspace_ids.data.ids is a map of workspace name => workspace id. - workflowIds = [for name, id in data.tfe_workspace_ids.data.ids : id] - workflowNames = [for name, id in data.tfe_workspace_ids.data.ids : name] + # Workspace selection. data.tfe_workspace_ids applies workspacenames and the + # tag filters inside TFC (name => id); tfWorkspaceIgnoreNames has no provider + # equivalent and is applied here, so everything below iterates + # local.selectedWorkspaces rather than the data source. + ignoreNameRegexes = [for p in var.tfWorkspaceIgnoreNames : "^${replace(replace(replace(p, ".", "\\."), "*", ".*"), "?", ".")}$"] + excludedWorkspaces = sort([ + for name, id in data.tfe_workspace_ids.data.ids : name + if anytrue([for r in local.ignoreNameRegexes : can(regex(r, name))]) + ]) + selectedWorkspaces = { + for name, id in data.tfe_workspace_ids.data.ids : name => id if !contains(local.excludedWorkspaces, name) + } + workflowIds = [for name, id in local.selectedWorkspaces : id] + workflowNames = [for name, id in local.selectedWorkspaces : name] # project id => project name, used to name the per-project payload files and # to set a human-readable WorkflowGroup name in the payload. @@ -60,7 +71,7 @@ locals { # TFC-specific variables (TFC_*, TFE_* by default) are meaningless in SG and # are stripped; recorded per workspace so the summary can list them. strippedVars = { - for name, id in data.tfe_workspace_ids.data.ids : + for name, id in local.selectedWorkspaces : name => [for v in data.tfe_variables.data[id].variables : "${v.category}:${v.name}" if anytrue([for p in var.ignoreVarPatterns : can(regex(p, v.name))])] } @@ -69,7 +80,7 @@ locals { # (stripCloudAuthVars). Stripped whether sensitive or not: a sensitive one # must not become a placeholder secret that fights the connector either. strippedCloudAuthVars = { - for name, id in data.tfe_workspace_ids.data.ids : + for name, id in local.selectedWorkspaces : name => [for v in data.tfe_variables.data[id].variables : "${v.category}:${v.name}" if v.category == "env" && !anytrue([for p in var.ignoreVarPatterns : can(regex(p, v.name))]) @@ -80,7 +91,7 @@ locals { # migrated. Record them per workspace so the summary can flag them # (stripped variables excluded — nobody needs a secret stub for those). sensitiveVars = { - for name, id in data.tfe_workspace_ids.data.ids : + for name, id in local.selectedWorkspaces : name => [for v in data.tfe_variables.data[id].variables : "${v.category}:${v.name}" if v.sensitive && !anytrue([for p in var.ignoreVarPatterns : can(regex(p, v.name))]) @@ -142,7 +153,7 @@ locals { # win over the SGDefault* values; everything else is derived from the TFC # workspace. workflowPayload = { - for wsName, wsId in data.tfe_workspace_ids.data.ids : wsName => merge({ + for wsName, wsId in local.selectedWorkspaces : wsName => merge({ CLIConfiguration = { "WorkflowGroup" : { # SG workflow group per TFC project: projectOverrides[<project>].workflowGroup, @@ -284,8 +295,11 @@ locals { # Machine-readable migration summary (also rendered to markdown). summary = { - organization = var.tfOrg - workspaceCount = length(local.workflowNames) + organization = var.tfOrg + workspaceCount = length(local.workflowNames) + # Workspaces matched by the TFC filters but dropped by tfWorkspaceIgnoreNames. + excludedWorkspaces = local.excludedWorkspaces + tfWorkspaceIgnoreNames = var.tfWorkspaceIgnoreNames projectWorkspaceCounts = { for pid in local.projectsUsed : try(local.projectNames[pid], pid) => length(local.payloadByProject[pid]) } # Per project (raw TFC name): payload file segment, SG workflow group, size. projects = { diff --git a/transformer/terraform-cloud/resources.tf b/transformer/terraform-cloud/resources.tf index fd58be0..6dacc5c 100644 --- a/transformer/terraform-cloud/resources.tf +++ b/transformer/terraform-cloud/resources.tf @@ -23,7 +23,7 @@ resource "local_file" "summaryMd" { # `terraform login` credentials file or TFE_TOKEN, so it never enters TF state. # Requires `curl` and `jq` on PATH (provided by the Docker image / orchestrator). resource "null_resource" "exportState" { - for_each = var.exportStateFiles ? data.tfe_workspace_ids.data.ids : {} + for_each = var.exportStateFiles ? local.selectedWorkspaces : {} # Idempotent by default (keyed by stable workspace name/id); forceStateRefresh # re-pulls every workspace. diff --git a/transformer/terraform-cloud/summary.tmpl b/transformer/terraform-cloud/summary.tmpl index 687fb43..b738146 100644 --- a/transformer/terraform-cloud/summary.tmpl +++ b/transformer/terraform-cloud/summary.tmpl @@ -2,6 +2,9 @@ - Organization: ${summary.organization} - Workspaces processed: ${summary.workspaceCount} +%{ if length(summary.excludedWorkspaces) > 0 ~} +- Excluded by name (tfWorkspaceIgnoreNames ${jsonencode(summary.tfWorkspaceIgnoreNames)}): ${join(", ", summary.excludedWorkspaces)} +%{ endif ~} ## Workflows per project %{ for project, info in summary.projects ~} diff --git a/transformer/terraform-cloud/terraform.tfvars.example b/transformer/terraform-cloud/terraform.tfvars.example index 2e01d31..e8870e6 100644 --- a/transformer/terraform-cloud/terraform.tfvars.example +++ b/transformer/terraform-cloud/terraform.tfvars.example @@ -13,6 +13,10 @@ tfWorkspaceTags = null # Specify a list of tags in workspace tags to exclude, or leave empty to include all, for example: [exclude"] tfWorkspaceIgnoreTags = null +# Workspace names to leave out (globs, e.g. ["sandbox-*", "*-scratch"]), applied +# after the filters above. sg-migrate.sh --exclude-workspace adds to this list for one run. +tfWorkspaceIgnoreNames = [] + # Directory to export Terraform files to exportPath = "export" diff --git a/transformer/terraform-cloud/variables.tf b/transformer/terraform-cloud/variables.tf index 3585ee9..5744685 100644 --- a/transformer/terraform-cloud/variables.tf +++ b/transformer/terraform-cloud/variables.tf @@ -33,6 +33,12 @@ variable "tfWorkspaceIgnoreTags" { type = list(string) } +variable "tfWorkspaceIgnoreNames" { + default = [] + description = "Workspace names (globs, * and ? supported) to leave out of the export, applied after workspacenames and the tag filters. Excluded workspaces are listed in migration-summary.md. The orchestrator's --exclude-workspace adds to this list for one run." + type = list(string) +} + variable "exportPath" { default = "export" description = "name of the folder to export the payload, state files to. ./export is the default" From e39bd863bafdc9094b8beeaee67307f77c05e8a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adis=20Halilovi=C4=87?= <adis.halilovic@stackguardian.io> Date: Mon, 14 Sep 2026 16:02:31 +0200 Subject: [PATCH 48/71] feat: --project selects at export time, by TFC name or slug Until now --project only picked payload files after a full export and had to be the file segment. The module gets tfProjects (names or slugs, [] = all), applied after the name filters: tfe_workspace is read for the name-filtered set because the project of a workspace is only known from there. The orchestrator passes --project as tfProjects, matches payload files by slug so 'My First Project' and my-first-project are the same, and preflight refuses a --project that names no project (a tfProjects typo is a warning) and previews the count the apply will export. --- scripts/lib/preflight.sh | 29 +++++++++++++++-- scripts/lib/scope.sh | 31 ++++++++++++++----- scripts/migrate.sh | 8 +++-- transformer/terraform-cloud/data.tf | 4 ++- transformer/terraform-cloud/locals.tf | 29 ++++++++++++----- transformer/terraform-cloud/summary.tmpl | 6 ++++ .../terraform-cloud/terraform.tfvars.example | 4 +++ transformer/terraform-cloud/variables.tf | 6 ++++ 8 files changed, 96 insertions(+), 21 deletions(-) diff --git a/scripts/lib/preflight.sh b/scripts/lib/preflight.sh index 4669bef..a1171fc 100644 --- a/scripts/lib/preflight.sh +++ b/scripts/lib/preflight.sh @@ -49,23 +49,46 @@ preflight_tfc() { # tags, exclude names) with the CLI scope applied (lib/scope.sh, when loaded), # so the count is the one the apply that follows will export. if body="$(tfc_list_workspaces "$org" 2>/dev/null)"; then - local names ignore_names + local names ignore_names projects from_cli=0 p slug hit + PF_TFC_PROJECTS="$(tfc_list_projects "$org" 2>/dev/null || echo '[]')" names="$(tfvars_get_json .workspacenames)" ignore_names="$(tfvars_get_json .tfWorkspaceIgnoreNames)" + projects="$(tfvars_get_json .tfProjects)" + [ "$projects" = "null" ] && projects='[]' if declare -F ws_exclude_json >/dev/null; then [ "${#WS_FILTER[@]}" -gt 0 ] && names="$(names_json "${WS_FILTER[@]}")" + [ "${#PROJECT_FILTER[@]}" -gt 0 ] && { projects="$(names_json "${PROJECT_FILTER[@]}")"; from_cli=1; } ignore_names="$(ws_exclude_json)" fi sel="$(tfc_select_workspaces "$body" "$names" "$(tfvars_get_json .tfWorkspaceTags)" "$(tfvars_get_json .tfWorkspaceIgnoreTags)" "$ignore_names")" + # Project selection (tfProjects / --project): names or slugs; a selector + # that names no project is a typo — fatal from the CLI, a warning in tfvars. + if [ "$projects" != "[]" ] && [ "$PF_TFC_PROJECTS" != "[]" ]; then + while IFS= read -r p; do + [ -n "$p" ] || continue + slug="$(printf '%s' "$p" | tr '[:upper:]' '[:lower:]' | sed -E 's/[^a-z0-9-]+/-/g')" + hit="$(printf '%s' "$PF_TFC_PROJECTS" | "$jqb" -r --arg s "$slug" '[.[] | select((.name | ascii_downcase | gsub("[^a-z0-9-]+"; "-")) == $s) | .name] | first // empty')" + if [ -n "$hit" ]; then + pf_ok "project '$p' is TFC project '$hit'" + elif [ "$from_cli" -eq 1 ]; then + pf_fail "--project '$p' matches no TFC project in '$org' (projects: $(printf '%s' "$PF_TFC_PROJECTS" | "$jqb" -r '[.[].name] | join(", ")'))" + else + pf_warn "tfProjects entry '$p' matches no TFC project in '$org' (projects: $(printf '%s' "$PF_TFC_PROJECTS" | "$jqb" -r '[.[].name] | join(", ")'))" + fi + done < <(printf '%s' "$projects" | "$jqb" -r '.[]') + sel="$(printf '%s' "$sel" | "$jqb" -c --argjson pr "$PF_TFC_PROJECTS" --argjson want "$projects" ' + ($want | map(ascii_downcase | gsub("[^a-z0-9-]+"; "-"))) as $w + | ($pr | map(select((.name | ascii_downcase | gsub("[^a-z0-9-]+"; "-")) as $s | $w | index($s) != null) | .id)) as $ids + | map(select(.project as $p | $ids | index($p) != null))')" + fi n="$(printf '%s' "$sel" | "$jqb" 'length')" if [ "$n" -gt 0 ]; then pf_ok "$n workspace(s) match the selection (of $(printf '%s' "$body" | "$jqb" 'length') in the org)" else - pf_warn "no workspace matches workspacenames/tfWorkspaceTags/tfWorkspaceIgnoreTags/tfWorkspaceIgnoreNames${WS_FILTER[*]:+ and the --workspace/--exclude-workspace flags} — apply would export nothing" + pf_warn "no workspace matches workspacenames/tfWorkspaceTags/tfWorkspaceIgnoreTags/tfWorkspaceIgnoreNames/tfProjects${WS_FILTER[*]:+ and the --workspace/--exclude-workspace/--project flags} — apply would export nothing" fi PF_TFC_WORKSPACES="$body" PF_TFC_SELECTED="$sel" - PF_TFC_PROJECTS="$(tfc_list_projects "$org" 2>/dev/null || echo '[]')" else pf_warn "could not list workspaces for '$org' (HTTP $TFC_HTTP_CODE)" fi diff --git a/scripts/lib/scope.sh b/scripts/lib/scope.sh index 4e7173a..f5cc515 100644 --- a/scripts/lib/scope.sh +++ b/scripts/lib/scope.sh @@ -2,23 +2,38 @@ # Run scope for the migrator (sourced; needs tools.sh, lib/tfvars.sh, jq). # # terraform.tfvars holds the widest scope (workspacenames, tfWorkspaceIgnoreNames, -# the tag filters) and the CLI narrows one run: +# tfProjects, the tag filters) and the CLI narrows one run: +# --project NAME|SLUG replaces tfProjects for the run # --workspace GLOB replaces workspacenames for the run ("*" = all) # --exclude-workspace GLOB adds to tfWorkspaceIgnoreNames for the run # Every phase applies the same selection: apply hands the lists to terraform, -# the later phases match them against the payload entries (ResourceName), so -# 'import --workspace "team-*"' picks the workflows 'apply --workspace "team-*"' -# exported. Globs support * and ?. +# the later phases match them against the payload files (project slug) and +# entries (ResourceName), so 'import --workspace "team-*"' picks the workflows +# 'apply --workspace "team-*"' exported. Globs support * and ?. PROJECT_FILTER=() WS_FILTER=() WS_EXCLUDE=() # names_json <name>... — the arguments as a JSON array of strings. -names_json() { printf '%s\n' "$@" | "$JQ_BIN" -R . | "$JQ_BIN" -s .; } +names_json() { _scope_jq; printf '%s\n' "$@" | "$JQ_BIN" -R . | "$JQ_BIN" -s .; } _scope_jq() { JQ_BIN="${JQ_BIN:-$(sg_resolve jq sg_ensure_jq)}"; } +# slug_of <project name> — the payload file segment of a TFC project: the same +# rule as the transformer's projectSlugs (lowercase, runs of anything but +# [a-z0-9-] become "-"), so --project accepts the name or the slug. +slug_of() { printf '%s' "$1" | tr '[:upper:]' '[:lower:]' | sed -E 's/[^a-z0-9-]+/-/g'; } + +# project_selected <segment> — exit 0 when no --project is set or one of them +# names this payload segment. +project_selected() { + local p + [ "${#PROJECT_FILTER[@]}" -eq 0 ] && return 0 + for p in "${PROJECT_FILTER[@]}"; do [ "$(slug_of "$p")" = "$1" ] && return 0; done + return 1 +} + # ws_narrowed — exit 0 when a CLI flag restricts the run ("*" alone does not, # so a CI run over everything keeps the unchanged-file skip). ws_narrowed() { @@ -95,6 +110,7 @@ SCOPE_TFVAR_ARGS=() scope_tfvar_args() { SCOPE_TFVAR_ARGS=() _scope_jq + [ "${#PROJECT_FILTER[@]}" -gt 0 ] && SCOPE_TFVAR_ARGS+=(-var "tfProjects=$(names_json "${PROJECT_FILTER[@]}")") [ "${#WS_FILTER[@]}" -gt 0 ] && SCOPE_TFVAR_ARGS+=(-var "workspacenames=$(names_json "${WS_FILTER[@]}")") [ "${#WS_EXCLUDE[@]}" -gt 0 ] && SCOPE_TFVAR_ARGS+=(-var "tfWorkspaceIgnoreNames=$(ws_exclude_json)") return 0 @@ -103,11 +119,12 @@ scope_tfvar_args() { # scope_describe — one line for the log: what the CLI flags select. scope_describe() { local out="" - [ "${#WS_FILTER[@]}" -gt 0 ] && out="workspaces ${WS_FILTER[*]}" + [ "${#PROJECT_FILTER[@]}" -gt 0 ] && out="project(s) ${PROJECT_FILTER[*]}" + [ "${#WS_FILTER[@]}" -gt 0 ] && out="${out:+$out, }workspaces ${WS_FILTER[*]}" [ "${#WS_EXCLUDE[@]}" -gt 0 ] && out="${out:+$out, }excluding ${WS_EXCLUDE[*]}" printf '%s' "$out" } # scope_sha_input — the CLI scope as a string for the apply phase hash, so a # run with a different selection re-runs the export. -scope_sha_input() { printf '%s|%s' "$(ws_filter_json)" "$(ws_exclude_json)"; } +scope_sha_input() { printf '%s|%s|%s' "${PROJECT_FILTER[*]-}" "$(ws_filter_json)" "$(ws_exclude_json)"; } diff --git a/scripts/migrate.sh b/scripts/migrate.sh index f21a762..6b9ce06 100755 --- a/scripts/migrate.sh +++ b/scripts/migrate.sh @@ -120,7 +120,8 @@ Options: --dry-run With 'import': show the per-workflow plan and stop (nothing is created) --no-secret-stubs Do not create placeholder SG secrets for sensitive variables --fresh Ignore the saved run state: redo every phase and re-import everything - --project SEG Only handle this TFC project (repeatable; matches sg-payload.<SEG>.json) + --project NAME Only handle this TFC project, by name or slug (repeatable; apply exports + only its workspaces, later phases use sg-payload.<slug>.json) --workspace GLOB Only handle matching workspaces (repeatable; "team-*", "*" = all; replaces workspacenames for this run, every phase selects the same ones) --exclude-workspace GLOB @@ -256,11 +257,12 @@ payload_files() { shopt -s nullglob PF=("$EXPORT_DIR"/sg-payload.*.json) shopt -u nullglob + # --project accepts the TFC name or the payload segment (lib/scope.sh). if [ "${#PROJECT_FILTER[@]}" -gt 0 ]; then keep=() for f in "${PF[@]}"; do seg="$(seg_of "$f")" - case " ${PROJECT_FILTER[*]} " in *" $seg "*) keep+=("$f") ;; esac + project_selected "$seg" && keep+=("$f") done PF=(${keep[@]+"${keep[@]}"}) fi @@ -1134,7 +1136,7 @@ _sg_migrate() { '--dry-run[With import: show the plan and stop]' \\ '--no-secret-stubs[Do not create placeholder SG secrets for sensitive vars]' \\ '--fresh[Ignore saved run state: redo every phase]' \\ - '*--project[Only this TFC project segment]:segment' \\ + '*--project[Only this TFC project (name or slug)]:project' \\ '*--workspace[Only matching workspaces (glob)]:glob' \\ '*--exclude-workspace[Leave matching workspaces out (glob)]:glob' \\ '--all[With clean: also remove config]' \\ diff --git a/transformer/terraform-cloud/data.tf b/transformer/terraform-cloud/data.tf index 5925934..4c195da 100644 --- a/transformer/terraform-cloud/data.tf +++ b/transformer/terraform-cloud/data.tf @@ -5,8 +5,10 @@ data "tfe_workspace_ids" "data" { exclude_tags = var.tfWorkspaceIgnoreTags } +# Read for every workspace that passed the name filters: the project filter +# (tfProjects) needs the project_id from here, see local.selectedWorkspaces. data "tfe_workspace" "data" { - for_each = toset(local.workflowNames) + for_each = local.namedWorkspaces name = each.key organization = var.tfOrg diff --git a/transformer/terraform-cloud/locals.tf b/transformer/terraform-cloud/locals.tf index 291e126..fe6f660 100644 --- a/transformer/terraform-cloud/locals.tf +++ b/transformer/terraform-cloud/locals.tf @@ -1,22 +1,34 @@ locals { # Workspace selection. data.tfe_workspace_ids applies workspacenames and the - # tag filters inside TFC (name => id); tfWorkspaceIgnoreNames has no provider - # equivalent and is applied here, so everything below iterates - # local.selectedWorkspaces rather than the data source. + # tag filters inside TFC (name => id). tfWorkspaceIgnoreNames and tfProjects + # have no provider equivalent and are applied here, so everything below + # iterates local.selectedWorkspaces rather than the data source. ignoreNameRegexes = [for p in var.tfWorkspaceIgnoreNames : "^${replace(replace(replace(p, ".", "\\."), "*", ".*"), "?", ".")}$"] excludedWorkspaces = sort([ for name, id in data.tfe_workspace_ids.data.ids : name if anytrue([for r in local.ignoreNameRegexes : can(regex(r, name))]) ]) - selectedWorkspaces = { + # Name filters applied; data.tfe_workspace is read for these, because the + # project of a workspace is only known from there. + namedWorkspaces = { for name, id in data.tfe_workspace_ids.data.ids : name => id if !contains(local.excludedWorkspaces, name) } + # tfProjects entries are TFC project names or their slug (the payload file + # segment); [] = every project. + projectSelectors = [for p in var.tfProjects : replace(lower(p), "/[^a-z0-9-]+/", "-")] + unknownProjects = sort([for p in var.tfProjects : p if !contains(values(local.projectSlugs), replace(lower(p), "/[^a-z0-9-]+/", "-"))]) + selectedWorkspaces = { + for name, id in local.namedWorkspaces : name => id + if length(var.tfProjects) == 0 || contains(local.projectSelectors, try(local.projectSlugs[data.tfe_workspace.data[name].project_id], data.tfe_workspace.data[name].project_id)) + } workflowIds = [for name, id in local.selectedWorkspaces : id] workflowNames = [for name, id in local.selectedWorkspaces : name] # project id => project name, used to name the per-project payload files and - # to set a human-readable WorkflowGroup name in the payload. + # to set a human-readable WorkflowGroup name in the payload; and its slug, + # the filesystem-safe segment of the payload filename (sg-payload.<slug>.json). projectNames = { for p in data.tfe_projects.data.projects : p.id => p.name } + projectSlugs = { for pid, name in local.projectNames : pid => replace(lower(name), "/[^a-z0-9-]+/", "-") } # TFC project per workspace: id, and the raw name (falls back to the id when # the project is not visible to the token). @@ -282,7 +294,7 @@ locals { # project id => filesystem-safe segment for the per-project payload filename. projectFileSegment = { for pid in local.projectsUsed : - pid => replace(lower(try(local.projectNames[pid], pid)), "/[^a-z0-9-]+/", "-") + pid => try(local.projectSlugs[pid], replace(lower(pid), "/[^a-z0-9-]+/", "-")) } # SG workflow group per project: projectOverrides[<name>].workflowGroup, else @@ -297,9 +309,12 @@ locals { summary = { organization = var.tfOrg workspaceCount = length(local.workflowNames) - # Workspaces matched by the TFC filters but dropped by tfWorkspaceIgnoreNames. + # Workspaces matched by the TFC filters but dropped by tfWorkspaceIgnoreNames, + # and the project selection ([] = all; unknownProjects = selectors matching none). excludedWorkspaces = local.excludedWorkspaces tfWorkspaceIgnoreNames = var.tfWorkspaceIgnoreNames + tfProjects = var.tfProjects + unknownProjects = local.unknownProjects projectWorkspaceCounts = { for pid in local.projectsUsed : try(local.projectNames[pid], pid) => length(local.payloadByProject[pid]) } # Per project (raw TFC name): payload file segment, SG workflow group, size. projects = { diff --git a/transformer/terraform-cloud/summary.tmpl b/transformer/terraform-cloud/summary.tmpl index b738146..a4018c6 100644 --- a/transformer/terraform-cloud/summary.tmpl +++ b/transformer/terraform-cloud/summary.tmpl @@ -5,6 +5,12 @@ %{ if length(summary.excludedWorkspaces) > 0 ~} - Excluded by name (tfWorkspaceIgnoreNames ${jsonencode(summary.tfWorkspaceIgnoreNames)}): ${join(", ", summary.excludedWorkspaces)} %{ endif ~} +%{ if length(summary.tfProjects) > 0 ~} +- Limited to projects (tfProjects): ${join(", ", summary.tfProjects)} +%{ endif ~} +%{ if length(summary.unknownProjects) > 0 ~} +- WARNING: tfProjects entries matching no TFC project: ${join(", ", summary.unknownProjects)} +%{ endif ~} ## Workflows per project %{ for project, info in summary.projects ~} diff --git a/transformer/terraform-cloud/terraform.tfvars.example b/transformer/terraform-cloud/terraform.tfvars.example index e8870e6..e345830 100644 --- a/transformer/terraform-cloud/terraform.tfvars.example +++ b/transformer/terraform-cloud/terraform.tfvars.example @@ -17,6 +17,10 @@ tfWorkspaceIgnoreTags = null # after the filters above. sg-migrate.sh --exclude-workspace adds to this list for one run. tfWorkspaceIgnoreNames = [] +# Only these TFC projects, by name or slug (e.g. ["Platform", "my-first-project"]); +# [] = every project. sg-migrate.sh --project sets this for one run. +tfProjects = [] + # Directory to export Terraform files to exportPath = "export" diff --git a/transformer/terraform-cloud/variables.tf b/transformer/terraform-cloud/variables.tf index 5744685..3f6a05f 100644 --- a/transformer/terraform-cloud/variables.tf +++ b/transformer/terraform-cloud/variables.tf @@ -33,6 +33,12 @@ variable "tfWorkspaceIgnoreTags" { type = list(string) } +variable "tfProjects" { + default = [] + description = "TFC/TFE projects to export, by name (as shown in TFC) or by slug (the payload file segment, e.g. my-first-project); [] = every project. Applied after the workspace filters. Entries matching no project are listed in migration-summary.md (unknownProjects). The orchestrator's --project sets this for one run." + type = list(string) +} + variable "tfWorkspaceIgnoreNames" { default = [] description = "Workspace names (globs, * and ? supported) to leave out of the export, applied after workspacenames and the tag filters. Excluded workspaces are listed in migration-summary.md. The orchestrator's --exclude-workspace adds to this list for one run." From 5055406b7ed2e775b1c19d8e62e5b58a496f903f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adis=20Halilovi=C4=87?= <adis.halilovic@stackguardian.io> Date: Mon, 14 Sep 2026 16:04:46 +0200 Subject: [PATCH 49/71] feat: --tag and --exclude-tag select workspaces by TFC tag for one run --tag replaces tfWorkspaceTags, --exclude-tag adds to tfWorkspaceIgnoreTags, both are handed to terraform apply and previewed by preflight. Tags are not in the payload, so they shape the export only. --- scripts/lib/preflight.sh | 12 +++++++++--- scripts/lib/scope.sh | 33 ++++++++++++++++++++++++++++++--- scripts/migrate.sh | 19 +++++++++++++++++-- 3 files changed, 56 insertions(+), 8 deletions(-) diff --git a/scripts/lib/preflight.sh b/scripts/lib/preflight.sh index a1171fc..ae70a2d 100644 --- a/scripts/lib/preflight.sh +++ b/scripts/lib/preflight.sh @@ -49,18 +49,22 @@ preflight_tfc() { # tags, exclude names) with the CLI scope applied (lib/scope.sh, when loaded), # so the count is the one the apply that follows will export. if body="$(tfc_list_workspaces "$org" 2>/dev/null)"; then - local names ignore_names projects from_cli=0 p slug hit + local names ignore_names tags ignore_tags projects from_cli=0 p slug hit PF_TFC_PROJECTS="$(tfc_list_projects "$org" 2>/dev/null || echo '[]')" names="$(tfvars_get_json .workspacenames)" ignore_names="$(tfvars_get_json .tfWorkspaceIgnoreNames)" + tags="$(tfvars_get_json .tfWorkspaceTags)" + ignore_tags="$(tfvars_get_json .tfWorkspaceIgnoreTags)" projects="$(tfvars_get_json .tfProjects)" [ "$projects" = "null" ] && projects='[]' if declare -F ws_exclude_json >/dev/null; then [ "${#WS_FILTER[@]}" -gt 0 ] && names="$(names_json "${WS_FILTER[@]}")" [ "${#PROJECT_FILTER[@]}" -gt 0 ] && { projects="$(names_json "${PROJECT_FILTER[@]}")"; from_cli=1; } ignore_names="$(ws_exclude_json)" + tags="$(scope_tags_json)" + ignore_tags="$(scope_ignore_tags_json)" fi - sel="$(tfc_select_workspaces "$body" "$names" "$(tfvars_get_json .tfWorkspaceTags)" "$(tfvars_get_json .tfWorkspaceIgnoreTags)" "$ignore_names")" + sel="$(tfc_select_workspaces "$body" "$names" "$tags" "$ignore_tags" "$ignore_names")" # Project selection (tfProjects / --project): names or slugs; a selector # that names no project is a typo — fatal from the CLI, a warning in tfvars. if [ "$projects" != "[]" ] && [ "$PF_TFC_PROJECTS" != "[]" ]; then @@ -85,7 +89,9 @@ preflight_tfc() { if [ "$n" -gt 0 ]; then pf_ok "$n workspace(s) match the selection (of $(printf '%s' "$body" | "$jqb" 'length') in the org)" else - pf_warn "no workspace matches workspacenames/tfWorkspaceTags/tfWorkspaceIgnoreTags/tfWorkspaceIgnoreNames/tfProjects${WS_FILTER[*]:+ and the --workspace/--exclude-workspace/--project flags} — apply would export nothing" + local cli="" + declare -F scope_describe >/dev/null && cli="$(scope_describe)" + pf_warn "no workspace matches workspacenames/tfWorkspaceTags/tfWorkspaceIgnoreTags/tfWorkspaceIgnoreNames/tfProjects${cli:+ with the run scope ($cli)} — apply would export nothing" fi PF_TFC_WORKSPACES="$body" PF_TFC_SELECTED="$sel" diff --git a/scripts/lib/scope.sh b/scripts/lib/scope.sh index f5cc515..cc96072 100644 --- a/scripts/lib/scope.sh +++ b/scripts/lib/scope.sh @@ -6,17 +6,40 @@ # --project NAME|SLUG replaces tfProjects for the run # --workspace GLOB replaces workspacenames for the run ("*" = all) # --exclude-workspace GLOB adds to tfWorkspaceIgnoreNames for the run +# --tag NAME replaces tfWorkspaceTags for the run +# --exclude-tag NAME adds to tfWorkspaceIgnoreTags for the run # Every phase applies the same selection: apply hands the lists to terraform, # the later phases match them against the payload files (project slug) and # entries (ResourceName), so 'import --workspace "team-*"' picks the workflows -# 'apply --workspace "team-*"' exported. Globs support * and ?. +# 'apply --workspace "team-*"' exported. Tags are not in the payload, so the +# tag flags only shape the export. Globs support * and ?. PROJECT_FILTER=() WS_FILTER=() WS_EXCLUDE=() +TAG_FILTER=() +TAG_EXCLUDE=() + +# scope_tags_json / scope_ignore_tags_json — the effective tag filters: +# --tag replaces tfWorkspaceTags, --exclude-tag adds to tfWorkspaceIgnoreTags +# (null = no filter, as in the module). +scope_tags_json() { + local tf + if [ "${#TAG_FILTER[@]}" -gt 0 ]; then names_json "${TAG_FILTER[@]}"; return; fi + tf="$(tfvars_get_json .tfWorkspaceTags)" + printf '%s' "${tf:-null}" +} +scope_ignore_tags_json() { + local tf + _scope_jq + tf="$(tfvars_get_json .tfWorkspaceIgnoreTags)" + [ "$tf" = "null" ] || [ -z "$tf" ] && tf='[]' + if [ "${#TAG_EXCLUDE[@]}" -eq 0 ]; then printf '%s' "$tf"; return; fi + "$JQ_BIN" -nc --argjson a "$tf" --argjson b "$(names_json "${TAG_EXCLUDE[@]}")" '$a + $b | unique' +} # names_json <name>... — the arguments as a JSON array of strings. -names_json() { _scope_jq; printf '%s\n' "$@" | "$JQ_BIN" -R . | "$JQ_BIN" -s .; } +names_json() { _scope_jq; printf '%s\n' "$@" | "$JQ_BIN" -R . | "$JQ_BIN" -sc .; } _scope_jq() { JQ_BIN="${JQ_BIN:-$(sg_resolve jq sg_ensure_jq)}"; } @@ -113,6 +136,8 @@ scope_tfvar_args() { [ "${#PROJECT_FILTER[@]}" -gt 0 ] && SCOPE_TFVAR_ARGS+=(-var "tfProjects=$(names_json "${PROJECT_FILTER[@]}")") [ "${#WS_FILTER[@]}" -gt 0 ] && SCOPE_TFVAR_ARGS+=(-var "workspacenames=$(names_json "${WS_FILTER[@]}")") [ "${#WS_EXCLUDE[@]}" -gt 0 ] && SCOPE_TFVAR_ARGS+=(-var "tfWorkspaceIgnoreNames=$(ws_exclude_json)") + [ "${#TAG_FILTER[@]}" -gt 0 ] && SCOPE_TFVAR_ARGS+=(-var "tfWorkspaceTags=$(scope_tags_json)") + [ "${#TAG_EXCLUDE[@]}" -gt 0 ] && SCOPE_TFVAR_ARGS+=(-var "tfWorkspaceIgnoreTags=$(scope_ignore_tags_json)") return 0 } @@ -122,9 +147,11 @@ scope_describe() { [ "${#PROJECT_FILTER[@]}" -gt 0 ] && out="project(s) ${PROJECT_FILTER[*]}" [ "${#WS_FILTER[@]}" -gt 0 ] && out="${out:+$out, }workspaces ${WS_FILTER[*]}" [ "${#WS_EXCLUDE[@]}" -gt 0 ] && out="${out:+$out, }excluding ${WS_EXCLUDE[*]}" + [ "${#TAG_FILTER[@]}" -gt 0 ] && out="${out:+$out, }tagged ${TAG_FILTER[*]}" + [ "${#TAG_EXCLUDE[@]}" -gt 0 ] && out="${out:+$out, }not tagged ${TAG_EXCLUDE[*]}" printf '%s' "$out" } # scope_sha_input — the CLI scope as a string for the apply phase hash, so a # run with a different selection re-runs the export. -scope_sha_input() { printf '%s|%s|%s' "${PROJECT_FILTER[*]-}" "$(ws_filter_json)" "$(ws_exclude_json)"; } +scope_sha_input() { printf '%s|%s|%s|%s|%s' "${PROJECT_FILTER[*]-}" "$(ws_filter_json)" "$(ws_exclude_json)" "$(scope_tags_json)" "$(scope_ignore_tags_json)"; } diff --git a/scripts/migrate.sh b/scripts/migrate.sh index 6b9ce06..600cc14 100755 --- a/scripts/migrate.sh +++ b/scripts/migrate.sh @@ -126,6 +126,9 @@ Options: replaces workspacenames for this run, every phase selects the same ones) --exclude-workspace GLOB Leave matching workspaces out (repeatable; adds to tfWorkspaceIgnoreNames) + --tag NAME Only workspaces carrying every given tag (repeatable; replaces + tfWorkspaceTags for the export) + --exclude-tag NAME Leave workspaces carrying the tag out (repeatable; adds to tfWorkspaceIgnoreTags) --all With 'clean': also remove config (terraform.tfvars, mapping, .sg) -v, --verbose Show full terraform/tool output (default: concise) -y, --yes Skip the import confirmation prompt @@ -1060,7 +1063,7 @@ finish_line() { # Single source of truth for shell completion (keep in sync with the parser below # and the host-only flags in sg-migrate.sh). SG_COMMANDS="init preflight apply enrich convert validate import triggers checklist all clean completion update" -SG_OPTIONS="--org --export-dir --mapping --concurrency --no-create-groups --no-variable-sets --no-vcs-triggers --skip-preflight --dry-run --no-secret-stubs --fresh --project --workspace --exclude-workspace --all -v --verbose -y --yes -h --help --native --local --build" +SG_OPTIONS="--org --export-dir --mapping --concurrency --no-create-groups --no-variable-sets --no-vcs-triggers --skip-preflight --dry-run --no-secret-stubs --fresh --project --workspace --exclude-workspace --tag --exclude-tag --all -v --verbose -y --yes -h --help --native --local --build" # cmd_completion <bash|zsh> — print a completion script for sg-migrate.sh / # migrate.sh to stdout. Both shells fall back to the basename when the command @@ -1084,7 +1087,7 @@ _sg_migrate() { case "\$prev" in --export-dir) COMPREPLY=(\$(compgen -d -- "\$cur")); return ;; --mapping) COMPREPLY=(\$(compgen -f -- "\$cur")); return ;; - --org | --concurrency | --project | --workspace | --exclude-workspace) COMPREPLY=(); return ;; + --org | --concurrency | --project | --workspace | --exclude-workspace | --tag | --exclude-tag) COMPREPLY=(); return ;; completion) COMPREPLY=(\$(compgen -W "bash zsh" -- "\$cur")); return ;; esac for w in "\${COMP_WORDS[@]:1:COMP_CWORD-1}"; do @@ -1139,6 +1142,8 @@ _sg_migrate() { '*--project[Only this TFC project (name or slug)]:project' \\ '*--workspace[Only matching workspaces (glob)]:glob' \\ '*--exclude-workspace[Leave matching workspaces out (glob)]:glob' \\ + '*--tag[Only workspaces carrying this tag]:tag' \\ + '*--exclude-tag[Leave workspaces carrying this tag out]:tag' \\ '--all[With clean: also remove config]' \\ '(-v --verbose)'{-v,--verbose}'[Show full terraform/tool output]' \\ '(-y --yes)'{-y,--yes}'[Skip the import confirmation prompt]' \\ @@ -1210,6 +1215,16 @@ main() { shift ;; --exclude-workspace=*) WS_EXCLUDE+=("${1#*=}") ;; + --tag) + TAG_FILTER+=("$2") + shift + ;; + --tag=*) TAG_FILTER+=("${1#*=}") ;; + --exclude-tag) + TAG_EXCLUDE+=("$2") + shift + ;; + --exclude-tag=*) TAG_EXCLUDE+=("${1#*=}") ;; -v | --verbose) VERBOSE=1 ;; --all) PURGE=1 ;; -h | --help) From 4cb90dd6c696d74a8ff30054cc7b54259a11b6c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adis=20Halilovi=C4=87?= <adis.halilovic@stackguardian.io> Date: Mon, 14 Sep 2026 16:08:06 +0200 Subject: [PATCH 50/71] fix: --tfvars / SG_TFVARS is the file terraform, enrich and clean use The orchestrator read SG_TFVARS but terraform apply still loaded terraform.tfvars from the module dir, the enrich script fell back to the default path and the Docker wrapper did not forward the variable. The path is now resolved once (absolute), passed to -var-file and to enrich, and the wrapper translates a host path into the container: under /app when the file is in the checkout, mounted read-only otherwise. clean --all only removes the module's own file, and apply warns when both files exist because terraform auto-loads terraform.tfvars on top of -var-file. --- scripts/migrate.sh | 34 ++++++++++++++++++++++++++++------ sg-migrate.sh | 23 +++++++++++++++++++++++ 2 files changed, 51 insertions(+), 6 deletions(-) diff --git a/scripts/migrate.sh b/scripts/migrate.sh index 600cc14..42b9cdb 100755 --- a/scripts/migrate.sh +++ b/scripts/migrate.sh @@ -36,7 +36,15 @@ source "$SCRIPT_DIR/lib/checklist.sh" # SCRIPT_DIR holds the sibling scripts; SG_REPO_ROOT (from tools.sh) is the repo # root used for all repo-relative paths. TRANSFORMER_DIR="$SG_REPO_ROOT/transformer/terraform-cloud" -TFVARS="${SG_TFVARS:-$TRANSFORMER_DIR/terraform.tfvars}" +# The tfvars file: terraform.tfvars in the module dir unless --tfvars / SG_TFVARS +# points elsewhere (CI keeps its copy outside the checkout). abs_path keeps it +# valid after the cd into the module dir that terraform needs. +abs_path() { + case "$1" in /*) printf '%s' "$1" ;; *) printf '%s/%s' "$(cd "$(dirname "$1")" 2>/dev/null && pwd || dirname "$1")" "$(basename "$1")" ;; esac +} +TFVARS_DEFAULT="$TRANSFORMER_DIR/terraform.tfvars" +TFVARS="$TFVARS_DEFAULT" +[ -n "${SG_TFVARS:-}" ] && TFVARS="$(abs_path "$SG_TFVARS")" PROG="${SG_PROG:-$0}" EXPORT_DIR="${SG_EXPORT_DIR:-$SG_REPO_ROOT/export}" MAPPING="${SG_WFGROUP_MAP:-$SG_REPO_ROOT/.sg/workflow-groups.json}" @@ -110,6 +118,8 @@ in another group the plan stops and says so. Options: --org NAME StackGuardian org for import (or set SG_ORG) --export-dir DIR Payload/state output dir (default: ./export) + --tfvars FILE Use this tfvars file instead of transformer/terraform-cloud/terraform.tfvars + (or set SG_TFVARS); the file the transformer, enrich and preflight read --mapping FILE Deprecated: project-segment -> group override map (default: .sg/workflow-groups.json); use projectOverrides.<project>.workflowGroup instead --concurrency N Max parallel jobs for convert/import (default: 4) @@ -139,6 +149,7 @@ Environment: SG_ORG StackGuardian org (alternative to --org) SG_RETRIES Import retry attempts on failure (default: 4) SG_TF_PARALLELISM terraform apply -parallelism (default: 20) + SG_TFVARS tfvars file to use (same as --tfvars) SG_UI_URL StackGuardian UI base for checklist links (default: https://app.stackguardian.io) EOF } @@ -548,7 +559,8 @@ cmd_clean() { rm -rf "$SG_CACHE_DIR" state_reset if [ "$PURGE" -eq 1 ]; then - rm -f "$TFVARS" "$MAPPING" + # Only the module's own file; a --tfvars / SG_TFVARS file belongs to the user. + rm -f "$TFVARS_DEFAULT" "$MAPPING" rm -rf "$SG_REPO_ROOT/.sg" sg_log "also removed config (terraform.tfvars, workflow-groups.json, .sg)" fi @@ -560,12 +572,16 @@ cmd_clean() { # them; the verbose path wraps them in parentheses). # shellcheck disable=SC2120 # extra terraform flags (-no-color) come from the quiet path tf_init() { cd "$TRANSFORMER_DIR" && export PATH="$TF_PATH" TF_IN_AUTOMATION=1 && terraform init -input=false "$@"; } -tf_apply() { cd "$TRANSFORMER_DIR" && export PATH="$TF_PATH" TF_IN_AUTOMATION=1 && terraform apply -auto-approve -compact-warnings -parallelism="$TF_PARALLELISM" -var-file=terraform.tfvars "$@"; } +tf_apply() { cd "$TRANSFORMER_DIR" && export PATH="$TF_PATH" TF_IN_AUTOMATION=1 && terraform apply -auto-approve -compact-warnings -parallelism="$TF_PARALLELISM" -var-file="$TFVARS" "$@"; } cmd_apply() { phase_begin "apply (terraform)" command -v terraform >/dev/null 2>&1 || die "terraform not found on PATH" [ -f "$TFVARS" ] || die "Missing $(sg_rel "$TFVARS"). Run: $PROG init" + # terraform auto-loads terraform.tfvars from the module dir on top of -var-file. + if [ "$TFVARS" != "$TFVARS_DEFAULT" ] && [ -f "$TFVARS_DEFAULT" ]; then + sg_warn "$(sg_rel "$TFVARS_DEFAULT") exists too: terraform loads it first, $(sg_rel "$TFVARS") overrides per variable — remove it if that is not intended" + fi preflight_run apply # State export (TFC API) calls curl + jq from terraform's local-exec; make sure # both are on PATH for the apply (jq from cache if not already installed). @@ -614,7 +630,7 @@ cmd_enrich() { local tforg tforg="$(tfvars_get '.tfOrg')" [ -n "$tforg" ] || die "tfOrg not found in $(sg_rel "$TFVARS")" - SG_TFC_HOSTNAME="$(tfc_hostname)" "$SCRIPT_DIR/enrich_variable_sets.sh" "$tforg" "${PF[@]}" + TFVARS="$TFVARS" SG_TFC_HOSTNAME="$(tfc_hostname)" "$SCRIPT_DIR/enrich_variable_sets.sh" "$tforg" "${PF[@]}" } do_convert() { "$SCRIPT_DIR/convert_hcl_to_json.sh" "$1"; } @@ -1063,7 +1079,7 @@ finish_line() { # Single source of truth for shell completion (keep in sync with the parser below # and the host-only flags in sg-migrate.sh). SG_COMMANDS="init preflight apply enrich convert validate import triggers checklist all clean completion update" -SG_OPTIONS="--org --export-dir --mapping --concurrency --no-create-groups --no-variable-sets --no-vcs-triggers --skip-preflight --dry-run --no-secret-stubs --fresh --project --workspace --exclude-workspace --tag --exclude-tag --all -v --verbose -y --yes -h --help --native --local --build" +SG_OPTIONS="--org --export-dir --tfvars --mapping --concurrency --no-create-groups --no-variable-sets --no-vcs-triggers --skip-preflight --dry-run --no-secret-stubs --fresh --project --workspace --exclude-workspace --tag --exclude-tag --all -v --verbose -y --yes -h --help --native --local --build" # cmd_completion <bash|zsh> — print a completion script for sg-migrate.sh / # migrate.sh to stdout. Both shells fall back to the basename when the command @@ -1086,7 +1102,7 @@ _sg_migrate() { opts="$SG_OPTIONS" case "\$prev" in --export-dir) COMPREPLY=(\$(compgen -d -- "\$cur")); return ;; - --mapping) COMPREPLY=(\$(compgen -f -- "\$cur")); return ;; + --mapping | --tfvars) COMPREPLY=(\$(compgen -f -- "\$cur")); return ;; --org | --concurrency | --project | --workspace | --exclude-workspace | --tag | --exclude-tag) COMPREPLY=(); return ;; completion) COMPREPLY=(\$(compgen -W "bash zsh" -- "\$cur")); return ;; esac @@ -1130,6 +1146,7 @@ _sg_migrate() { _arguments -s \\ '--org[StackGuardian org for import]:org' \\ '--export-dir[Payload/state output dir]:dir:_files -/' \\ + '--tfvars[tfvars file to use instead of terraform.tfvars]:file:_files' \\ '--mapping[Project-segment -> group override map]:file:_files' \\ '--concurrency[Max parallel jobs for convert/import]:n' \\ '--no-create-groups[Require workflow groups to pre-exist]' \\ @@ -1188,6 +1205,11 @@ main() { shift ;; --mapping=*) MAPPING="${1#*=}" ;; + --tfvars) + TFVARS="$(abs_path "$2")" + shift + ;; + --tfvars=*) TFVARS="$(abs_path "${1#*=}")" ;; --concurrency) CONC="$2" shift diff --git a/sg-migrate.sh b/sg-migrate.sh index 54c622c..e57cb0e 100755 --- a/sg-migrate.sh +++ b/sg-migrate.sh @@ -17,16 +17,27 @@ IMAGE="${SG_IMAGE:-stackguardian/migrator:local}" CREDS="${TF_CREDENTIALS_FILE:-$HOME/.terraform.d/credentials.tfrc.json}" # Parse host-only flags (--native/--local, --build); everything else passes through. +# --tfvars FILE (or SG_TFVARS) is a host path: it is resolved here and handed to +# migrate.sh as SG_TFVARS, translated to the container's view when running in Docker. NATIVE="${SG_NATIVE:-0}" BUILD=0 +TFVARS_HOST="${SG_TFVARS:-}" ARGS=() +want_tfvars=0 for a in "$@"; do + if [ "$want_tfvars" -eq 1 ]; then TFVARS_HOST="$a"; want_tfvars=0; continue; fi case "$a" in --native | --local) NATIVE=1 ;; --build) BUILD=1 ;; + --tfvars) want_tfvars=1 ;; + --tfvars=*) TFVARS_HOST="${a#*=}" ;; *) ARGS+=("$a") ;; esac done +if [ -n "$TFVARS_HOST" ]; then + case "$TFVARS_HOST" in /*) ;; *) TFVARS_HOST="$(cd "$(dirname "$TFVARS_HOST")" 2>/dev/null && pwd || dirname "$TFVARS_HOST")/$(basename "$TFVARS_HOST")" ;; esac + export SG_TFVARS="$TFVARS_HOST" +fi # Help, 'clean', 'completion' and 'update' only touch the local shell/filesystem # (or git) — no container. With no command at all, migrate.sh prints the help menu. @@ -124,6 +135,18 @@ DOCKER_ARGS=(--rm -i # but CI/non-tty invocations still run — use -y there). if [ -t 0 ] && [ -t 1 ]; then DOCKER_ARGS+=(-t); fi +# A tfvars file inside the checkout is visible under /app; one outside is +# mounted read-only at a fixed path. +if [ -n "$TFVARS_HOST" ]; then + case "$TFVARS_HOST" in + "$SCRIPT_DIR"/*) DOCKER_ARGS+=(-e "SG_TFVARS=/app/${TFVARS_HOST#"$SCRIPT_DIR"/}") ;; + *) + [ -f "$TFVARS_HOST" ] || { sg_err "tfvars file not found: $TFVARS_HOST"; exit 1; } + DOCKER_ARGS+=(-v "$TFVARS_HOST:/tmp/sg-run.tfvars:ro" -e "SG_TFVARS=/tmp/sg-run.tfvars") + ;; + esac +fi + # TFC auth: prefer a long-lived TFE_TOKEN (forwarded via -e above); otherwise # mount the `terraform login` credentials file read-only. if [ -n "${TFE_TOKEN:-}" ]; then From 663fbb1f081deacce6f6ac3ec732c939f7320093 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adis=20Halilovi=C4=87?= <adis.halilovic@stackguardian.io> Date: Mon, 14 Sep 2026 16:15:00 +0200 Subject: [PATCH 51/71] fix: re-importing an existing workflow without variables failed under set -u sg_create_workflow tested SG_HTTP_CODE after a $(...) call that never sets it in the calling shell, so with set -u the 409 fallback died before the PATCH and the workflow was reported as failed with an empty message. The '<code>: <body>' text the helper prints already carries the status. --- scripts/lib/sg_api.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/lib/sg_api.sh b/scripts/lib/sg_api.sh index d9d153f..5c3ae23 100644 --- a/scripts/lib/sg_api.sh +++ b/scripts/lib/sg_api.sh @@ -176,7 +176,9 @@ sg_update_workflow() { sg_create_workflow() { local grp="$1" entry="$2" err rc=0 err="$(_sg_wf_call POST "$(sg_org_url)/wfgrps/$grp/wfs/" "$(_sg_wf_body "$entry")")" || rc=$? - if [ "$rc" -eq 22 ] && { [ "$SG_HTTP_CODE" = "409" ] || [[ "$err" == *"not unique"* ]]; }; then + # SG_HTTP_CODE is set inside the $(...) above and lost here (unset under set -u + # when no call ran in this shell yet); the "<code>: <body>" text carries it. + if [ "$rc" -eq 22 ] && { [[ "$err" == 409:* ]] || [[ "$err" == *"not unique"* ]]; }; then sg_update_workflow "$grp" "$entry" && echo updated return fi From 271981129b2c5da45e9b2ef67eb2ef28b38884c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adis=20Halilovi=C4=87?= <adis.halilovic@stackguardian.io> Date: Mon, 14 Sep 2026 16:15:00 +0200 Subject: [PATCH 52/71] feat: run result files for CI, created vs updated in the results import (and all) write export/run-result.json and run-summary.md: outcome (planned, blocked, success, failed), the scope flags, the group table, the plan problems and one row per workflow with plan, result, Terraform version, state and trigger status; the markdown is made for a CI job summary. Import results and state gain an 'updated' list so a PATCHed workflow is told from a created one (the probe workflow counts as created). --dry-run is documented for 'all' too: the local export runs, nothing is created in StackGuardian. --- scripts/lib/report.sh | 71 +++++++++++++++++++++++++++++++++++++++++-- scripts/lib/state.sh | 2 +- scripts/migrate.sh | 47 ++++++++++++++++++++-------- 3 files changed, 105 insertions(+), 15 deletions(-) diff --git a/scripts/lib/report.sh b/scripts/lib/report.sh index 2c9e371..c72cbf4 100644 --- a/scripts/lib/report.sh +++ b/scripts/lib/report.sh @@ -139,11 +139,15 @@ show_import_plan() { ((.RunnerConstraints // null) | if . == null then "preset" elif .type == "private" then "private" else "shared" end), (if (.VCSTriggers // null) != null then "yes" else "no" end), ((.VCSConfig.iacInputData.data // {}) | length), - (($S.skippedSensitiveVars // {})[$wsName] // [] | length) + (($S.skippedSensitiveVars // {})[$wsName] // [] | length), + $seg, $wsName, (($S.workspaceProjects // {})[$wsName] // "") ] | @tsv' "$f")" [ -n "$rows" ] && all_rows="$all_rows${all_rows:+$'\n'}$rows" names+=("$grp") done + # Kept for the run result: name, group, action, version, runner, triggers, + # vars, secrets, segment, workspace, project (tab-separated). + PLAN_ROWS_TSV="$all_rows" while IFS=$'\t' read -r name grp _; do [ -n "$name" ] && names+=("$name"); done <<<"$all_rows" wn="$(sg_maxlen 8 ${names[@]+"${names[@]}"})" while IFS=$'\t' read -r _ grp _; do [ -n "$grp" ] && groups+=("$grp"); done <<<"$all_rows" @@ -152,7 +156,7 @@ show_import_plan() { rw=8 case "$all_rows" in *$'\t'preset$'\t'*) rw="$(sg_maxlen 8 "preset (${SG_PRESET_RUNNER_SHORT:-})")" ;; esac printf "\n %s%-${wn}s %-${gn}s %-7s %-28s %-${rw}s %-8s %-5s %s%s\n" "$C_BOLD" "WORKFLOW" "GROUP" "ACTION" "TERRAFORM" "RUNNER" "TRIGGERS" "VARS" "SECRETS" "$C_RESET" >&2 - while IFS=$'\t' read -r name grp action tfv runner trig vars secrets; do + while IFS=$'\t' read -r name grp action tfv runner trig vars secrets _; do [ -n "$name" ] || continue if [ "$tfv" = "preset" ]; then tfv="preset${SG_PRESET_TFV:+ ($SG_PRESET_TFV)}" elif tf_version_above_ceiling "$tfv"; then tfv="${tfv#TERRAFORM-} -> ${SG_TF_FALLBACK_SHORT:-${SG_DEFAULT_TF_VERSION#TERRAFORM-}} (fallback)" @@ -166,3 +170,66 @@ show_import_plan() { sg_dim "ACTION create = new workflow; update = exists in the group, PATCHed with the current payload; skip = file already imported with identical content (--fresh re-imports)" sg_dim "TERRAFORM '-> fallback' = pinned above SG's managed ceiling (1.5.7, last FOSS release); 'preset' = left to the org's execution preset at import; SECRETS = sensitive vars recreated as placeholder secrets" } + +# write_run_result <outcome> — export/run-result.json and run-summary.md: what +# this run planned or did, per workflow, for CI jobs (the markdown is made for +# `cat export/run-summary.md >> "$GITHUB_STEP_SUMMARY"`). Outcome: planned (dry +# run), blocked (plan problems), success, failed. Reads PLAN_ROWS_TSV +# (show_import_plan), PLAN_GROUP_ROWS (plan_groups), PLAN_PROBLEMS, the merged +# run state and CHECKLIST_OPEN. Needs JQ_BIN. +write_run_result() { + local outcome="$1" json="$EXPORT_DIR/run-result.json" md="$EXPORT_DIR/run-summary.md" rows problems scope + rows="$(printf '%s\n' "${PLAN_ROWS_TSV:-}" | "$JQ_BIN" -Rc 'select(length > 0) | split("\t") + | {name: .[0], group: .[1], plan: .[2], terraformVersion: (.[3] | ltrimstr("TERRAFORM-")), runner: .[4], triggers: .[5], + vars: (.[6] | tonumber? // 0), secrets: (.[7] | tonumber? // 0), segment: .[8], workspace: .[9], project: .[10]}' | "$JQ_BIN" -sc .)" + problems="$(printf '%s\n' ${PLAN_PROBLEMS[@]+"${PLAN_PROBLEMS[@]}"} | "$JQ_BIN" -Rc 'select(length > 0)' | "$JQ_BIN" -sc .)" + scope="$("$JQ_BIN" -nc --argjson p "$([ "${#PROJECT_FILTER[@]}" -gt 0 ] && names_json "${PROJECT_FILTER[@]}" || echo '[]')" \ + --argjson w "$([ "${#WS_FILTER[@]}" -gt 0 ] && names_json "${WS_FILTER[@]}" || echo '[]')" \ + --argjson x "$([ "${#WS_EXCLUDE[@]}" -gt 0 ] && names_json "${WS_EXCLUDE[@]}" || echo '[]')" \ + --argjson t "$([ "${#TAG_FILTER[@]}" -gt 0 ] && names_json "${TAG_FILTER[@]}" || echo '[]')" \ + --argjson xt "$([ "${#TAG_EXCLUDE[@]}" -gt 0 ] && names_json "${TAG_EXCLUDE[@]}" || echo '[]')" \ + '{projects: $p, workspaces: $w, excludeWorkspaces: $x, tags: $t, excludeTags: $xt}')" + state_read | "$JQ_BIN" --arg cmd "$PROG ${SG_RUN_ARGS:-}" --arg at "$(state_now)" --argjson took "$((SECONDS - RUN_T0))" \ + --arg org "$ORG" --arg url "$SG_BASE_URL" --argjson dry "$([ "${DRY_RUN:-0}" -eq 1 ] && echo true || echo false)" \ + --arg outcome "$outcome" --argjson scope "$scope" --argjson groups "${PLAN_GROUP_ROWS:-[]}" \ + --argjson problems "$problems" --argjson rows "$rows" --argjson open "${CHECKLIST_OPEN:-0}" ' + . as $st + | { + command: $cmd, at: $at, tookSeconds: $took, org: $org, apiUrl: $url, dryRun: $dry, outcome: $outcome, + scope: $scope, groups: $groups, problems: $problems, + workflows: [ $rows[] | . as $r + | ($st.import[$r.segment] // {}) as $imp | ($st.triggers[$r.segment] // {}) as $tr + | . + { + result: (if $outcome == "planned" or $outcome == "blocked" then "planned" + elif $r.plan == "skip" then "skipped" + elif (($imp.failed // []) | index($r.name)) != null then "failed" + # planned as create but PATCHed: the probe workflow, imported once alone and once with its file + elif (($imp.updated // []) | index($r.name)) != null and $r.plan != "create" then "updated" + elif (($imp.imported // []) | index($r.name)) != null then "created" + else "not-imported" end), + tfFallback: ((($imp.tf_fallback // []) | index($r.name)) != null), + state: (if (($imp.state_uploaded // []) | index($r.name)) != null then "uploaded" + elif (($imp.state_failed // []) | index($r.name)) != null then "failed" + elif (($imp.imported // []) | index($r.name)) != null then "none" else null end), + triggers: (if $r.triggers == "no" then "none" + elif (($tr.failed // []) | index($r.name)) != null then "failed" + elif (($tr.missing // []) | index($r.name)) != null then "missing" + elif (($tr.unchanged // []) | index($r.name)) != null then "unchanged" + elif (($tr.set // []) | index($r.name)) != null then "set" else null end) + } ], + checklistOpen: $open, checklist: "post-import-checklist.md" + }' >"$json" + "$JQ_BIN" -r ' + "# StackGuardian migration: \(.outcome)", "", + "- Org: `\(.org)` (\(.apiUrl))", "- Command: `\(.command)`", "- Finished: \(.at) after \(.tookSeconds)s", + (if .dryRun then "- Dry run: nothing was created or changed in StackGuardian" else empty end), + (if (.scope | [.[]] | add | length) > 0 then "- Scope: \(.scope | to_entries | map(select(.value | length > 0) | "\(.key) \(.value | join(", "))") | join("; "))" else empty end), + "", + "| Workflow | Group | Plan | Result | Terraform | State | Triggers |", "|---|---|---|---|---|---|---|", + (.workflows[] | "| \(.name) | \(.group) | \(.plan) | \(.result) | \(.terraformVersion)\(if .tfFallback then " (fallback)" else "" end) | \(.state // "-") | \(.triggers // "-") |"), + "", + (if (.problems | length) > 0 then "## Problems", (.problems[] | "- \(.)"), "" else empty end), + (if .outcome == "planned" or .outcome == "blocked" then empty + elif .checklistOpen > 0 then "\(.checklistOpen) item(s) still need a human: see post-import-checklist.md" + else "Nothing left to do by hand." end)' "$json" >"$md" +} diff --git a/scripts/lib/state.sh b/scripts/lib/state.sh index 2cf8dc9..3ef1013 100644 --- a/scripts/lib/state.sh +++ b/scripts/lib/state.sh @@ -9,7 +9,7 @@ # # Layout: { "phases": { "<phase>": {"at": iso, "input_sha": sha} }, # "import": { "<seg>": {"at": iso, "payload_sha": sha, "group": g, -# "imported": [...], "failed": [...], +# "imported": [...], "updated": [...], "failed": [...], # "tf_fallback": [...], # "state_uploaded": [...], "state_failed": [...]} }, # "triggers": { "<seg>": {"at": iso, "group": g, "set": [...], "unchanged": [...], diff --git a/scripts/migrate.sh b/scripts/migrate.sh index 42b9cdb..a817d80 100755 --- a/scripts/migrate.sh +++ b/scripts/migrate.sh @@ -127,7 +127,9 @@ Options: --no-variable-sets Skip merging TFC Variable Set variables in the 'all' flow --no-vcs-triggers Skip registering VCS triggers after import --skip-preflight Skip the preflight checks (not recommended) - --dry-run With 'import': show the per-workflow plan and stop (nothing is created) + --dry-run With 'import' or 'all': show the per-workflow plan and stop before anything is + created in StackGuardian ('all' still runs the local export phases); + the plan is also written to export/run-result.json and run-summary.md --no-secret-stubs Do not create placeholder SG secrets for sensitive variables --fresh Ignore the saved run state: redo every phase and re-import everything --project NAME Only handle this TFC project, by name or slug (repeatable; apply exports @@ -370,8 +372,8 @@ group_for() { # name is unique within a group). Sets PLAN_TO_CREATE, PLAN_N_CREATE, # PLAN_TOTAL_WF; the caller shows the problems and stops after the full plan. plan_groups() { - local f seg grp count code status prev still n_still names fw gw i legacy=0 line - local -a paths=("$@") files=() groups=() counts=() statuses=() + local f seg grp count code status plain prev still n_still names fw gw i legacy=0 line + local -a paths=("$@") files=() groups=() counts=() statuses=() plains=() PLAN_TO_CREATE=" " PLAN_N_CREATE=0 PLAN_TOTAL_WF=0 @@ -390,17 +392,17 @@ plan_groups() { code="$(wfgroup_http_code "$grp")" case "$code" in - 200) status="${C_GREEN}reuse${C_RESET}" ;; + 200) status="${C_GREEN}reuse${C_RESET}"; plain=reuse ;; 404) if [ "$CREATE_GROUPS" -eq 1 ]; then - status="${C_YELLOW}create${C_RESET}" + status="${C_YELLOW}create${C_RESET}"; plain=create case "$PLAN_TO_CREATE" in *" $grp "*) ;; *) PLAN_TO_CREATE="$PLAN_TO_CREATE$grp " PLAN_N_CREATE=$((PLAN_N_CREATE + 1)) ;; esac else - status="${C_RED}missing!${C_RESET}" + status="${C_RED}missing!${C_RESET}"; plain="missing!" PLAN_PROBLEMS+=("workflow group '$grp' ($(basename "$f")) does not exist and --no-create-groups is set — create it in StackGuardian or drop the flag") fi ;; @@ -418,7 +420,7 @@ plan_groups() { still="$("$JQ_BIN" -nc --argjson ex "$(sg_list_workflows "$prev")" --argjson mine "$names" '[$mine[] | select(. as $n | $ex | index($n) != null)]')" n_still="$("$JQ_BIN" 'length' <<<"$still")" if [ "$n_still" -gt 0 ]; then - status="${C_RED}moved!${C_RESET}" + status="${C_RED}moved!${C_RESET}"; plain="moved!" PLAN_PROBLEMS+=("$(basename "$f"): $n_still workflow(s) already live in group '$prev' but the target is now '$grp' — StackGuardian cannot move workflows between groups; keep '$prev' (projectOverrides.\"<project>\".workflowGroup) or delete them from '$prev' first: $("$JQ_BIN" -r 'join(", ")' <<<"$still")") else sg_dim "$(basename "$f"): previous group '$prev' no longer holds these workflows — importing into '$grp'" @@ -428,7 +430,12 @@ plan_groups() { groups+=("$grp") counts+=("$count") statuses+=("$status") + plains+=("$plain") done + # Plain copy of the table for the run result (report.sh write_run_result). + PLAN_GROUP_ROWS="$(for ((i = 0; i < ${#files[@]}; i++)); do + "$JQ_BIN" -nc --arg f "${files[i]}" --arg g "${groups[i]}" --argjson n "${counts[i]}" --arg s "${plains[i]}" '{file: $f, group: $g, workflows: $n, status: $s}' + done | "$JQ_BIN" -sc .)" # Two projects may share a group only when their workflow names do not overlap. while IFS= read -r line; do @@ -755,6 +762,7 @@ import_bulk() { sg_log " $name: updated" updated+=("$name") redo+=("$name") + printf '[updated] %s\n' "$name" >>"$out" else sg_warn " $name: update failed — $(tail -n1 <<<"$err")" fi @@ -782,7 +790,12 @@ import_bulk() { while IFS= read -r entry; do name="$("$JQ_BIN" -r '.ResourceName' <<<"$entry")" if err="$(SG_NO_RETRY_RC=22 sg_retry "$RETRIES" "$RETRY_BASE" -- sg_create_workflow "$grp" "$entry" 2>/dev/null)"; then - if [ "$(tail -n1 <<<"$err")" = "updated" ]; then sg_log " $name: already existed — updated"; else sg_log " $name: created"; fi + if [ "$(tail -n1 <<<"$err")" = "updated" ]; then + sg_log " $name: already existed — updated" + printf '[updated] %s\n' "$name" >>"$out" + else + sg_log " $name: created" + fi upload_state "$grp" "$name" "$file" "$out" || true else # Same shape as sg-cli's failure line (parsed by do_import). @@ -803,7 +816,7 @@ TF_CEILING_RE='Failed to create ([^:]+): 400: .*above the highest managed versio # the trigger pass see what was actually imported); each fallback is appended to # terraform-version-fallbacks.log. Any other per-workflow failure fails the file. do_import() { - local f="$1" seg grp out rc=0 ceiling="" failed=() fb=() st_ok=() st_failed=() name line tmp names work all_names patch + local f="$1" seg grp out rc=0 ceiling="" failed=() fb=() st_ok=() st_failed=() upd=() name line tmp names work all_names patch seg="$(seg_of "$f")" grp="$(group_for "$seg")" # With --workspace / --exclude-workspace, import only the selected workflows @@ -837,6 +850,8 @@ do_import() { st_ok+=("${BASH_REMATCH[1]}") elif [[ "$line" =~ ^\[state\]\ failed\ ([^:]+): ]]; then st_failed+=("${BASH_REMATCH[1]}") + elif [[ "$line" =~ ^\[updated\]\ (.+)$ ]]; then + upd+=("${BASH_REMATCH[1]}") fi done <"$out" rm -f "$out" @@ -870,6 +885,7 @@ do_import() { printf '%s/%s: %s -> %s (above SG managed ceiling %s)\n' "$grp" "$name" "$line" "${SG_DEFAULT_TF_VERSION:-execution preset}" "$ceiling" >>"$EXPORT_DIR/terraform-version-fallbacks.log" grep -q "^\[state\] uploaded $name\$" "$out" && st_ok+=("$name") grep -q "^\[state\] failed $name:" "$out" && st_failed+=("$name") + grep -q "^\[updated\] $name\$" "$out" && upd+=("$name") fi done rm -f "$out" @@ -886,7 +902,8 @@ do_import() { --argjson fallback "$([ "${#fb[@]}" -gt 0 ] && names_json "${fb[@]}" || echo '[]')" \ --argjson st_ok "$([ "${#st_ok[@]}" -gt 0 ] && names_json "${st_ok[@]}" || echo '[]')" \ --argjson st_failed "$([ "${#st_failed[@]}" -gt 0 ] && names_json "${st_failed[@]}" || echo '[]')" \ - '{group: $g, payload_sha: $sha, imported: ($all - $failed), failed: $failed, tf_fallback: ($fallback - $failed), + --argjson upd "$([ "${#upd[@]}" -gt 0 ] && names_json "${upd[@]}" || echo '[]')" \ + '{group: $g, payload_sha: $sha, imported: ($all - $failed), updated: ($upd - $failed | unique), failed: $failed, tf_fallback: ($fallback - $failed), state_uploaded: ($st_ok - $failed - $st_failed | unique), state_failed: ($st_failed - $failed | unique)}' \ >"$EXPORT_DIR/.import-result.$seg.json" @@ -1004,10 +1021,12 @@ cmd_import() { preset_labels show_import_plan "${PF[@]}" if [ "${#PLAN_PROBLEMS[@]}" -gt 0 ]; then + write_run_result blocked die "${#PLAN_PROBLEMS[@]} problem(s) block the import (the ✗ lines above) — nothing was changed in $ORG" fi if [ "$DRY_RUN" -eq 1 ]; then - sg_success "dry run — nothing was created or changed" + write_run_result planned + sg_success "dry run — nothing was created or changed (plan written to $(sg_rel "$EXPORT_DIR")/run-result.json and run-summary.md)" return 0 fi @@ -1058,6 +1077,7 @@ cmd_import() { fi [ "$SECRET_STUBS" -eq 1 ] && create_secret_stubs write_checklist + write_run_result "$([ "$import_rc" -eq 0 ] && echo success || echo failed)" finish_line "$import_rc" return "$import_rc" } @@ -1074,6 +1094,7 @@ finish_line() { else sg_success "migration complete in $took — nothing left to do by hand" fi + sg_dim "run result: $(sg_rel "$EXPORT_DIR")/run-result.json, run-summary.md (markdown, e.g. for a CI job summary)" } # Single source of truth for shell completion (keep in sync with the parser below @@ -1153,7 +1174,7 @@ _sg_migrate() { '--no-variable-sets[Skip merging TFC Variable Sets]' \\ '--no-vcs-triggers[Skip registering VCS triggers after import]' \\ '--skip-preflight[Skip the preflight checks]' \\ - '--dry-run[With import: show the plan and stop]' \\ + '--dry-run[With import or all: show the plan and stop before importing]' \\ '--no-secret-stubs[Do not create placeholder SG secrets for sensitive vars]' \\ '--fresh[Ignore saved run state: redo every phase]' \\ '*--project[Only this TFC project (name or slug)]:project' \\ @@ -1187,6 +1208,8 @@ ZSH # half-old, half-new. main() { local CMD="" + # Recorded in export/run-result.json (write_run_result). + SG_RUN_ARGS="$*" while [ $# -gt 0 ]; do case "$1" in -y | --yes) ASSUME_YES=1 ;; From e0dcec29168be63da40c4d7869cb32926bf82bcd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adis=20Halilovi=C4=87?= <adis.halilovic@stackguardian.io> Date: Mon, 14 Sep 2026 16:17:58 +0200 Subject: [PATCH 53/71] docs: run scope flags, --tfvars, run result files, running in CI --- CLAUDE.md | 9 +++++---- README.md | 9 +++++---- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 14503a9..7a8593b 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -23,9 +23,9 @@ The migration is a user-driven pipeline, not a single program: The five phases are wrapped by an orchestrator so users don't run them by hand: -- `sg-migrate.sh` (host entrypoint, repo root) — runs `scripts/migrate.sh` **inside the Docker image** (`Dockerfile`), bind-mounting the repo at `/app` and forwarding `SG_API_TOKEN`/`SG_ORG`/`SG_BASE_URL`/`TFE_TOKEN`/`TF_TOKEN_*`. For TFC auth it prefers a long-lived `TFE_TOKEN` (env); otherwise it mounts `~/.terraform.d/credentials.tfrc.json` read-only. Runs natively instead when `--native`/`--local` is passed, `SG_NATIVE=1` is set, the command is `clean`/`completion`/help (or no command is given), or Docker is absent. Exports `SG_PROG` so `migrate.sh` shows `./sg-migrate.sh` in its usage/hints. `update` is host-only too (`cmd_update` in the wrapper, never passed to `migrate.sh`): refuses a non-git checkout, a detached HEAD or dirty tracked files, then `git pull --ff-only` and rebuilds the image only when the `Dockerfile` changed between the old and new HEAD — this is how customers pick up fixes from a plain clone (untracked config/output survive). `migrate.sh` lists `update` in its usage/`SG_COMMANDS` and rejects it with a host-only hint. `.gitattributes` forces LF so a Git-for-Windows clone does not CRLF the scripts. -- `scripts/migrate.sh` — the actual orchestrator (sources `scripts/lib/*.sh`, see below). Subcommands `init|preflight|apply|enrich|convert|validate|import|triggers|checklist|all|clean|completion` (no command prints the help menu; `enrich` runs in `all` unless `--no-variable-sets`). Flags beyond the basics: `--dry-run` (import: plan only), `--fresh` (ignore run state), `--project SEG` / `--workspace NAME` (repeatable filters; apply maps `--workspace` to `-var workspacenames=[...]`, import works on a jq-filtered temp copy), `--skip-preflight`, `--no-secret-stubs`. Resolves each tool via `sg_resolve` (PATH first — the image installs them — else `sg_ensure_*` cache). `apply`/`all` first run `require_tfc_auth`, which resolves the token the tfe provider will use (`TFE_TOKEN`, `TF_TOKEN_<host>`, or the `terraform login` file for `tfHostname`) and verifies it with `GET /api/v2/account/details`, failing fast on a missing or rejected (expired) credential. Runs `convert` and `import` in parallel (`--concurrency`, default 4), raises `terraform apply -parallelism`, and retries `sg-cli` imports with backoff (`sg_retry`, `SG_RETRIES`). Import resolves each project's workflow group with `group_for` (legacy `.sg/workflow-groups.json` entry → the payload's `CLIConfiguration.WorkflowGroup.name` via `payload_group` → `tfc-<seg>`) and runs `plan_groups`: STATUS `reuse` (exists) / `create` (missing; created after the confirmation) / `missing!` (`--no-create-groups`) / `moved!`; it collects `PLAN_PROBLEMS` — a project whose workflows still live in the group recorded in `state.json` (or the default group) when the target changed (SG cannot move workflows), and two projects sharing a group with overlapping names — prints them as ✗ lines, still shows the per-workflow plan (also under `--dry-run`) and then dies. The skip set (`state_import_done`) is computed before the plan so the ACTION column can say `skip`/`update`/`create`. Groups are never PATCHed. Imports go through `import_bulk`: workflows with Terraform variables via `sg-cli workflow create --bulk`, workflows **without** via a direct `POST .../wfs/` (`sg_create_workflow` + `sg_upload_tfstate` in `sg_api.sh`) because sg-cli drops an empty `iacInputData.data` and the API rejects the workflow — a workaround (`TODO(sg-cli)` markers) until sg-cli ships with sg-sdk-go >= v1.5.7, which sends the empty object; the direct path prints sg-cli-style `Failed to create <name>: <code>: <body>` lines so the parsing below covers both. State files are the migrator's job on both paths: `import_bulk` tracks sg-cli's per-workflow upload lines (`State file uploaded successfully` / `Failed to upload state file for <wf>`), re-uploads the failures via `upload_state` → `sg_upload_tfstate` (adds the `x-ms-blob-type: BlockBlob` header Azure Blob requires; sg-cli omits it and also treats anything but a literal `HTTP/1.1 200 OK` as failure — second `TODO(sg-cli)`), and writes `[state] uploaded|failed|none <wf>` markers into the parsed output; `do_import` turns them into `state_uploaded`/`state_failed` in the result, a workflow without its state fails the file (and `state_import_done` re-tries it next run). Existing workflows come back from sg-cli as `Failed to create <wf>: 409: Workflow ID not unique` (its update branch matches the outdated text `Workflow name not unique` — third `TODO(sg-cli)`); `import_bulk` collects those, PATCHes them via `sg_update_workflow`, re-uploads their state and drops the failure line, and `sg_create_workflow` does the same fallback on the direct path (prints `updated`). Before the parallel import, `probe_import` imports one workflow alone (first selected of the first file) and dies unless the create and the state upload both succeeded — the fail-fast for environment problems. sg-cli exits 0 even when individual workflows fail, so `do_import` parses its output: a workflow rejected as above SG's managed Terraform ceiling (1.5.7, last MPL/FOSS release) is re-imported with `SGDefaultTerraformVersion` (read from `terraform.tfvars`; an explicit `null` there means the `terraformVersion` key is deleted instead, so the org's execution preset decides — `tfvars_is_null` tells an explicit null from a missing key), the payload patched in place, and the case logged to `export/terraform-version-fallbacks.log` plus a printed notice; any other per-workflow failure fails the run. `cmd_import` reads the org's execution preset (`sg_execution_preset`, `GET /orgs/{org}/` → `Settings.workflowDefaults`) and `preset_labels` (report.sh) turns it into the `preset (1.5.7)` / `preset (private:rg)` cells of the plan and the `SG_TF_FALLBACK_LABEL` used in messages. The trigger pass (`do_set_triggers` → `sg_set_vcs_triggers`) skips workflows that do not exist in SG, keeps the sha of each posted `{VCSConfig, VCSTriggers}` body in `triggers.<seg>.sha` and does not re-POST an unchanged, already-set workflow (`--fresh` does; the endpoint upserts anyway, a 4xx "already exists" from older builds counts as success), and `sg_api_post` returns 22 on 4xx so `sg_retry` (via `SG_NO_RETRY_RC`) does not retry definitive errors. `clean` removes local artifacts (`export/`, TF state, tool cache); `clean --all` also removes config. `completion bash|zsh` prints a completion script (`cmd_completion`; commands/options come from `SG_COMMANDS`/`SG_OPTIONS`, keep them in sync with the parser and the host flags); like `clean` it always runs natively. Output is concise by default — `apply` captures terraform's init/plan output and shows only progress + the final summary (or the full log on failure, `TF_IN_AUTOMATION=1`/`-no-color`); `-v`/`--verbose` (`SG_VERBOSE`) streams everything and un-gates the convert detail lines. Colored logging via `sg_step`/`sg_log`/`sg_success`/`sg_warn`/`sg_err`/`sg_row`; paths shown relative via `sg_rel` (all auto-off when not a TTY / `NO_COLOR`). Long steps show a live progress line (`sg_run_quiet` for terraform init/apply, `run_parallel <fn> <max> <label> <items>` for convert/import/triggers — animated only when `SG_ANIMATE=1`, i.e. colors are on; plain log lines otherwise) and report their duration (`sg_fmt_secs`). Phases start with `phase_begin` — inside `all` the header is numbered (`Phase 2/5: ...`, `PHASE_TOTAL` set in `main`) and `phase_took` gives the elapsed time; `run_parallel` prints a job's output as-is when it is a single line and adds the `── <file> ──` header only for longer or failed output (always with `-v`). The import confirmation goes through `sg_confirm` (so `SG_ANSWERS_FILE` works; a declined prompt is a neutral "cancelled" line, not an error) and `import`/`all` end with `finish_line` (outcome, total time, open checklist items from `CHECKLIST_OPEN`). A Ctrl-C is trapped to clear the progress line and print a resume hint. -- `scripts/lib/` — sourced libraries (a `lib/` gitignore rule is negated for this dir): `prompt.sh` (`sg_ask`/`sg_confirm`/`sg_select`; tty or `SG_ANSWERS_FILE` for tests; `SG_NONINTERACTIVE=1`/`-y`/no TTY → defaults), `tfvars.sh` (`tfvars_get` cached hcl2json reads, `tfvars_has`/`tfvars_is_null` to tell an explicit `null` from a missing key, `tfvars_write` renders the wizard's `W_*` vars — `W_TF_SOURCE`, and `W_TF_VERSION`/`W_RUNNER_JSON` may be the literal `null`; plain strings go through `_tfvars_str`, so quotes/backslashes in names stay valid HCL; `projectOverrides`/`workspaceOverrides` are rendered from `W_PROJECT_OVERRIDES_JSON`/`W_WS_OVERRIDES_JSON` by `_tfvars_map` — `"key" = { attr = <json> }`, which hcl2json round-trips — so hand-written blocks survive a re-run of `init` (`wizard_run` reads them before writing; inner comments are lost)), `tfc_api.sh` (`tfc_http`/`tfc_get_all` paginated, `tfc_list_{orgs,projects,workspaces}` — workspaces include `vcs_provider`/`vcs_url`/`vcs_identifier` —, `tfc_select_workspaces` (the module's name-glob / include-tags(all) / exclude-tags filter, shared by wizard and preflight), `tfc_vcs_summary`/`tfc_vcs_kind_for`/`tfc_vcs_label_for`, `tfc_token` with provider precedence, `require_tfc_auth`), `sg_api.sh` (`_sg_api` 0/22/1 contract + `SG_HTTP_CODE`, `_sg_listall` paginated listall reader (the API pages at 50) behind `sg_list_workflows`/`sg_list_wfgrps`, `sg_set_vcs_triggers`, `sg_list_integrations`, `sg_integration_type`, `sg_vcs_kind_of` (connector type → `sourceConfigDestKind`), `sg_execution_preset` + `sg_preset_{desc,runner_desc,version_desc,runner_provided}` (the org's execution preset and how to describe it), `sg_*_exists`, `sg_list_workflows`, `sg_create_secret`, `sg_patch_workflow`, `wfgroup_*`), `wizard.sh` (`wizard_run`: 5 steps TFC → SG → projects → defaults → review; `wizard_projects` (with >1 selected project, from `W_SEL_PROJECTS_JSON`) asks whether the global connectors and `tfc-<project>` groups apply to all, else per project the cloud connector, VCS connector (kind + prefix follow) and group (default / existing via `sg_list_wfgrps` / typed), writing `projectOverrides`; "same as the default" deletes the key, unmanaged fields are kept, previous picks come first (`_w_first`); the defaults step also asks `stripCloudAuthVars`; the TFC step keeps the workspace list and derives, via `tfc_vcs_summary`, which VCS provider the selected workspaces use and the repositories' URL prefix — the SG step lists matching connectors first and takes the repo URL prefix from TFC (never from a previous tfvars written for another provider kind), warns on a kind mismatch, reads the org's execution preset and offers it as the runner choice; the defaults step asks whether to carry TFC versions or use the preset (`SGTerraformVersionSource`) and, for carry, which fallback (`SGDefaultTerraformVersion` or the preset = null); the review shows human-readable rows plus the `tfc-<project>` groups; `_w_select_lines` feeds menus line-by-line so names with spaces/globs survive), `preflight.sh` (`preflight_run <apply|import|all>`, once per process; ✓ lines are plain language and name the connector's role/kind, ✗/! lines name the tfvars field to fix; connector ids, runner groups and DPC kinds are collected from `SGDefault*`, `projectOverrides` and `workspaceOverrides`; besides existence it checks consistency — VCS kind vs. the connector's type, globally and per `projectOverrides` entry (fail), repo URL prefix vs. where the TFC repositories live or the kind's well-known host (warn), `projectOverrides` keys vs. the TFC project names (warn); the import context states the group policy and warns when a project's group changed since its last import; it also reports the org's execution preset next to the version/runner policy and warns when `carry` meets a preset that mounts a runner-provided binary), `state.sh` (`.sg/state.json`: `phases.<name>.input_sha`, `import.<seg>.{group,imported,failed,tf_fallback,state_uploaded,state_failed}` (`group` anchors the never-move check), `triggers.<seg>.{set,unchanged,failed,missing,sha}`, `secrets.<name>`; `run_phase` in migrate.sh skips unchanged phases), `report.sh` (`show_migration_summary` after apply — incl. a state-files row —, `show_import_plan` per-workflow table with columns sized to their content), `errors.sh` (`explain_api_error` regex→hint table, `SG_API_HINTS`), `checklist.sh` (`create_secret_stubs` → SG secret `tfc-<wf>-<VAR>` = `CHANGE_ME`, referenced as `${secret::<name>}` via PATCH + payload patch; `write_checklist` → `export/post-import-checklist.md`, printed to the terminal as one ✓/! status line per section plus the imported groups with UI links — the file has the details; section 5 covers both the TFC state export and the SG state upload (`state_failed` from the import results); sets `CHECKLIST_OPEN`). Parallel jobs (`run_parallel`) run in subshells, so `do_import`/`do_set_triggers` write `.import-result.<seg>.json` / `.triggers-result.<seg>.json` into the export dir and the caller merges them into state. +- `sg-migrate.sh` (host entrypoint, repo root) — runs `scripts/migrate.sh` **inside the Docker image** (`Dockerfile`), bind-mounting the repo at `/app` and forwarding `SG_API_TOKEN`/`SG_ORG`/`SG_BASE_URL`/`TFE_TOKEN`/`TF_TOKEN_*`. `--tfvars FILE` (or `SG_TFVARS`) is a host path: the wrapper strips the flag, resolves it and passes it on as `SG_TFVARS` — rewritten to `/app/<relative>` when the file is inside the checkout, otherwise mounted read-only at `/tmp/sg-run.tfvars`. For TFC auth it prefers a long-lived `TFE_TOKEN` (env); otherwise it mounts `~/.terraform.d/credentials.tfrc.json` read-only. Runs natively instead when `--native`/`--local` is passed, `SG_NATIVE=1` is set, the command is `clean`/`completion`/help (or no command is given), or Docker is absent. Exports `SG_PROG` so `migrate.sh` shows `./sg-migrate.sh` in its usage/hints. `update` is host-only too (`cmd_update` in the wrapper, never passed to `migrate.sh`): refuses a non-git checkout, a detached HEAD or dirty tracked files, then `git pull --ff-only` and rebuilds the image only when the `Dockerfile` changed between the old and new HEAD — this is how customers pick up fixes from a plain clone (untracked config/output survive). `migrate.sh` lists `update` in its usage/`SG_COMMANDS` and rejects it with a host-only hint. `.gitattributes` forces LF so a Git-for-Windows clone does not CRLF the scripts. +- `scripts/migrate.sh` — the actual orchestrator (sources `scripts/lib/*.sh`, see below). Subcommands `init|preflight|apply|enrich|convert|validate|import|triggers|checklist|all|clean|completion` (no command prints the help menu; `enrich` runs in `all` unless `--no-variable-sets`). Flags beyond the basics: `--dry-run` (import, or all: the local export runs, then the plan is shown and nothing is created in SG), `--fresh` (ignore run state), the scope flags from `lib/scope.sh` — `--project NAME|SLUG`, `--workspace GLOB`, `--exclude-workspace GLOB`, `--tag NAME`, `--exclude-tag NAME`, all repeatable; tfvars holds the widest scope, include flags replace the tfvars list, exclude flags add to it; apply passes them as `-var` (`tfProjects`, `workspacenames`, `tfWorkspaceIgnoreNames`, `tfWorkspaceTags`, `tfWorkspaceIgnoreTags`), the later phases apply the same names/excludes to the payload entries (`ws_selected` in bash, `WS_SCOPE_JQ` + `WS_JQ_ARGS` in jq) and `--project` to the payload files by slug (`project_selected`); `--workspace '*'` alone is not a narrowing, so the unchanged-file skip still applies; a scope that matches nothing is an error —, `--tfvars FILE` (`TFVARS`, absolute; also `-var-file` for terraform and exported to the enrich script; `clean --all` only removes the module's own file), `--skip-preflight`, `--no-secret-stubs`. `import`/`all` end by writing `export/run-result.json` + `run-summary.md` (`write_run_result`, outcomes planned/blocked/success/failed) for CI. Resolves each tool via `sg_resolve` (PATH first — the image installs them — else `sg_ensure_*` cache). `apply`/`all` first run `require_tfc_auth`, which resolves the token the tfe provider will use (`TFE_TOKEN`, `TF_TOKEN_<host>`, or the `terraform login` file for `tfHostname`) and verifies it with `GET /api/v2/account/details`, failing fast on a missing or rejected (expired) credential. Runs `convert` and `import` in parallel (`--concurrency`, default 4), raises `terraform apply -parallelism`, and retries `sg-cli` imports with backoff (`sg_retry`, `SG_RETRIES`). Import resolves each project's workflow group with `group_for` (legacy `.sg/workflow-groups.json` entry → the payload's `CLIConfiguration.WorkflowGroup.name` via `payload_group` → `tfc-<seg>`) and runs `plan_groups`: STATUS `reuse` (exists) / `create` (missing; created after the confirmation) / `missing!` (`--no-create-groups`) / `moved!`; it collects `PLAN_PROBLEMS` — a project whose workflows still live in the group recorded in `state.json` (or the default group) when the target changed (SG cannot move workflows), and two projects sharing a group with overlapping names — prints them as ✗ lines, still shows the per-workflow plan (also under `--dry-run`) and then dies. The skip set (`state_import_done`) is computed before the plan so the ACTION column can say `skip`/`update`/`create`. Groups are never PATCHed. Imports go through `import_bulk`: workflows with Terraform variables via `sg-cli workflow create --bulk`, workflows **without** via a direct `POST .../wfs/` (`sg_create_workflow` + `sg_upload_tfstate` in `sg_api.sh`) because sg-cli drops an empty `iacInputData.data` and the API rejects the workflow — a workaround (`TODO(sg-cli)` markers) until sg-cli ships with sg-sdk-go >= v1.5.7, which sends the empty object; the direct path prints sg-cli-style `Failed to create <name>: <code>: <body>` lines so the parsing below covers both. State files are the migrator's job on both paths: `import_bulk` tracks sg-cli's per-workflow upload lines (`State file uploaded successfully` / `Failed to upload state file for <wf>`), re-uploads the failures via `upload_state` → `sg_upload_tfstate` (adds the `x-ms-blob-type: BlockBlob` header Azure Blob requires; sg-cli omits it and also treats anything but a literal `HTTP/1.1 200 OK` as failure — second `TODO(sg-cli)`), and writes `[state] uploaded|failed|none <wf>` markers into the parsed output; `do_import` turns them into `state_uploaded`/`state_failed` in the result, a workflow without its state fails the file (and `state_import_done` re-tries it next run). Existing workflows come back from sg-cli as `Failed to create <wf>: 409: Workflow ID not unique` (its update branch matches the outdated text `Workflow name not unique` — third `TODO(sg-cli)`); `import_bulk` collects those, PATCHes them via `sg_update_workflow`, re-uploads their state and drops the failure line, and `sg_create_workflow` does the same fallback on the direct path (prints `updated`). Before the parallel import, `probe_import` imports one workflow alone (first selected of the first file) and dies unless the create and the state upload both succeeded — the fail-fast for environment problems. sg-cli exits 0 even when individual workflows fail, so `do_import` parses its output: a workflow rejected as above SG's managed Terraform ceiling (1.5.7, last MPL/FOSS release) is re-imported with `SGDefaultTerraformVersion` (read from `terraform.tfvars`; an explicit `null` there means the `terraformVersion` key is deleted instead, so the org's execution preset decides — `tfvars_is_null` tells an explicit null from a missing key), the payload patched in place, and the case logged to `export/terraform-version-fallbacks.log` plus a printed notice; any other per-workflow failure fails the run. `cmd_import` reads the org's execution preset (`sg_execution_preset`, `GET /orgs/{org}/` → `Settings.workflowDefaults`) and `preset_labels` (report.sh) turns it into the `preset (1.5.7)` / `preset (private:rg)` cells of the plan and the `SG_TF_FALLBACK_LABEL` used in messages. The trigger pass (`do_set_triggers` → `sg_set_vcs_triggers`) skips workflows that do not exist in SG, keeps the sha of each posted `{VCSConfig, VCSTriggers}` body in `triggers.<seg>.sha` and does not re-POST an unchanged, already-set workflow (`--fresh` does; the endpoint upserts anyway, a 4xx "already exists" from older builds counts as success), and `sg_api_post` returns 22 on 4xx so `sg_retry` (via `SG_NO_RETRY_RC`) does not retry definitive errors. `clean` removes local artifacts (`export/`, TF state, tool cache); `clean --all` also removes config. `completion bash|zsh` prints a completion script (`cmd_completion`; commands/options come from `SG_COMMANDS`/`SG_OPTIONS`, keep them in sync with the parser and the host flags); like `clean` it always runs natively. Output is concise by default — `apply` captures terraform's init/plan output and shows only progress + the final summary (or the full log on failure, `TF_IN_AUTOMATION=1`/`-no-color`); `-v`/`--verbose` (`SG_VERBOSE`) streams everything and un-gates the convert detail lines. Colored logging via `sg_step`/`sg_log`/`sg_success`/`sg_warn`/`sg_err`/`sg_row`; paths shown relative via `sg_rel` (all auto-off when not a TTY / `NO_COLOR`). Long steps show a live progress line (`sg_run_quiet` for terraform init/apply, `run_parallel <fn> <max> <label> <items>` for convert/import/triggers — animated only when `SG_ANIMATE=1`, i.e. colors are on; plain log lines otherwise) and report their duration (`sg_fmt_secs`). Phases start with `phase_begin` — inside `all` the header is numbered (`Phase 2/5: ...`, `PHASE_TOTAL` set in `main`) and `phase_took` gives the elapsed time; `run_parallel` prints a job's output as-is when it is a single line and adds the `── <file> ──` header only for longer or failed output (always with `-v`). The import confirmation goes through `sg_confirm` (so `SG_ANSWERS_FILE` works; a declined prompt is a neutral "cancelled" line, not an error) and `import`/`all` end with `finish_line` (outcome, total time, open checklist items from `CHECKLIST_OPEN`). A Ctrl-C is trapped to clear the progress line and print a resume hint. +- `scripts/lib/` — sourced libraries (a `lib/` gitignore rule is negated for this dir): `prompt.sh` (`sg_ask`/`sg_confirm`/`sg_select`; tty or `SG_ANSWERS_FILE` for tests; `SG_NONINTERACTIVE=1`/`-y`/no TTY → defaults), `tfvars.sh` (`tfvars_get` cached hcl2json reads, `tfvars_has`/`tfvars_is_null` to tell an explicit `null` from a missing key, `tfvars_write` renders the wizard's `W_*` vars — `W_TF_SOURCE`, and `W_TF_VERSION`/`W_RUNNER_JSON` may be the literal `null`; plain strings go through `_tfvars_str`, so quotes/backslashes in names stay valid HCL; `projectOverrides`/`workspaceOverrides` are rendered from `W_PROJECT_OVERRIDES_JSON`/`W_WS_OVERRIDES_JSON` by `_tfvars_map` — `"key" = { attr = <json> }`, which hcl2json round-trips — so hand-written blocks survive a re-run of `init` (`wizard_run` reads them before writing; inner comments are lost); `tfWorkspaceIgnoreNames` is kept the same way (`W_IGNORE_NAMES_JSON`, never asked, shown in the review's scope row)), `scope.sh` (the run scope: `PROJECT_FILTER`/`WS_FILTER`/`WS_EXCLUDE`/`TAG_FILTER`/`TAG_EXCLUDE` arrays, `names_json`, `slug_of` (the transformer's project-slug rule in bash), `project_selected <seg>`, `ws_narrowed`/`ws_filter_json`/`ws_exclude_json` (tfvars `tfWorkspaceIgnoreNames` ∪ flags, memoized)/`ws_selected <name>` (bash `case` globs), `WS_SCOPE_JQ` + `ws_jq_args` → `WS_JQ_ARGS` for the jq sites, `scope_tags_json`/`scope_ignore_tags_json`, `scope_tfvar_args` → `SCOPE_TFVAR_ARGS` for apply, `scope_describe`, `scope_sha_input` for the apply phase hash), `tfc_api.sh` (`tfc_http`/`tfc_get_all` paginated, `tfc_list_{orgs,projects,workspaces}` — workspaces include `vcs_provider`/`vcs_url`/`vcs_identifier` —, `tfc_select_workspaces` (the module's name-glob / include-tags(all) / exclude-tags / exclude-name-globs filter, shared by wizard and preflight), `tfc_vcs_summary`/`tfc_vcs_kind_for`/`tfc_vcs_label_for`, `tfc_token` with provider precedence, `require_tfc_auth`), `sg_api.sh` (`_sg_api` 0/22/1 contract + `SG_HTTP_CODE`, `_sg_listall` paginated listall reader (the API pages at 50) behind `sg_list_workflows`/`sg_list_wfgrps`, `sg_set_vcs_triggers`, `sg_list_integrations`, `sg_integration_type`, `sg_vcs_kind_of` (connector type → `sourceConfigDestKind`), `sg_execution_preset` + `sg_preset_{desc,runner_desc,version_desc,runner_provided}` (the org's execution preset and how to describe it), `sg_*_exists`, `sg_list_workflows`, `sg_create_secret`, `sg_patch_workflow`, `wfgroup_*`), `wizard.sh` (`wizard_run`: 5 steps TFC → SG → projects → defaults → review; `wizard_projects` (with >1 selected project, from `W_SEL_PROJECTS_JSON`) asks whether the global connectors and `tfc-<project>` groups apply to all, else per project the cloud connector, VCS connector (kind + prefix follow) and group (default / existing via `sg_list_wfgrps` / typed), writing `projectOverrides`; "same as the default" deletes the key, unmanaged fields are kept, previous picks come first (`_w_first`); the defaults step also asks `stripCloudAuthVars`; the TFC step keeps the workspace list and derives, via `tfc_vcs_summary`, which VCS provider the selected workspaces use and the repositories' URL prefix — the SG step lists matching connectors first and takes the repo URL prefix from TFC (never from a previous tfvars written for another provider kind), warns on a kind mismatch, reads the org's execution preset and offers it as the runner choice; the defaults step asks whether to carry TFC versions or use the preset (`SGTerraformVersionSource`) and, for carry, which fallback (`SGDefaultTerraformVersion` or the preset = null); the review shows human-readable rows plus the `tfc-<project>` groups; `_w_select_lines` feeds menus line-by-line so names with spaces/globs survive), `preflight.sh` (`preflight_run <apply|import|all>`, once per process; ✓ lines are plain language and name the connector's role/kind, ✗/! lines name the tfvars field to fix; connector ids, runner groups and DPC kinds are collected from `SGDefault*`, `projectOverrides` and `workspaceOverrides`; besides existence it checks consistency — VCS kind vs. the connector's type, globally and per `projectOverrides` entry (fail), repo URL prefix vs. where the TFC repositories live or the kind's well-known host (warn), `projectOverrides` keys vs. the TFC project names (warn), `--project` values vs. the project names/slugs (fail; a `tfProjects` typo warns), and the workspace-selection preview applies the CLI scope (names, excludes, tags, projects) so its count is what the apply will export; the import context states the group policy and warns when a project's group changed since its last import; it also reports the org's execution preset next to the version/runner policy and warns when `carry` meets a preset that mounts a runner-provided binary), `state.sh` (`.sg/state.json`: `phases.<name>.input_sha`, `import.<seg>.{group,imported,updated,failed,tf_fallback,state_uploaded,state_failed}` (`group` anchors the never-move check; `updated` = PATCHed on a 409), `triggers.<seg>.{set,unchanged,failed,missing,sha}`, `secrets.<name>`; `run_phase` in migrate.sh skips unchanged phases), `report.sh` (`show_migration_summary` after apply — incl. a state-files row —, `show_import_plan` per-workflow table with columns sized to their content — it keeps the rows in `PLAN_ROWS_TSV` (name, group, action, version, runner, triggers, vars, secrets, segment, workspace, project) —, `write_run_result <outcome>` → `export/run-result.json` + `run-summary.md` from `PLAN_ROWS_TSV`, `PLAN_GROUP_ROWS` (plain copy of the group table, set by `plan_groups`), `PLAN_PROBLEMS`, the merged state and `CHECKLIST_OPEN`; per workflow `result` is planned/skipped/failed/updated/created, with the probe workflow — planned `create`, PATCHed on its second pass — reported as created), `errors.sh` (`explain_api_error` regex→hint table, `SG_API_HINTS`), `checklist.sh` (`create_secret_stubs` → SG secret `tfc-<wf>-<VAR>` = `CHANGE_ME`, referenced as `${secret::<name>}` via PATCH + payload patch; `write_checklist` → `export/post-import-checklist.md`, printed to the terminal as one ✓/! status line per section plus the imported groups with UI links — the file has the details; section 5 covers both the TFC state export and the SG state upload (`state_failed` from the import results); sets `CHECKLIST_OPEN`). Parallel jobs (`run_parallel`) run in subshells, so `do_import`/`do_set_triggers` write `.import-result.<seg>.json` / `.triggers-result.<seg>.json` into the export dir and the caller merges them into state. - `scripts/tools.sh` — sourced by the other scripts (sets `SG_REPO_ROOT` to its parent dir). Provides `sg_resolve` (PATH-or-cache), `sg_ensure_{jq,hcl2json,yajsv,sgcli}` (pinned downloads into `.sg/cached/`; `sg-cli` is the Go binary), `sg_retry`, the color vars + log helpers. The Docker image bundles these tools at build time (`yajsv` built from source so arm64 works), so in-container runs never download. - **Variable sets** — `scripts/enrich_variable_sets.sh <tfc-org> <payload...>` (run by `cmd_enrich`, which reads `tfOrg`/`tfHostname` from `terraform.tfvars` via `hcl2json` — variable sets live in the **TFC** org, not `SG_ORG`). Lists all sets + vars via the TFC API (Bearer token from the `terraform login` creds file or `TFE_TOKEN`), computes per-workspace effective vars (scope: global/project/workspace; precedence: priority sets > workspace vars > non-priority sets), and merges them into each workflow (matched by workspace name parsed from `TfStateFilePath`). Sensitive set vars and key conflicts are reported. Skipped by `--no-variable-sets`. - **Workflow groups** — each TFC project maps to the SG workflow group the transformer wrote into its payload (`projectOverrides[<project>].workflowGroup`, default `tfc-<project-segment>`); an existing group is reused, a missing one created via `POST /api/v1/orgs/{org}/wfgrps/` (auth `Authorization: apikey <token>`) unless `--no-create-groups`. (Both the API calls and sg-cli honor the undocumented `SG_BASE_URL` for non-prod targets.) Groups are addressed by name, so no generated ID is tracked, and they are never PATCHed; workflows are never moved between groups (see `plan_groups`). `.sg/workflow-groups.json` (gitignored) is a deprecated per-segment override that still applies on top, with a warning. @@ -34,7 +34,7 @@ The five phases are wrapped by an orchestrator so users don't run them by hand: The whole transformation lives in `locals.tf` — there are no `outputs.tf`/`main.tf` business logic files; `main.tf` only pins provider versions. -- `data.tf` — four data sources: `tfe_workspace_ids` (selects workspaces by name/tags), `tfe_workspace` (per-workspace details), `tfe_variables` (per-workspace variables), `tfe_projects` (project id→name, used to name per-project payload files). +- `data.tf` — four data sources: `tfe_workspace_ids` (selects workspaces by name/tags), `tfe_workspace` (per-workspace details, read for `local.namedWorkspaces` = the name-filtered set, because the project filter needs its `project_id`), `tfe_variables` (per-workspace variables, for the fully filtered ids), `tfe_projects` (project id→name, used to name per-project payload files). - `locals.tf` — builds `local.workflowPayload` (workspace name → SG workflow object), groups it into `local.payloadByProject`, and assembles `local.summary`. This is the core mapping from TFC concepts to StackGuardian's payload schema. Key mappings: - TFC `terraform` (non-sensitive) variables → `VCSConfig.iacInputData.data` (kept as strings here; `try(jsondecode(...), v.value)` only decodes values that are already valid JSON). - TFC `env` (non-sensitive) variables → `EnvironmentVariables` as `PLAIN_TEXT`. @@ -43,6 +43,7 @@ The whole transformation lives in `locals.tf` — there are no `outputs.tf`/`mai - `project_id` → `CLIConfiguration.WorkflowGroup.name` = `tfc-<project-segment>` (matches the per-project filename and the group the importer creates/targets). - Sensitive variables (terraform + env) are skipped and recorded in the summary. - Overrides resolve once per workspace into `local.effective[<name>]` (workspaceOverrides > projectOverrides[<raw TFC project name>] > null = the `SGDefault*` value decides; each layer is null-filtered before `merge()`). Both maps are typed `any` with a field-name validation, because `map(object({... = optional(any)}))` refuses entries whose object-shaped fields differ. Object-shaped fields (`DeploymentPlatformConfig`, `RunnerConstraints`, `VCSTriggers`) are picked with the tuple idiom, not `? :`, since a partial override object does not unify with the derived one. `projectOverrides[<project>].workflowGroup` sets `CLIConfiguration.WorkflowGroup.name` for the whole project (default `tfc-<segment>`; `local.projectGroups`/`local.workflowGroups`); the summary records `projects` (name → segment/group/count), `workspaceProjects`, `workflowGroups` and `unknownProjectOverrides` (keys matching no TFC project). + - Workspace selection: `tfe_workspace_ids` applies `workspacenames` and the tag filters in TFC; `tfWorkspaceIgnoreNames` (globs `*`/`?`, turned into anchored regexes) and `tfProjects` (project names or slugs, `[]` = all) have no provider equivalent and are applied in `locals.tf` — `local.namedWorkspaces` (names filtered) → `local.selectedWorkspaces` (project filtered) — and everything downstream, `null_resource.exportState` included, iterates `local.selectedWorkspaces`, never the data source. `local.projectSlugs` is the single slug rule (`lower`, `[^a-z0-9-]+` → `-`), shared with `projectFileSegment`; the summary records `excludedWorkspaces`, `tfWorkspaceIgnoreNames`, `tfProjects` and `unknownProjects`. - `stripCloudAuthVars` (default true) drops the cloud credential **env** vars the workflow's connector replaces: the family is the prefix of the effective `DeploymentPlatformConfig[0].kind` (`AWS`/`AZURE`/`GCP`), the patterns come from `cloudAuthVarPatterns` (per family; setting it replaces the whole map). Stripped even when sensitive, so no placeholder secret is created for them; recorded in `strippedCloudAuthVars` + `workspaceCloudKinds`. `scripts/enrich_variable_sets.sh` applies the same rule to variable-set env vars using the payload's connector kind (defaults duplicated there — keep in sync). - `local.resourceNames` sanitizes workspace names to a valid SG `ResourceName` (≤100 chars, `^[-a-zA-Z0-9_]+$`, collision-disambiguated). For normal TFC names this is a no-op; any actual rename is reported in the summary. This is the single place to adjust naming rules. - `resources.tf` — writes one `sg-payload.<project>.json` per project directly via `for_each` (no `mv`), plus `migration-summary.{md,json}`. When `exportStateFiles=true`, `null_resource.exportState` pulls each workspace's state **directly from the TFC/TFE API** (`GET /api/v2/workspaces/{id}/current-state-version` → `hosted-state-download-url`) via a `local-exec` `curl`/`jq` script — no `terraform init` or providers per workspace (avoids the plugin-cache concurrency bug and per-workspace provider downloads). The token is read at runtime from `~/.terraform.d/credentials.tfrc.json` (the `terraform login` file) or `TFE_TOKEN`, so it never enters TF state. Idempotent (keyed by workspace name/id; `forceStateRefresh` re-pulls), with per-workspace failures (no token / no state / download error) recorded in `state-export-failures.log` instead of aborting. `cmd_apply` ensures `jq`/`curl` are on PATH for the apply. diff --git a/README.md b/README.md index 0af6a70..0bbc541 100644 --- a/README.md +++ b/README.md @@ -33,15 +33,16 @@ That's it — no IDs to look up and no workflow-group mapping to fill in. `init` - **Updating.** Clone the repo (don't fork it or download the release zip) and run `./sg-migrate.sh update` to pull the latest version; it fast-forwards the checkout and rebuilds the Docker image only if the `Dockerfile` changed. Your `terraform.tfvars`, `export/` and `.sg/` are never tracked, so they survive every update. To stay on a fixed release instead, `git checkout v1.2.2` (then `git checkout master` to follow the latest again). - Single phase: `./sg-migrate.sh preflight|apply|enrich|convert|validate|import|triggers|checklist`. Running `./sg-migrate.sh` with no command prints the help menu. - **Resume.** `all` remembers what it completed (`.sg/state.json`) and skips phases whose inputs have not changed, so after a failure you just re-run it; files already imported in full are skipped and files with failures are retried. `--fresh` redoes everything. -- **Scope.** `--project <segment>` and `--workspace <name>` (repeatable) limit every phase to a subset — migrate one team first, then the rest. -- **Dry run.** `./sg-migrate.sh import --dry-run` prints the per-workflow plan and stops; nothing is created. +- **Scope.** `terraform.tfvars` holds the widest selection (`workspacenames`, `tfWorkspaceTags`/`tfWorkspaceIgnoreTags`, `tfWorkspaceIgnoreNames`, `tfProjects`); flags narrow one run and every phase applies the same selection: `--project <name or slug>` (a TFC project, exported and imported on its own), `--workspace <glob>` (`team-*`, `*` = all; replaces `workspacenames`), `--exclude-workspace <glob>` (adds to `tfWorkspaceIgnoreNames`), `--tag <name>` / `--exclude-tag <name>` (export only). All repeatable — migrate one team first, then the rest, or drive the selection from a CI trigger. +- **Dry run.** `./sg-migrate.sh import --dry-run` (or `all --dry-run`, which still runs the local export) prints the per-workflow plan and stops; nothing is created or changed in StackGuardian. The plan is also written to `export/run-result.json` and `export/run-summary.md`. - **Sensitive variables** (which TFC never exposes) are recreated as SG secrets with the value `CHANGE_ME` and referenced from the workflows as `${secret::<name>}`; the checklist lists each one to fill in. Opt out with `--no-secret-stubs`. - TFC **Variable Set** variables are merged into the payloads automatically (the `enrich` phase, via the TFC API); skip it with `--no-variable-sets`. - `./sg-migrate.sh clean` removes local working artifacts (`export/`, Terraform state, run state, tool cache) for a fresh start; add `--all` to also remove config. `clean` (like `update` and `completion`) always runs locally. - **Per-project settings.** `projectOverrides` in `terraform.tfvars` (keyed by the TFC project name, written by `init` or by hand) sets the cloud connector, VCS connector, runners, approvers, Terraform version and the target workflow group (`workflowGroup`) for every workspace of a project; precedence is `workspaceOverrides` > `projectOverrides` > `SGDefault*`. Re-running `init` keeps hand-written `projectOverrides`/`workspaceOverrides` blocks (comments inside them are not preserved). The older `.sg/workflow-groups.json` mapping still works but is deprecated. - Output is concise by default (terraform's plan/init noise is hidden; shown on error). Add `-v`/`--verbose` for full output. Known API errors come with a hint naming the `terraform.tfvars` field to fix. -- Flags: `-y` skip the import prompt (CI; also makes `init` non-interactive), `--concurrency N` parallel jobs, `--org NAME`, `--no-create-groups` require groups to pre-exist, `--skip-preflight`, `--build` rebuild the image, `--native`/`--local` force a local run even when Docker is available, `--mapping FILE` (deprecated group override map, see *Per-project settings*). -- Tuning via env: `SG_RETRIES`, `SG_TF_PARALLELISM`, `SG_NATIVE=1`, `SG_UI_URL` (base URL for the checklist's links, default `https://app.stackguardian.io`). +- Flags: `-y` skip the import prompt (required without a terminal, e.g. in CI), `--tfvars FILE` use a tfvars file kept elsewhere, `--concurrency N` parallel jobs, `--org NAME`, `--no-create-groups` require groups to pre-exist, `--skip-preflight`, `--build` rebuild the image, `--native`/`--local` force a local run even when Docker is available, `--mapping FILE` (deprecated group override map, see *Per-project settings*). +- Tuning via env: `SG_RETRIES`, `SG_TF_PARALLELISM`, `SG_NATIVE=1`, `SG_TFVARS` (same as `--tfvars`), `SG_UI_URL` (base URL for the checklist's links, default `https://app.stackguardian.io`). +- **Running in CI.** Without a terminal every prompt takes its default: `init` cannot run the wizard (it only copies the example file), so generate `terraform.tfvars` once with `init` on a workstation, keep it next to the pipeline (it is gitignored here) and pass it with `--tfvars`; `import`/`all` need `-y`. Pick the scope per trigger with the flags above, run `all --dry-run` on a pull request and `-y all` on merge, and publish `export/run-summary.md` (e.g. `cat export/run-summary.md >> "$GITHUB_STEP_SUMMARY"`); `export/run-result.json` has the same data for scripts. Cache `.sg/` and `export/` between runs to keep the resume logic; without them every run re-exports and updates every workflow, which is correct but slower. `export/states/` holds real Terraform state — do not publish it as an artifact. - Tab completion for the current shell session: `source <(./sg-migrate.sh completion)` (bash/zsh detected; or pass `bash`/`zsh`). `init` prints this line. - TFC auth: set `TFE_TOKEN` (recommended — a long-lived token avoids re-running `terraform login`); otherwise the `terraform login` credentials file is mounted read-only into the container. Tokens are only ever read from the environment; `init` never writes them to disk. From 99e57234462f9d2934f6b5fa4482491f88323efe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adis=20Halilovi=C4=87?= <adis.halilovic@stackguardian.io> Date: Mon, 14 Sep 2026 17:08:16 +0200 Subject: [PATCH 54/71] feat: init --upgrade appends new settings to an existing tfvars; re-runs keep unmanaged keys A file written by an older version keeps working (missing settings take their defaults), but users had no way to see what is new short of diffing the example. init --upgrade appends every setting the file lacks with the comment and default from terraform.tfvars.example under a dated header and touches nothing else, so hand comments survive and it runs without a terminal; preflight names the missing settings until then. The wizard re-run now also carries over settings it does not ask about (tfProjects, cloudAuthVarPatterns, hand-added keys) instead of dropping them. --- CLAUDE.md | 4 +-- README.md | 3 +- scripts/lib/preflight.sh | 7 +++++ scripts/lib/tfvars.sh | 62 ++++++++++++++++++++++++++++++++++++++++ scripts/lib/wizard.sh | 2 ++ scripts/migrate.sh | 24 +++++++++++++++- 6 files changed, 98 insertions(+), 4 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 7a8593b..2f36750 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -24,8 +24,8 @@ The migration is a user-driven pipeline, not a single program: The five phases are wrapped by an orchestrator so users don't run them by hand: - `sg-migrate.sh` (host entrypoint, repo root) — runs `scripts/migrate.sh` **inside the Docker image** (`Dockerfile`), bind-mounting the repo at `/app` and forwarding `SG_API_TOKEN`/`SG_ORG`/`SG_BASE_URL`/`TFE_TOKEN`/`TF_TOKEN_*`. `--tfvars FILE` (or `SG_TFVARS`) is a host path: the wrapper strips the flag, resolves it and passes it on as `SG_TFVARS` — rewritten to `/app/<relative>` when the file is inside the checkout, otherwise mounted read-only at `/tmp/sg-run.tfvars`. For TFC auth it prefers a long-lived `TFE_TOKEN` (env); otherwise it mounts `~/.terraform.d/credentials.tfrc.json` read-only. Runs natively instead when `--native`/`--local` is passed, `SG_NATIVE=1` is set, the command is `clean`/`completion`/help (or no command is given), or Docker is absent. Exports `SG_PROG` so `migrate.sh` shows `./sg-migrate.sh` in its usage/hints. `update` is host-only too (`cmd_update` in the wrapper, never passed to `migrate.sh`): refuses a non-git checkout, a detached HEAD or dirty tracked files, then `git pull --ff-only` and rebuilds the image only when the `Dockerfile` changed between the old and new HEAD — this is how customers pick up fixes from a plain clone (untracked config/output survive). `migrate.sh` lists `update` in its usage/`SG_COMMANDS` and rejects it with a host-only hint. `.gitattributes` forces LF so a Git-for-Windows clone does not CRLF the scripts. -- `scripts/migrate.sh` — the actual orchestrator (sources `scripts/lib/*.sh`, see below). Subcommands `init|preflight|apply|enrich|convert|validate|import|triggers|checklist|all|clean|completion` (no command prints the help menu; `enrich` runs in `all` unless `--no-variable-sets`). Flags beyond the basics: `--dry-run` (import, or all: the local export runs, then the plan is shown and nothing is created in SG), `--fresh` (ignore run state), the scope flags from `lib/scope.sh` — `--project NAME|SLUG`, `--workspace GLOB`, `--exclude-workspace GLOB`, `--tag NAME`, `--exclude-tag NAME`, all repeatable; tfvars holds the widest scope, include flags replace the tfvars list, exclude flags add to it; apply passes them as `-var` (`tfProjects`, `workspacenames`, `tfWorkspaceIgnoreNames`, `tfWorkspaceTags`, `tfWorkspaceIgnoreTags`), the later phases apply the same names/excludes to the payload entries (`ws_selected` in bash, `WS_SCOPE_JQ` + `WS_JQ_ARGS` in jq) and `--project` to the payload files by slug (`project_selected`); `--workspace '*'` alone is not a narrowing, so the unchanged-file skip still applies; a scope that matches nothing is an error —, `--tfvars FILE` (`TFVARS`, absolute; also `-var-file` for terraform and exported to the enrich script; `clean --all` only removes the module's own file), `--skip-preflight`, `--no-secret-stubs`. `import`/`all` end by writing `export/run-result.json` + `run-summary.md` (`write_run_result`, outcomes planned/blocked/success/failed) for CI. Resolves each tool via `sg_resolve` (PATH first — the image installs them — else `sg_ensure_*` cache). `apply`/`all` first run `require_tfc_auth`, which resolves the token the tfe provider will use (`TFE_TOKEN`, `TF_TOKEN_<host>`, or the `terraform login` file for `tfHostname`) and verifies it with `GET /api/v2/account/details`, failing fast on a missing or rejected (expired) credential. Runs `convert` and `import` in parallel (`--concurrency`, default 4), raises `terraform apply -parallelism`, and retries `sg-cli` imports with backoff (`sg_retry`, `SG_RETRIES`). Import resolves each project's workflow group with `group_for` (legacy `.sg/workflow-groups.json` entry → the payload's `CLIConfiguration.WorkflowGroup.name` via `payload_group` → `tfc-<seg>`) and runs `plan_groups`: STATUS `reuse` (exists) / `create` (missing; created after the confirmation) / `missing!` (`--no-create-groups`) / `moved!`; it collects `PLAN_PROBLEMS` — a project whose workflows still live in the group recorded in `state.json` (or the default group) when the target changed (SG cannot move workflows), and two projects sharing a group with overlapping names — prints them as ✗ lines, still shows the per-workflow plan (also under `--dry-run`) and then dies. The skip set (`state_import_done`) is computed before the plan so the ACTION column can say `skip`/`update`/`create`. Groups are never PATCHed. Imports go through `import_bulk`: workflows with Terraform variables via `sg-cli workflow create --bulk`, workflows **without** via a direct `POST .../wfs/` (`sg_create_workflow` + `sg_upload_tfstate` in `sg_api.sh`) because sg-cli drops an empty `iacInputData.data` and the API rejects the workflow — a workaround (`TODO(sg-cli)` markers) until sg-cli ships with sg-sdk-go >= v1.5.7, which sends the empty object; the direct path prints sg-cli-style `Failed to create <name>: <code>: <body>` lines so the parsing below covers both. State files are the migrator's job on both paths: `import_bulk` tracks sg-cli's per-workflow upload lines (`State file uploaded successfully` / `Failed to upload state file for <wf>`), re-uploads the failures via `upload_state` → `sg_upload_tfstate` (adds the `x-ms-blob-type: BlockBlob` header Azure Blob requires; sg-cli omits it and also treats anything but a literal `HTTP/1.1 200 OK` as failure — second `TODO(sg-cli)`), and writes `[state] uploaded|failed|none <wf>` markers into the parsed output; `do_import` turns them into `state_uploaded`/`state_failed` in the result, a workflow without its state fails the file (and `state_import_done` re-tries it next run). Existing workflows come back from sg-cli as `Failed to create <wf>: 409: Workflow ID not unique` (its update branch matches the outdated text `Workflow name not unique` — third `TODO(sg-cli)`); `import_bulk` collects those, PATCHes them via `sg_update_workflow`, re-uploads their state and drops the failure line, and `sg_create_workflow` does the same fallback on the direct path (prints `updated`). Before the parallel import, `probe_import` imports one workflow alone (first selected of the first file) and dies unless the create and the state upload both succeeded — the fail-fast for environment problems. sg-cli exits 0 even when individual workflows fail, so `do_import` parses its output: a workflow rejected as above SG's managed Terraform ceiling (1.5.7, last MPL/FOSS release) is re-imported with `SGDefaultTerraformVersion` (read from `terraform.tfvars`; an explicit `null` there means the `terraformVersion` key is deleted instead, so the org's execution preset decides — `tfvars_is_null` tells an explicit null from a missing key), the payload patched in place, and the case logged to `export/terraform-version-fallbacks.log` plus a printed notice; any other per-workflow failure fails the run. `cmd_import` reads the org's execution preset (`sg_execution_preset`, `GET /orgs/{org}/` → `Settings.workflowDefaults`) and `preset_labels` (report.sh) turns it into the `preset (1.5.7)` / `preset (private:rg)` cells of the plan and the `SG_TF_FALLBACK_LABEL` used in messages. The trigger pass (`do_set_triggers` → `sg_set_vcs_triggers`) skips workflows that do not exist in SG, keeps the sha of each posted `{VCSConfig, VCSTriggers}` body in `triggers.<seg>.sha` and does not re-POST an unchanged, already-set workflow (`--fresh` does; the endpoint upserts anyway, a 4xx "already exists" from older builds counts as success), and `sg_api_post` returns 22 on 4xx so `sg_retry` (via `SG_NO_RETRY_RC`) does not retry definitive errors. `clean` removes local artifacts (`export/`, TF state, tool cache); `clean --all` also removes config. `completion bash|zsh` prints a completion script (`cmd_completion`; commands/options come from `SG_COMMANDS`/`SG_OPTIONS`, keep them in sync with the parser and the host flags); like `clean` it always runs natively. Output is concise by default — `apply` captures terraform's init/plan output and shows only progress + the final summary (or the full log on failure, `TF_IN_AUTOMATION=1`/`-no-color`); `-v`/`--verbose` (`SG_VERBOSE`) streams everything and un-gates the convert detail lines. Colored logging via `sg_step`/`sg_log`/`sg_success`/`sg_warn`/`sg_err`/`sg_row`; paths shown relative via `sg_rel` (all auto-off when not a TTY / `NO_COLOR`). Long steps show a live progress line (`sg_run_quiet` for terraform init/apply, `run_parallel <fn> <max> <label> <items>` for convert/import/triggers — animated only when `SG_ANIMATE=1`, i.e. colors are on; plain log lines otherwise) and report their duration (`sg_fmt_secs`). Phases start with `phase_begin` — inside `all` the header is numbered (`Phase 2/5: ...`, `PHASE_TOTAL` set in `main`) and `phase_took` gives the elapsed time; `run_parallel` prints a job's output as-is when it is a single line and adds the `── <file> ──` header only for longer or failed output (always with `-v`). The import confirmation goes through `sg_confirm` (so `SG_ANSWERS_FILE` works; a declined prompt is a neutral "cancelled" line, not an error) and `import`/`all` end with `finish_line` (outcome, total time, open checklist items from `CHECKLIST_OPEN`). A Ctrl-C is trapped to clear the progress line and print a resume hint. -- `scripts/lib/` — sourced libraries (a `lib/` gitignore rule is negated for this dir): `prompt.sh` (`sg_ask`/`sg_confirm`/`sg_select`; tty or `SG_ANSWERS_FILE` for tests; `SG_NONINTERACTIVE=1`/`-y`/no TTY → defaults), `tfvars.sh` (`tfvars_get` cached hcl2json reads, `tfvars_has`/`tfvars_is_null` to tell an explicit `null` from a missing key, `tfvars_write` renders the wizard's `W_*` vars — `W_TF_SOURCE`, and `W_TF_VERSION`/`W_RUNNER_JSON` may be the literal `null`; plain strings go through `_tfvars_str`, so quotes/backslashes in names stay valid HCL; `projectOverrides`/`workspaceOverrides` are rendered from `W_PROJECT_OVERRIDES_JSON`/`W_WS_OVERRIDES_JSON` by `_tfvars_map` — `"key" = { attr = <json> }`, which hcl2json round-trips — so hand-written blocks survive a re-run of `init` (`wizard_run` reads them before writing; inner comments are lost); `tfWorkspaceIgnoreNames` is kept the same way (`W_IGNORE_NAMES_JSON`, never asked, shown in the review's scope row)), `scope.sh` (the run scope: `PROJECT_FILTER`/`WS_FILTER`/`WS_EXCLUDE`/`TAG_FILTER`/`TAG_EXCLUDE` arrays, `names_json`, `slug_of` (the transformer's project-slug rule in bash), `project_selected <seg>`, `ws_narrowed`/`ws_filter_json`/`ws_exclude_json` (tfvars `tfWorkspaceIgnoreNames` ∪ flags, memoized)/`ws_selected <name>` (bash `case` globs), `WS_SCOPE_JQ` + `ws_jq_args` → `WS_JQ_ARGS` for the jq sites, `scope_tags_json`/`scope_ignore_tags_json`, `scope_tfvar_args` → `SCOPE_TFVAR_ARGS` for apply, `scope_describe`, `scope_sha_input` for the apply phase hash), `tfc_api.sh` (`tfc_http`/`tfc_get_all` paginated, `tfc_list_{orgs,projects,workspaces}` — workspaces include `vcs_provider`/`vcs_url`/`vcs_identifier` —, `tfc_select_workspaces` (the module's name-glob / include-tags(all) / exclude-tags / exclude-name-globs filter, shared by wizard and preflight), `tfc_vcs_summary`/`tfc_vcs_kind_for`/`tfc_vcs_label_for`, `tfc_token` with provider precedence, `require_tfc_auth`), `sg_api.sh` (`_sg_api` 0/22/1 contract + `SG_HTTP_CODE`, `_sg_listall` paginated listall reader (the API pages at 50) behind `sg_list_workflows`/`sg_list_wfgrps`, `sg_set_vcs_triggers`, `sg_list_integrations`, `sg_integration_type`, `sg_vcs_kind_of` (connector type → `sourceConfigDestKind`), `sg_execution_preset` + `sg_preset_{desc,runner_desc,version_desc,runner_provided}` (the org's execution preset and how to describe it), `sg_*_exists`, `sg_list_workflows`, `sg_create_secret`, `sg_patch_workflow`, `wfgroup_*`), `wizard.sh` (`wizard_run`: 5 steps TFC → SG → projects → defaults → review; `wizard_projects` (with >1 selected project, from `W_SEL_PROJECTS_JSON`) asks whether the global connectors and `tfc-<project>` groups apply to all, else per project the cloud connector, VCS connector (kind + prefix follow) and group (default / existing via `sg_list_wfgrps` / typed), writing `projectOverrides`; "same as the default" deletes the key, unmanaged fields are kept, previous picks come first (`_w_first`); the defaults step also asks `stripCloudAuthVars`; the TFC step keeps the workspace list and derives, via `tfc_vcs_summary`, which VCS provider the selected workspaces use and the repositories' URL prefix — the SG step lists matching connectors first and takes the repo URL prefix from TFC (never from a previous tfvars written for another provider kind), warns on a kind mismatch, reads the org's execution preset and offers it as the runner choice; the defaults step asks whether to carry TFC versions or use the preset (`SGTerraformVersionSource`) and, for carry, which fallback (`SGDefaultTerraformVersion` or the preset = null); the review shows human-readable rows plus the `tfc-<project>` groups; `_w_select_lines` feeds menus line-by-line so names with spaces/globs survive), `preflight.sh` (`preflight_run <apply|import|all>`, once per process; ✓ lines are plain language and name the connector's role/kind, ✗/! lines name the tfvars field to fix; connector ids, runner groups and DPC kinds are collected from `SGDefault*`, `projectOverrides` and `workspaceOverrides`; besides existence it checks consistency — VCS kind vs. the connector's type, globally and per `projectOverrides` entry (fail), repo URL prefix vs. where the TFC repositories live or the kind's well-known host (warn), `projectOverrides` keys vs. the TFC project names (warn), `--project` values vs. the project names/slugs (fail; a `tfProjects` typo warns), and the workspace-selection preview applies the CLI scope (names, excludes, tags, projects) so its count is what the apply will export; the import context states the group policy and warns when a project's group changed since its last import; it also reports the org's execution preset next to the version/runner policy and warns when `carry` meets a preset that mounts a runner-provided binary), `state.sh` (`.sg/state.json`: `phases.<name>.input_sha`, `import.<seg>.{group,imported,updated,failed,tf_fallback,state_uploaded,state_failed}` (`group` anchors the never-move check; `updated` = PATCHed on a 409), `triggers.<seg>.{set,unchanged,failed,missing,sha}`, `secrets.<name>`; `run_phase` in migrate.sh skips unchanged phases), `report.sh` (`show_migration_summary` after apply — incl. a state-files row —, `show_import_plan` per-workflow table with columns sized to their content — it keeps the rows in `PLAN_ROWS_TSV` (name, group, action, version, runner, triggers, vars, secrets, segment, workspace, project) —, `write_run_result <outcome>` → `export/run-result.json` + `run-summary.md` from `PLAN_ROWS_TSV`, `PLAN_GROUP_ROWS` (plain copy of the group table, set by `plan_groups`), `PLAN_PROBLEMS`, the merged state and `CHECKLIST_OPEN`; per workflow `result` is planned/skipped/failed/updated/created, with the probe workflow — planned `create`, PATCHed on its second pass — reported as created), `errors.sh` (`explain_api_error` regex→hint table, `SG_API_HINTS`), `checklist.sh` (`create_secret_stubs` → SG secret `tfc-<wf>-<VAR>` = `CHANGE_ME`, referenced as `${secret::<name>}` via PATCH + payload patch; `write_checklist` → `export/post-import-checklist.md`, printed to the terminal as one ✓/! status line per section plus the imported groups with UI links — the file has the details; section 5 covers both the TFC state export and the SG state upload (`state_failed` from the import results); sets `CHECKLIST_OPEN`). Parallel jobs (`run_parallel`) run in subshells, so `do_import`/`do_set_triggers` write `.import-result.<seg>.json` / `.triggers-result.<seg>.json` into the export dir and the caller merges them into state. +- `scripts/migrate.sh` — the actual orchestrator (sources `scripts/lib/*.sh`, see below). Subcommands `init|preflight|apply|enrich|convert|validate|import|triggers|checklist|all|clean|completion` (no command prints the help menu; `enrich` runs in `all` unless `--no-variable-sets`). Flags beyond the basics: `--dry-run` (import, or all: the local export runs, then the plan is shown and nothing is created in SG), `--fresh` (ignore run state), the scope flags from `lib/scope.sh` — `--project NAME|SLUG`, `--workspace GLOB`, `--exclude-workspace GLOB`, `--tag NAME`, `--exclude-tag NAME`, all repeatable; tfvars holds the widest scope, include flags replace the tfvars list, exclude flags add to it; apply passes them as `-var` (`tfProjects`, `workspacenames`, `tfWorkspaceIgnoreNames`, `tfWorkspaceTags`, `tfWorkspaceIgnoreTags`), the later phases apply the same names/excludes to the payload entries (`ws_selected` in bash, `WS_SCOPE_JQ` + `WS_JQ_ARGS` in jq) and `--project` to the payload files by slug (`project_selected`); `--workspace '*'` alone is not a narrowing, so the unchanged-file skip still applies; a scope that matches nothing is an error —, `--tfvars FILE` (`TFVARS`, absolute; also `-var-file` for terraform and exported to the enrich script; `clean --all` only removes the module's own file), `--skip-preflight`, `--no-secret-stubs`, `--upgrade` (with `init`: `tfvars_upgrade`, append the settings an older tfvars lacks; preflight warns about them until then). `import`/`all` end by writing `export/run-result.json` + `run-summary.md` (`write_run_result`, outcomes planned/blocked/success/failed) for CI. Resolves each tool via `sg_resolve` (PATH first — the image installs them — else `sg_ensure_*` cache). `apply`/`all` first run `require_tfc_auth`, which resolves the token the tfe provider will use (`TFE_TOKEN`, `TF_TOKEN_<host>`, or the `terraform login` file for `tfHostname`) and verifies it with `GET /api/v2/account/details`, failing fast on a missing or rejected (expired) credential. Runs `convert` and `import` in parallel (`--concurrency`, default 4), raises `terraform apply -parallelism`, and retries `sg-cli` imports with backoff (`sg_retry`, `SG_RETRIES`). Import resolves each project's workflow group with `group_for` (legacy `.sg/workflow-groups.json` entry → the payload's `CLIConfiguration.WorkflowGroup.name` via `payload_group` → `tfc-<seg>`) and runs `plan_groups`: STATUS `reuse` (exists) / `create` (missing; created after the confirmation) / `missing!` (`--no-create-groups`) / `moved!`; it collects `PLAN_PROBLEMS` — a project whose workflows still live in the group recorded in `state.json` (or the default group) when the target changed (SG cannot move workflows), and two projects sharing a group with overlapping names — prints them as ✗ lines, still shows the per-workflow plan (also under `--dry-run`) and then dies. The skip set (`state_import_done`) is computed before the plan so the ACTION column can say `skip`/`update`/`create`. Groups are never PATCHed. Imports go through `import_bulk`: workflows with Terraform variables via `sg-cli workflow create --bulk`, workflows **without** via a direct `POST .../wfs/` (`sg_create_workflow` + `sg_upload_tfstate` in `sg_api.sh`) because sg-cli drops an empty `iacInputData.data` and the API rejects the workflow — a workaround (`TODO(sg-cli)` markers) until sg-cli ships with sg-sdk-go >= v1.5.7, which sends the empty object; the direct path prints sg-cli-style `Failed to create <name>: <code>: <body>` lines so the parsing below covers both. State files are the migrator's job on both paths: `import_bulk` tracks sg-cli's per-workflow upload lines (`State file uploaded successfully` / `Failed to upload state file for <wf>`), re-uploads the failures via `upload_state` → `sg_upload_tfstate` (adds the `x-ms-blob-type: BlockBlob` header Azure Blob requires; sg-cli omits it and also treats anything but a literal `HTTP/1.1 200 OK` as failure — second `TODO(sg-cli)`), and writes `[state] uploaded|failed|none <wf>` markers into the parsed output; `do_import` turns them into `state_uploaded`/`state_failed` in the result, a workflow without its state fails the file (and `state_import_done` re-tries it next run). Existing workflows come back from sg-cli as `Failed to create <wf>: 409: Workflow ID not unique` (its update branch matches the outdated text `Workflow name not unique` — third `TODO(sg-cli)`); `import_bulk` collects those, PATCHes them via `sg_update_workflow`, re-uploads their state and drops the failure line, and `sg_create_workflow` does the same fallback on the direct path (prints `updated`). Before the parallel import, `probe_import` imports one workflow alone (first selected of the first file) and dies unless the create and the state upload both succeeded — the fail-fast for environment problems. sg-cli exits 0 even when individual workflows fail, so `do_import` parses its output: a workflow rejected as above SG's managed Terraform ceiling (1.5.7, last MPL/FOSS release) is re-imported with `SGDefaultTerraformVersion` (read from `terraform.tfvars`; an explicit `null` there means the `terraformVersion` key is deleted instead, so the org's execution preset decides — `tfvars_is_null` tells an explicit null from a missing key), the payload patched in place, and the case logged to `export/terraform-version-fallbacks.log` plus a printed notice; any other per-workflow failure fails the run. `cmd_import` reads the org's execution preset (`sg_execution_preset`, `GET /orgs/{org}/` → `Settings.workflowDefaults`) and `preset_labels` (report.sh) turns it into the `preset (1.5.7)` / `preset (private:rg)` cells of the plan and the `SG_TF_FALLBACK_LABEL` used in messages. The trigger pass (`do_set_triggers` → `sg_set_vcs_triggers`) skips workflows that do not exist in SG, keeps the sha of each posted `{VCSConfig, VCSTriggers}` body in `triggers.<seg>.sha` and does not re-POST an unchanged, already-set workflow (`--fresh` does; the endpoint upserts anyway, a 4xx "already exists" from older builds counts as success), and `sg_api_post` returns 22 on 4xx so `sg_retry` (via `SG_NO_RETRY_RC`) does not retry definitive errors. `clean` removes local artifacts (`export/`, TF state, tool cache); `clean --all` also removes config. `completion bash|zsh` prints a completion script (`cmd_completion`; commands/options come from `SG_COMMANDS`/`SG_OPTIONS`, keep them in sync with the parser and the host flags); like `clean` it always runs natively. Output is concise by default — `apply` captures terraform's init/plan output and shows only progress + the final summary (or the full log on failure, `TF_IN_AUTOMATION=1`/`-no-color`); `-v`/`--verbose` (`SG_VERBOSE`) streams everything and un-gates the convert detail lines. Colored logging via `sg_step`/`sg_log`/`sg_success`/`sg_warn`/`sg_err`/`sg_row`; paths shown relative via `sg_rel` (all auto-off when not a TTY / `NO_COLOR`). Long steps show a live progress line (`sg_run_quiet` for terraform init/apply, `run_parallel <fn> <max> <label> <items>` for convert/import/triggers — animated only when `SG_ANIMATE=1`, i.e. colors are on; plain log lines otherwise) and report their duration (`sg_fmt_secs`). Phases start with `phase_begin` — inside `all` the header is numbered (`Phase 2/5: ...`, `PHASE_TOTAL` set in `main`) and `phase_took` gives the elapsed time; `run_parallel` prints a job's output as-is when it is a single line and adds the `── <file> ──` header only for longer or failed output (always with `-v`). The import confirmation goes through `sg_confirm` (so `SG_ANSWERS_FILE` works; a declined prompt is a neutral "cancelled" line, not an error) and `import`/`all` end with `finish_line` (outcome, total time, open checklist items from `CHECKLIST_OPEN`). A Ctrl-C is trapped to clear the progress line and print a resume hint. +- `scripts/lib/` — sourced libraries (a `lib/` gitignore rule is negated for this dir): `prompt.sh` (`sg_ask`/`sg_confirm`/`sg_select`; tty or `SG_ANSWERS_FILE` for tests; `SG_NONINTERACTIVE=1`/`-y`/no TTY → defaults), `tfvars.sh` (`tfvars_get` cached hcl2json reads, `tfvars_has`/`tfvars_is_null` to tell an explicit `null` from a missing key, `tfvars_write` renders the wizard's `W_*` vars — `W_TF_SOURCE`, and `W_TF_VERSION`/`W_RUNNER_JSON` may be the literal `null`; plain strings go through `_tfvars_str`, so quotes/backslashes in names stay valid HCL; `projectOverrides`/`workspaceOverrides` are rendered from `W_PROJECT_OVERRIDES_JSON`/`W_WS_OVERRIDES_JSON` by `_tfvars_map` — `"key" = { attr = <json> }`, which hcl2json round-trips — so hand-written blocks survive a re-run of `init` (`wizard_run` reads them before writing; inner comments are lost); `tfWorkspaceIgnoreNames` is kept the same way (`W_IGNORE_NAMES_JSON`, never asked, shown in the review's scope row), and every other setting the wizard does not ask about (`tfProjects`, `cloudAuthVarPatterns`, hand-added keys) is appended verbatim from `W_PREV_JSON` under "Kept from the previous file"; `tfvars_missing_keys` (keys of `terraform.tfvars.example` the file lacks) and `tfvars_upgrade` (appends those paragraphs of the example — comment + default — under a dated header, `.bak` kept, touches nothing else; behind `init --upgrade`, prompt-free)), `scope.sh` (the run scope: `PROJECT_FILTER`/`WS_FILTER`/`WS_EXCLUDE`/`TAG_FILTER`/`TAG_EXCLUDE` arrays, `names_json`, `slug_of` (the transformer's project-slug rule in bash), `project_selected <seg>`, `ws_narrowed`/`ws_filter_json`/`ws_exclude_json` (tfvars `tfWorkspaceIgnoreNames` ∪ flags, memoized)/`ws_selected <name>` (bash `case` globs), `WS_SCOPE_JQ` + `ws_jq_args` → `WS_JQ_ARGS` for the jq sites, `scope_tags_json`/`scope_ignore_tags_json`, `scope_tfvar_args` → `SCOPE_TFVAR_ARGS` for apply, `scope_describe`, `scope_sha_input` for the apply phase hash), `tfc_api.sh` (`tfc_http`/`tfc_get_all` paginated, `tfc_list_{orgs,projects,workspaces}` — workspaces include `vcs_provider`/`vcs_url`/`vcs_identifier` —, `tfc_select_workspaces` (the module's name-glob / include-tags(all) / exclude-tags / exclude-name-globs filter, shared by wizard and preflight), `tfc_vcs_summary`/`tfc_vcs_kind_for`/`tfc_vcs_label_for`, `tfc_token` with provider precedence, `require_tfc_auth`), `sg_api.sh` (`_sg_api` 0/22/1 contract + `SG_HTTP_CODE`, `_sg_listall` paginated listall reader (the API pages at 50) behind `sg_list_workflows`/`sg_list_wfgrps`, `sg_set_vcs_triggers`, `sg_list_integrations`, `sg_integration_type`, `sg_vcs_kind_of` (connector type → `sourceConfigDestKind`), `sg_execution_preset` + `sg_preset_{desc,runner_desc,version_desc,runner_provided}` (the org's execution preset and how to describe it), `sg_*_exists`, `sg_list_workflows`, `sg_create_secret`, `sg_patch_workflow`, `wfgroup_*`), `wizard.sh` (`wizard_run`: 5 steps TFC → SG → projects → defaults → review; `wizard_projects` (with >1 selected project, from `W_SEL_PROJECTS_JSON`) asks whether the global connectors and `tfc-<project>` groups apply to all, else per project the cloud connector, VCS connector (kind + prefix follow) and group (default / existing via `sg_list_wfgrps` / typed), writing `projectOverrides`; "same as the default" deletes the key, unmanaged fields are kept, previous picks come first (`_w_first`); the defaults step also asks `stripCloudAuthVars`; the TFC step keeps the workspace list and derives, via `tfc_vcs_summary`, which VCS provider the selected workspaces use and the repositories' URL prefix — the SG step lists matching connectors first and takes the repo URL prefix from TFC (never from a previous tfvars written for another provider kind), warns on a kind mismatch, reads the org's execution preset and offers it as the runner choice; the defaults step asks whether to carry TFC versions or use the preset (`SGTerraformVersionSource`) and, for carry, which fallback (`SGDefaultTerraformVersion` or the preset = null); the review shows human-readable rows plus the `tfc-<project>` groups; `_w_select_lines` feeds menus line-by-line so names with spaces/globs survive), `preflight.sh` (`preflight_run <apply|import|all>`, once per process; ✓ lines are plain language and name the connector's role/kind, ✗/! lines name the tfvars field to fix; connector ids, runner groups and DPC kinds are collected from `SGDefault*`, `projectOverrides` and `workspaceOverrides`; besides existence it checks consistency — VCS kind vs. the connector's type, globally and per `projectOverrides` entry (fail), repo URL prefix vs. where the TFC repositories live or the kind's well-known host (warn), `projectOverrides` keys vs. the TFC project names (warn), `--project` values vs. the project names/slugs (fail; a `tfProjects` typo warns), and the workspace-selection preview applies the CLI scope (names, excludes, tags, projects) so its count is what the apply will export; the import context states the group policy and warns when a project's group changed since its last import; it also reports the org's execution preset next to the version/runner policy and warns when `carry` meets a preset that mounts a runner-provided binary), `state.sh` (`.sg/state.json`: `phases.<name>.input_sha`, `import.<seg>.{group,imported,updated,failed,tf_fallback,state_uploaded,state_failed}` (`group` anchors the never-move check; `updated` = PATCHed on a 409), `triggers.<seg>.{set,unchanged,failed,missing,sha}`, `secrets.<name>`; `run_phase` in migrate.sh skips unchanged phases), `report.sh` (`show_migration_summary` after apply — incl. a state-files row —, `show_import_plan` per-workflow table with columns sized to their content — it keeps the rows in `PLAN_ROWS_TSV` (name, group, action, version, runner, triggers, vars, secrets, segment, workspace, project) —, `write_run_result <outcome>` → `export/run-result.json` + `run-summary.md` from `PLAN_ROWS_TSV`, `PLAN_GROUP_ROWS` (plain copy of the group table, set by `plan_groups`), `PLAN_PROBLEMS`, the merged state and `CHECKLIST_OPEN`; per workflow `result` is planned/skipped/failed/updated/created, with the probe workflow — planned `create`, PATCHed on its second pass — reported as created), `errors.sh` (`explain_api_error` regex→hint table, `SG_API_HINTS`), `checklist.sh` (`create_secret_stubs` → SG secret `tfc-<wf>-<VAR>` = `CHANGE_ME`, referenced as `${secret::<name>}` via PATCH + payload patch; `write_checklist` → `export/post-import-checklist.md`, printed to the terminal as one ✓/! status line per section plus the imported groups with UI links — the file has the details; section 5 covers both the TFC state export and the SG state upload (`state_failed` from the import results); sets `CHECKLIST_OPEN`). Parallel jobs (`run_parallel`) run in subshells, so `do_import`/`do_set_triggers` write `.import-result.<seg>.json` / `.triggers-result.<seg>.json` into the export dir and the caller merges them into state. - `scripts/tools.sh` — sourced by the other scripts (sets `SG_REPO_ROOT` to its parent dir). Provides `sg_resolve` (PATH-or-cache), `sg_ensure_{jq,hcl2json,yajsv,sgcli}` (pinned downloads into `.sg/cached/`; `sg-cli` is the Go binary), `sg_retry`, the color vars + log helpers. The Docker image bundles these tools at build time (`yajsv` built from source so arm64 works), so in-container runs never download. - **Variable sets** — `scripts/enrich_variable_sets.sh <tfc-org> <payload...>` (run by `cmd_enrich`, which reads `tfOrg`/`tfHostname` from `terraform.tfvars` via `hcl2json` — variable sets live in the **TFC** org, not `SG_ORG`). Lists all sets + vars via the TFC API (Bearer token from the `terraform login` creds file or `TFE_TOKEN`), computes per-workspace effective vars (scope: global/project/workspace; precedence: priority sets > workspace vars > non-priority sets), and merges them into each workflow (matched by workspace name parsed from `TfStateFilePath`). Sensitive set vars and key conflicts are reported. Skipped by `--no-variable-sets`. - **Workflow groups** — each TFC project maps to the SG workflow group the transformer wrote into its payload (`projectOverrides[<project>].workflowGroup`, default `tfc-<project-segment>`); an existing group is reused, a missing one created via `POST /api/v1/orgs/{org}/wfgrps/` (auth `Authorization: apikey <token>`) unless `--no-create-groups`. (Both the API calls and sg-cli honor the undocumented `SG_BASE_URL` for non-prod targets.) Groups are addressed by name, so no generated ID is tracked, and they are never PATCHed; workflows are never moved between groups (see `plan_groups`). `.sg/workflow-groups.json` (gitignored) is a deprecated per-segment override that still applies on top, with a warning. diff --git a/README.md b/README.md index 0bbc541..03cfcd3 100644 --- a/README.md +++ b/README.md @@ -38,7 +38,8 @@ That's it — no IDs to look up and no workflow-group mapping to fill in. `init` - **Sensitive variables** (which TFC never exposes) are recreated as SG secrets with the value `CHANGE_ME` and referenced from the workflows as `${secret::<name>}`; the checklist lists each one to fill in. Opt out with `--no-secret-stubs`. - TFC **Variable Set** variables are merged into the payloads automatically (the `enrich` phase, via the TFC API); skip it with `--no-variable-sets`. - `./sg-migrate.sh clean` removes local working artifacts (`export/`, Terraform state, run state, tool cache) for a fresh start; add `--all` to also remove config. `clean` (like `update` and `completion`) always runs locally. -- **Per-project settings.** `projectOverrides` in `terraform.tfvars` (keyed by the TFC project name, written by `init` or by hand) sets the cloud connector, VCS connector, runners, approvers, Terraform version and the target workflow group (`workflowGroup`) for every workspace of a project; precedence is `workspaceOverrides` > `projectOverrides` > `SGDefault*`. Re-running `init` keeps hand-written `projectOverrides`/`workspaceOverrides` blocks (comments inside them are not preserved). The older `.sg/workflow-groups.json` mapping still works but is deprecated. +- **Per-project settings.** `projectOverrides` in `terraform.tfvars` (keyed by the TFC project name, written by `init` or by hand) sets the cloud connector, VCS connector, runners, approvers, Terraform version and the target workflow group (`workflowGroup`) for every workspace of a project; precedence is `workspaceOverrides` > `projectOverrides` > `SGDefault*`. Re-running `init` keeps hand-written `projectOverrides`/`workspaceOverrides` blocks (comments inside them are not preserved) and carries over every setting it does not ask about. The older `.sg/workflow-groups.json` mapping still works but is deprecated. +- **Keeping `terraform.tfvars` current.** After `./sg-migrate.sh update`, `./sg-migrate.sh init --upgrade` appends the settings a new version introduced to your existing file, each with its default and comment, and touches nothing else (no prompts, so it works in CI too; the previous file is kept as `.bak`). A default is the same as leaving the setting out, so this is optional — preflight lists the settings your file predates until you run it. - Output is concise by default (terraform's plan/init noise is hidden; shown on error). Add `-v`/`--verbose` for full output. Known API errors come with a hint naming the `terraform.tfvars` field to fix. - Flags: `-y` skip the import prompt (required without a terminal, e.g. in CI), `--tfvars FILE` use a tfvars file kept elsewhere, `--concurrency N` parallel jobs, `--org NAME`, `--no-create-groups` require groups to pre-exist, `--skip-preflight`, `--build` rebuild the image, `--native`/`--local` force a local run even when Docker is available, `--mapping FILE` (deprecated group override map, see *Per-project settings*). - Tuning via env: `SG_RETRIES`, `SG_TF_PARALLELISM`, `SG_NATIVE=1`, `SG_TFVARS` (same as `--tfvars`), `SG_UI_URL` (base URL for the checklist's links, default `https://app.stackguardian.io`). diff --git a/scripts/lib/preflight.sh b/scripts/lib/preflight.sh index ae70a2d..bc17b36 100644 --- a/scripts/lib/preflight.sh +++ b/scripts/lib/preflight.sh @@ -395,6 +395,13 @@ preflight_run() { pf_fail "$(sg_rel "$TFVARS") is not valid HCL: ${parse_err:-parse error}" die "fix the file (or re-run '$PROG init') and try again." fi + # A file written by an older version: the missing settings keep their + # defaults, so this only points at what is new. + local missing + missing="$(tfvars_missing_keys | tr '\n' ' ')" + if [ -n "$missing" ]; then + pf_warn "$(sg_rel "$TFVARS") predates these settings (defaults apply): ${missing% }— '$PROG init --upgrade' appends them with their defaults and comments" + fi case "$ctx" in apply) preflight_tfc diff --git a/scripts/lib/tfvars.sh b/scripts/lib/tfvars.sh index 937d7b1..261101c 100644 --- a/scripts/lib/tfvars.sh +++ b/scripts/lib/tfvars.sh @@ -217,4 +217,66 @@ $(if [ -n "${W_WS_TEMPLATE_JSON:-}" ] && [ "$W_WS_TEMPLATE_JSON" != "{}" ]; then fi) TFVARS tfvars_invalidate + # Settings the wizard does not manage (tfProjects, cloudAuthVarPatterns, a + # hand-added variable) are carried over from the previous file as they were + # (W_PREV_JSON, read by wizard_run before this rewrite), so a re-run of init + # never drops them. + if [ -n "${W_PREV_JSON:-}" ] && [ "$W_PREV_JSON" != "{}" ]; then + local jqb rendered extra k + jqb="$(sg_resolve jq sg_ensure_jq)" + rendered="$("$(sg_resolve hcl2json sg_ensure_hcl2json)" "$dest" 2>/dev/null | "$jqb" -c 'keys' || echo '[]')" + extra="$(printf '%s' "$W_PREV_JSON" | "$jqb" -r --argjson have "$rendered" 'keys - $have | .[]')" + if [ -n "$extra" ]; then + { + printf '\n# Kept from the previous file (not asked by init):\n' + while IFS= read -r k; do + [ -n "$k" ] || continue + printf '%s = %s\n' "$k" "$(_tfvars_hcl "$(printf '%s' "$W_PREV_JSON" | "$jqb" -c --arg k "$k" '.[$k]')")" + done <<<"$extra" + } >>"$dest" + fi + fi +} + +# tfvars_example — the shipped terraform.tfvars.example (every setting, with +# its default and comment); the reference for what a complete file contains. +tfvars_example() { printf '%s' "${TFVARS_EXAMPLE:-$SG_REPO_ROOT/transformer/terraform-cloud/terraform.tfvars.example}"; } + +# tfvars_missing_keys — the settings terraform.tfvars.example has that the +# current file does not, one per line (a file written by an older init). +tfvars_missing_keys() { + local jqb + jqb="$(sg_resolve jq sg_ensure_jq)" + "$jqb" -nr --argjson have "$(tfvars_json | "$jqb" -c 'keys')" \ + --argjson all "$("$(sg_resolve hcl2json sg_ensure_hcl2json)" "$(tfvars_example)" 2>/dev/null | "$jqb" -c 'keys' || echo '[]')" \ + '$all - $have | .[]' +} + +# tfvars_upgrade — append the settings the file lacks, each with the comment +# and default from terraform.tfvars.example, under a dated header. Nothing that +# is already in the file is touched (comments and formatting included), so +# this is safe for a hand-edited file and for CI. Prints the added keys, one +# per line; exit 0 with no output when the file is up to date. +tfvars_upgrade() { + local missing block keys="" + missing="$(tfvars_missing_keys)" + [ -n "$missing" ] || return 0 + # Paragraphs of the example (blank-line separated); a paragraph belongs to + # the setting it assigns (first uncommented `key =` line). Commented-out + # examples travel with the setting above them. + block="$(awk -v RS= -v ORS='\n\n' -v want="$(printf '%s' "$missing" | tr '\n' ' ') " ' + { + key = "" + n = split($0, lines, "\n") + for (i = 1; i <= n; i++) if (match(lines[i], /^[A-Za-z_][A-Za-z0-9_]* *=/)) { key = substr(lines[i], 1, RLENGTH); sub(/ *=$/, "", key); break } + if (key != "" && index(want, key " ") > 0) print + }' "$(tfvars_example)")" + [ -n "$block" ] || return 0 + cp "$TFVARS" "$TFVARS.bak" + { + printf '\n# --- Added by %s init --upgrade on %s: settings this file predates, with their\n# --- defaults (same as leaving them out). Review and adjust.\n\n' "${PROG:-sg-migrate.sh}" "$(date -u +%Y-%m-%d)" + printf '%s\n' "$block" + } >>"$TFVARS" + tfvars_invalidate + printf '%s\n' "$missing" } diff --git a/scripts/lib/wizard.sh b/scripts/lib/wizard.sh index 4a0313d..38625be 100644 --- a/scripts/lib/wizard.sh +++ b/scripts/lib/wizard.sh @@ -634,6 +634,8 @@ wizard_run() { # The name exclude list is a hand-edit / CI knob: kept as is, never asked. W_IGNORE_NAMES_JSON="$(tfvars_get_json .tfWorkspaceIgnoreNames)" [ "$W_IGNORE_NAMES_JSON" = "null" ] && W_IGNORE_NAMES_JSON='[]' + # Everything else the wizard does not ask about is carried over by tfvars_write. + W_PREV_JSON="$(tfvars_json)" wizard_tfc && wizard_sg && wizard_projects && wizard_policy || { sg_err "init aborted"; return 1; } wizard_templates wizard_review || { sg_log "nothing written"; return 1; } diff --git a/scripts/migrate.sh b/scripts/migrate.sh index a817d80..db2baa8 100755 --- a/scripts/migrate.sh +++ b/scripts/migrate.sh @@ -53,6 +53,7 @@ SG_BASE_URL_SET="${SG_BASE_URL:-}" SG_BASE_URL="${SG_BASE_URL:-https://api.app.stackguardian.io}" ASSUME_YES=0 PURGE=0 +UPGRADE=0 CREATE_GROUPS=1 ENRICH_VARSETS=1 VCS_TRIGGERS=1 @@ -89,6 +90,8 @@ Usage: $PROG [options] <command> Commands: init Guided setup: discovers TFC/SG resources and writes terraform.tfvars + (--upgrade: append the settings an older file lacks, with defaults; + nothing else is touched, no prompts) preflight Verify tokens and every connector/runner/org referenced in tfvars (runs automatically before apply, import and all) apply Run the transformer (terraform apply) to generate payloads + state @@ -142,6 +145,7 @@ Options: tfWorkspaceTags for the export) --exclude-tag NAME Leave workspaces carrying the tag out (repeatable; adds to tfWorkspaceIgnoreTags) --all With 'clean': also remove config (terraform.tfvars, mapping, .sg) + --upgrade With 'init': append missing settings to an existing terraform.tfvars -v, --verbose Show full terraform/tool output (default: concise) -y, --yes Skip the import confirmation prompt -h, --help Show this help @@ -294,6 +298,22 @@ seg_of() { cmd_init() { sg_step "Phase: init" mkdir -p "$SG_REPO_ROOT/.sg" "$SG_CACHE_BIN" + if [ "$UPGRADE" -eq 1 ]; then + # Bring an older file up to date without asking anything: append the + # settings it lacks with their defaults and comments, touch nothing else. + [ -f "$TFVARS" ] || die "Missing $(sg_rel "$TFVARS") — nothing to upgrade. Run: $PROG init" + local added parse_err + if ! parse_err="$(tfvars_valid)"; then die "$(sg_rel "$TFVARS") is not valid HCL: ${parse_err:-parse error}"; fi + added="$(tfvars_upgrade)" || die "could not upgrade $(sg_rel "$TFVARS")" + if [ -z "$added" ]; then + sg_success "$(sg_rel "$TFVARS") is up to date — every setting of this version is present" + else + sg_success "appended $(wc -l <<<"$added" | tr -d ' ') setting(s) to $(sg_rel "$TFVARS") with their defaults (previous version kept as $(basename "$TFVARS").bak):" + sed 's/^/ /' <<<"$added" >&2 + sg_dim "review them at the end of the file; a default is the same as leaving the setting out" + fi + return 0 + fi if ! sg_interactive; then # Non-interactive (CI, no TTY, -y): fall back to the template. if [ ! -f "$TFVARS" ]; then @@ -1100,7 +1120,7 @@ finish_line() { # Single source of truth for shell completion (keep in sync with the parser below # and the host-only flags in sg-migrate.sh). SG_COMMANDS="init preflight apply enrich convert validate import triggers checklist all clean completion update" -SG_OPTIONS="--org --export-dir --tfvars --mapping --concurrency --no-create-groups --no-variable-sets --no-vcs-triggers --skip-preflight --dry-run --no-secret-stubs --fresh --project --workspace --exclude-workspace --tag --exclude-tag --all -v --verbose -y --yes -h --help --native --local --build" +SG_OPTIONS="--org --export-dir --tfvars --mapping --concurrency --no-create-groups --no-variable-sets --no-vcs-triggers --skip-preflight --dry-run --no-secret-stubs --fresh --project --workspace --exclude-workspace --tag --exclude-tag --all --upgrade -v --verbose -y --yes -h --help --native --local --build" # cmd_completion <bash|zsh> — print a completion script for sg-migrate.sh / # migrate.sh to stdout. Both shells fall back to the basename when the command @@ -1183,6 +1203,7 @@ _sg_migrate() { '*--tag[Only workspaces carrying this tag]:tag' \\ '*--exclude-tag[Leave workspaces carrying this tag out]:tag' \\ '--all[With clean: also remove config]' \\ + '--upgrade[With init: append missing settings to terraform.tfvars]' \\ '(-v --verbose)'{-v,--verbose}'[Show full terraform/tool output]' \\ '(-y --yes)'{-y,--yes}'[Skip the import confirmation prompt]' \\ '(-h --help)'{-h,--help}'[Show help]' \\ @@ -1272,6 +1293,7 @@ main() { --exclude-tag=*) TAG_EXCLUDE+=("${1#*=}") ;; -v | --verbose) VERBOSE=1 ;; --all) PURGE=1 ;; + --upgrade) UPGRADE=1 ;; -h | --help) usage exit 0 From d4688c25a9d63b6f4dbe0d31358cb42c8222dd7f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adis=20Halilovi=C4=87?= <adis.halilovic@stackguardian.io> Date: Mon, 14 Sep 2026 17:13:57 +0200 Subject: [PATCH 55/71] fix: init --upgrade covers every module variable, not only the example's uncommented ones The first version derived the list from terraform.tfvars.example, so the override maps, cloudAuthVarPatterns and tfHostname were never appended. The list now comes from variables.tf; a setting the example only shows commented out is appended that way (plus 'key = <default>' when the default fits on one line), a mention in the file counts as present so a second run adds nothing, the result is parsed and rolled back if broken, and init writes tfHostname so a fresh file is complete. --- scripts/lib/tfvars.sh | 119 +++++++++++++----- .../terraform-cloud/terraform.tfvars.example | 1 + 2 files changed, 91 insertions(+), 29 deletions(-) diff --git a/scripts/lib/tfvars.sh b/scripts/lib/tfvars.sh index 261101c..5b41db8 100644 --- a/scripts/lib/tfvars.sh +++ b/scripts/lib/tfvars.sh @@ -118,11 +118,11 @@ _tfvars_str() { # The template maps are written as comments: one ready-to-uncomment entry per # selected project / workspace, pre-filled with the effective values. tfvars_write() { - local dest="$1" host_line="" tf_version_hcl + local dest="$1" host_line tf_version_hcl if [ "${W_TF_VERSION:-null}" = "null" ]; then tf_version_hcl="null"; else tf_version_hcl="$(_tfvars_str "$W_TF_VERSION")"; fi - if [ "${W_TFHOST:-app.terraform.io}" != "app.terraform.io" ]; then - host_line="$(printf '\n# Terraform Enterprise hostname (omit for Terraform Cloud)\ntfHostname = %s\n' "$(_tfvars_str "$W_TFHOST")")" - fi + # Always written (the default for Terraform Cloud), so a generated file sets + # every variable and 'init --upgrade' has nothing to add to it. + host_line="$(printf '\n# TFC/TFE hostname (app.terraform.io for Terraform Cloud)\ntfHostname = %s\n' "$(_tfvars_str "${W_TFHOST:-app.terraform.io}")")" cat >"$dest" <<TFVARS # Generated by 'sg-migrate.sh init' on $(date -u +%Y-%m-%dT%H:%M:%SZ). Safe to edit by hand; # re-run 'sg-migrate.sh init' to go through the wizard again. @@ -242,41 +242,102 @@ TFVARS # its default and comment); the reference for what a complete file contains. tfvars_example() { printf '%s' "${TFVARS_EXAMPLE:-$SG_REPO_ROOT/transformer/terraform-cloud/terraform.tfvars.example}"; } -# tfvars_missing_keys — the settings terraform.tfvars.example has that the -# current file does not, one per line (a file written by an older init). +# tfvars_variables_tf — the module's variables.tf, the source of truth for +# which settings exist. +tfvars_variables_tf() { printf '%s' "${TFVARS_VARIABLES_TF:-$SG_REPO_ROOT/transformer/terraform-cloud/variables.tf}"; } + +# tfvars_variable_names — every variable the module declares, in file order. +tfvars_variable_names() { sed -nE 's/^variable "([^"]+)".*/\1/p' "$(tfvars_variables_tf)"; } + +# _tfvars_variable_attr <name> <attr> — a single-line attribute of a variable +# block (`default = []`, `description = "..."`); empty when absent or when the +# value spans several lines (a `{`/`[` with nothing after it). +_tfvars_variable_attr() { + awk -v name="$1" -v attr="$2" ' + $0 ~ "^variable \"" name "\"" { inblock = 1; next } + inblock && /^}/ { exit } + inblock && $0 ~ "^ " attr " *=" { + sub("^ " attr " *= *", ""); sub(/ *$/, "") + if ($0 == "{" || $0 == "[") exit + print; exit + }' "$(tfvars_variables_tf)" +} + +# tfvars_missing_keys — the variables the module declares that the current +# file neither sets nor mentions as a commented-out `# key = ...` (the way the +# example and the upgrade present settings that are rarely changed, such as +# cloudAuthVarPatterns), one per line: a file written by an older init. tfvars_missing_keys() { - local jqb - jqb="$(sg_resolve jq sg_ensure_jq)" - "$jqb" -nr --argjson have "$(tfvars_json | "$jqb" -c 'keys')" \ - --argjson all "$("$(sg_resolve hcl2json sg_ensure_hcl2json)" "$(tfvars_example)" 2>/dev/null | "$jqb" -c 'keys' || echo '[]')" \ - '$all - $have | .[]' + local have + have="$(tfvars_json | "$(sg_resolve jq sg_ensure_jq)" -r 'keys[]')" + tfvars_variable_names | while IFS= read -r k; do + grep -qx -- "$k" <<<"$have" && continue + grep -qE "^# ?$k *=" "$TFVARS" 2>/dev/null && continue + printf '%s\n' "$k" + done +} + +# _tfvars_example_paragraph <key> — the paragraph of terraform.tfvars.example +# (blank-line separated) that assigns <key>, commented out or not; empty when +# the example has none. +_tfvars_example_paragraph() { + # When the match is a commented example, only the paragraph's comment lines + # are returned: an uncommented setting sharing the paragraph must not be + # appended a second time. + awk -v RS= -v key="$1" ' + { + n = split($0, lines, "\n"); hit = 0 + for (i = 1; i <= n; i++) if (lines[i] ~ ("^(# ?)?" key " *=")) { hit = (lines[i] ~ /^#/) ? 2 : 1; break } + if (!hit) next + for (i = 1; i <= n; i++) if (hit == 1 || lines[i] ~ /^#/) print lines[i] + exit + }' "$(tfvars_example)" } -# tfvars_upgrade — append the settings the file lacks, each with the comment -# and default from terraform.tfvars.example, under a dated header. Nothing that -# is already in the file is touched (comments and formatting included), so -# this is safe for a hand-edited file and for CI. Prints the added keys, one -# per line; exit 0 with no output when the file is up to date. +# tfvars_upgrade — append the settings the file lacks, each with its comment +# and default, under a dated header: the example's paragraph when it has one +# (the commented examples of the override maps and cloudAuthVarPatterns stay +# commented as guidance), plus `key = <default>` from variables.tf when the +# example does not set the key itself and the default fits on one line. +# Nothing already in the file is touched (comments and formatting included), +# so this is safe for a hand-edited file and for CI. Prints the added keys, +# one per line; exit 0 with no output when the file is up to date. tfvars_upgrade() { - local missing block keys="" + local missing k para def desc out="" missing="$(tfvars_missing_keys)" [ -n "$missing" ] || return 0 - # Paragraphs of the example (blank-line separated); a paragraph belongs to - # the setting it assigns (first uncommented `key =` line). Commented-out - # examples travel with the setting above them. - block="$(awk -v RS= -v ORS='\n\n' -v want="$(printf '%s' "$missing" | tr '\n' ' ') " ' - { - key = "" - n = split($0, lines, "\n") - for (i = 1; i <= n; i++) if (match(lines[i], /^[A-Za-z_][A-Za-z0-9_]* *=/)) { key = substr(lines[i], 1, RLENGTH); sub(/ *=$/, "", key); break } - if (key != "" && index(want, key " ") > 0) print - }' "$(tfvars_example)")" - [ -n "$block" ] || return 0 + while IFS= read -r k; do + [ -n "$k" ] || continue + para="$(_tfvars_example_paragraph "$k")" + def="$(_tfvars_variable_attr "$k" default)" + if [ -n "$para" ] && grep -qE "^$k *=" <<<"$para"; then + out="$out$para"$'\n\n' + else + if [ -z "$para" ]; then + desc="$(_tfvars_variable_attr "$k" description | sed -E 's/^"(.*)"$/\1/; s/\\"/"/g' | fold -s -w 76 | sed 's/^/# /; s/ *$//')" + [ -n "$desc" ] && out="$out$desc"$'\n' + fi + if [ -n "$def" ]; then + out="$out$k = $def"$'\n' + else + out="$out# $k: the default (see variables.tf) applies; set it here to change it."$'\n' + fi + [ -n "$para" ] && out="$out$para"$'\n' + out="$out"$'\n' + fi + done <<<"$missing" cp "$TFVARS" "$TFVARS.bak" { printf '\n# --- Added by %s init --upgrade on %s: settings this file predates, with their\n# --- defaults (same as leaving them out). Review and adjust.\n\n' "${PROG:-sg-migrate.sh}" "$(date -u +%Y-%m-%d)" - printf '%s\n' "$block" + printf '%s' "$out" } >>"$TFVARS" tfvars_invalidate + # Never leave a broken file behind: roll back when the result does not parse. + if ! tfvars_valid >/dev/null; then + cp "$TFVARS.bak" "$TFVARS" + tfvars_invalidate + sg_err "the upgraded file did not parse — restored the previous version; please report this" + return 1 + fi printf '%s\n' "$missing" } diff --git a/transformer/terraform-cloud/terraform.tfvars.example b/transformer/terraform-cloud/terraform.tfvars.example index e345830..25b6390 100644 --- a/transformer/terraform-cloud/terraform.tfvars.example +++ b/transformer/terraform-cloud/terraform.tfvars.example @@ -33,6 +33,7 @@ ignoreVarPatterns = ["^TFC_", "^TFE_"] # connector and are not migrated; which family is stripped follows each # workflow's DeploymentPlatformConfig kind. Set false to keep them. stripCloudAuthVars = true + # The patterns per family (AWS / AZURE / GCP). Setting this replaces the whole # map; the defaults are listed in variables.tf. # cloudAuthVarPatterns = { From bec15abef55c1bbaa90d0818769869b1696501f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adis=20Halilovi=C4=87?= <adis.halilovic@stackguardian.io> Date: Mon, 14 Sep 2026 17:29:02 +0200 Subject: [PATCH 56/71] feat: inline connectors and settings for one run (--cloud-connector, --vcs-connector, --runner-group, --workflow-group, --set) A pipeline can now pass the connectors with the trigger instead of editing them into terraform.tfvars. Only the connector id is typed: its kind is looked up in the org, and the VCS kind and repo URL prefix follow the connector. With --project the values become that project's projectOverrides entry, so every workspace of the project inherits them; without it they set the SGDefault* values. --set KEY=VALUE covers any other variable. The flags form an overlay merged over the tfvars for the run, so preflight, the plan, enrich, the import and the run result all see the same values and apply receives them as -var; nothing is written to the file, a later run without the flags PATCHes the workflows back. init, clean and completion refuse the flags, a standalone import warns that they only reach the workflows through the export. Also fixes tfvars_get/tfvars_get_json dropping a false value (jq's // treats false like a missing key), which read stripCloudAuthVars = false as unset. --- CLAUDE.md | 4 +- README.md | 3 +- scripts/lib/report.sh | 4 +- scripts/lib/scope.sh | 146 +++++++++++++++++++++++++++++++++++++++++- scripts/lib/tfvars.sh | 20 ++++-- scripts/migrate.sh | 62 +++++++++++++++++- 6 files changed, 227 insertions(+), 12 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 2f36750..e3e3752 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -24,8 +24,8 @@ The migration is a user-driven pipeline, not a single program: The five phases are wrapped by an orchestrator so users don't run them by hand: - `sg-migrate.sh` (host entrypoint, repo root) — runs `scripts/migrate.sh` **inside the Docker image** (`Dockerfile`), bind-mounting the repo at `/app` and forwarding `SG_API_TOKEN`/`SG_ORG`/`SG_BASE_URL`/`TFE_TOKEN`/`TF_TOKEN_*`. `--tfvars FILE` (or `SG_TFVARS`) is a host path: the wrapper strips the flag, resolves it and passes it on as `SG_TFVARS` — rewritten to `/app/<relative>` when the file is inside the checkout, otherwise mounted read-only at `/tmp/sg-run.tfvars`. For TFC auth it prefers a long-lived `TFE_TOKEN` (env); otherwise it mounts `~/.terraform.d/credentials.tfrc.json` read-only. Runs natively instead when `--native`/`--local` is passed, `SG_NATIVE=1` is set, the command is `clean`/`completion`/help (or no command is given), or Docker is absent. Exports `SG_PROG` so `migrate.sh` shows `./sg-migrate.sh` in its usage/hints. `update` is host-only too (`cmd_update` in the wrapper, never passed to `migrate.sh`): refuses a non-git checkout, a detached HEAD or dirty tracked files, then `git pull --ff-only` and rebuilds the image only when the `Dockerfile` changed between the old and new HEAD — this is how customers pick up fixes from a plain clone (untracked config/output survive). `migrate.sh` lists `update` in its usage/`SG_COMMANDS` and rejects it with a host-only hint. `.gitattributes` forces LF so a Git-for-Windows clone does not CRLF the scripts. -- `scripts/migrate.sh` — the actual orchestrator (sources `scripts/lib/*.sh`, see below). Subcommands `init|preflight|apply|enrich|convert|validate|import|triggers|checklist|all|clean|completion` (no command prints the help menu; `enrich` runs in `all` unless `--no-variable-sets`). Flags beyond the basics: `--dry-run` (import, or all: the local export runs, then the plan is shown and nothing is created in SG), `--fresh` (ignore run state), the scope flags from `lib/scope.sh` — `--project NAME|SLUG`, `--workspace GLOB`, `--exclude-workspace GLOB`, `--tag NAME`, `--exclude-tag NAME`, all repeatable; tfvars holds the widest scope, include flags replace the tfvars list, exclude flags add to it; apply passes them as `-var` (`tfProjects`, `workspacenames`, `tfWorkspaceIgnoreNames`, `tfWorkspaceTags`, `tfWorkspaceIgnoreTags`), the later phases apply the same names/excludes to the payload entries (`ws_selected` in bash, `WS_SCOPE_JQ` + `WS_JQ_ARGS` in jq) and `--project` to the payload files by slug (`project_selected`); `--workspace '*'` alone is not a narrowing, so the unchanged-file skip still applies; a scope that matches nothing is an error —, `--tfvars FILE` (`TFVARS`, absolute; also `-var-file` for terraform and exported to the enrich script; `clean --all` only removes the module's own file), `--skip-preflight`, `--no-secret-stubs`, `--upgrade` (with `init`: `tfvars_upgrade`, append the settings an older tfvars lacks; preflight warns about them until then). `import`/`all` end by writing `export/run-result.json` + `run-summary.md` (`write_run_result`, outcomes planned/blocked/success/failed) for CI. Resolves each tool via `sg_resolve` (PATH first — the image installs them — else `sg_ensure_*` cache). `apply`/`all` first run `require_tfc_auth`, which resolves the token the tfe provider will use (`TFE_TOKEN`, `TF_TOKEN_<host>`, or the `terraform login` file for `tfHostname`) and verifies it with `GET /api/v2/account/details`, failing fast on a missing or rejected (expired) credential. Runs `convert` and `import` in parallel (`--concurrency`, default 4), raises `terraform apply -parallelism`, and retries `sg-cli` imports with backoff (`sg_retry`, `SG_RETRIES`). Import resolves each project's workflow group with `group_for` (legacy `.sg/workflow-groups.json` entry → the payload's `CLIConfiguration.WorkflowGroup.name` via `payload_group` → `tfc-<seg>`) and runs `plan_groups`: STATUS `reuse` (exists) / `create` (missing; created after the confirmation) / `missing!` (`--no-create-groups`) / `moved!`; it collects `PLAN_PROBLEMS` — a project whose workflows still live in the group recorded in `state.json` (or the default group) when the target changed (SG cannot move workflows), and two projects sharing a group with overlapping names — prints them as ✗ lines, still shows the per-workflow plan (also under `--dry-run`) and then dies. The skip set (`state_import_done`) is computed before the plan so the ACTION column can say `skip`/`update`/`create`. Groups are never PATCHed. Imports go through `import_bulk`: workflows with Terraform variables via `sg-cli workflow create --bulk`, workflows **without** via a direct `POST .../wfs/` (`sg_create_workflow` + `sg_upload_tfstate` in `sg_api.sh`) because sg-cli drops an empty `iacInputData.data` and the API rejects the workflow — a workaround (`TODO(sg-cli)` markers) until sg-cli ships with sg-sdk-go >= v1.5.7, which sends the empty object; the direct path prints sg-cli-style `Failed to create <name>: <code>: <body>` lines so the parsing below covers both. State files are the migrator's job on both paths: `import_bulk` tracks sg-cli's per-workflow upload lines (`State file uploaded successfully` / `Failed to upload state file for <wf>`), re-uploads the failures via `upload_state` → `sg_upload_tfstate` (adds the `x-ms-blob-type: BlockBlob` header Azure Blob requires; sg-cli omits it and also treats anything but a literal `HTTP/1.1 200 OK` as failure — second `TODO(sg-cli)`), and writes `[state] uploaded|failed|none <wf>` markers into the parsed output; `do_import` turns them into `state_uploaded`/`state_failed` in the result, a workflow without its state fails the file (and `state_import_done` re-tries it next run). Existing workflows come back from sg-cli as `Failed to create <wf>: 409: Workflow ID not unique` (its update branch matches the outdated text `Workflow name not unique` — third `TODO(sg-cli)`); `import_bulk` collects those, PATCHes them via `sg_update_workflow`, re-uploads their state and drops the failure line, and `sg_create_workflow` does the same fallback on the direct path (prints `updated`). Before the parallel import, `probe_import` imports one workflow alone (first selected of the first file) and dies unless the create and the state upload both succeeded — the fail-fast for environment problems. sg-cli exits 0 even when individual workflows fail, so `do_import` parses its output: a workflow rejected as above SG's managed Terraform ceiling (1.5.7, last MPL/FOSS release) is re-imported with `SGDefaultTerraformVersion` (read from `terraform.tfvars`; an explicit `null` there means the `terraformVersion` key is deleted instead, so the org's execution preset decides — `tfvars_is_null` tells an explicit null from a missing key), the payload patched in place, and the case logged to `export/terraform-version-fallbacks.log` plus a printed notice; any other per-workflow failure fails the run. `cmd_import` reads the org's execution preset (`sg_execution_preset`, `GET /orgs/{org}/` → `Settings.workflowDefaults`) and `preset_labels` (report.sh) turns it into the `preset (1.5.7)` / `preset (private:rg)` cells of the plan and the `SG_TF_FALLBACK_LABEL` used in messages. The trigger pass (`do_set_triggers` → `sg_set_vcs_triggers`) skips workflows that do not exist in SG, keeps the sha of each posted `{VCSConfig, VCSTriggers}` body in `triggers.<seg>.sha` and does not re-POST an unchanged, already-set workflow (`--fresh` does; the endpoint upserts anyway, a 4xx "already exists" from older builds counts as success), and `sg_api_post` returns 22 on 4xx so `sg_retry` (via `SG_NO_RETRY_RC`) does not retry definitive errors. `clean` removes local artifacts (`export/`, TF state, tool cache); `clean --all` also removes config. `completion bash|zsh` prints a completion script (`cmd_completion`; commands/options come from `SG_COMMANDS`/`SG_OPTIONS`, keep them in sync with the parser and the host flags); like `clean` it always runs natively. Output is concise by default — `apply` captures terraform's init/plan output and shows only progress + the final summary (or the full log on failure, `TF_IN_AUTOMATION=1`/`-no-color`); `-v`/`--verbose` (`SG_VERBOSE`) streams everything and un-gates the convert detail lines. Colored logging via `sg_step`/`sg_log`/`sg_success`/`sg_warn`/`sg_err`/`sg_row`; paths shown relative via `sg_rel` (all auto-off when not a TTY / `NO_COLOR`). Long steps show a live progress line (`sg_run_quiet` for terraform init/apply, `run_parallel <fn> <max> <label> <items>` for convert/import/triggers — animated only when `SG_ANIMATE=1`, i.e. colors are on; plain log lines otherwise) and report their duration (`sg_fmt_secs`). Phases start with `phase_begin` — inside `all` the header is numbered (`Phase 2/5: ...`, `PHASE_TOTAL` set in `main`) and `phase_took` gives the elapsed time; `run_parallel` prints a job's output as-is when it is a single line and adds the `── <file> ──` header only for longer or failed output (always with `-v`). The import confirmation goes through `sg_confirm` (so `SG_ANSWERS_FILE` works; a declined prompt is a neutral "cancelled" line, not an error) and `import`/`all` end with `finish_line` (outcome, total time, open checklist items from `CHECKLIST_OPEN`). A Ctrl-C is trapped to clear the progress line and print a resume hint. -- `scripts/lib/` — sourced libraries (a `lib/` gitignore rule is negated for this dir): `prompt.sh` (`sg_ask`/`sg_confirm`/`sg_select`; tty or `SG_ANSWERS_FILE` for tests; `SG_NONINTERACTIVE=1`/`-y`/no TTY → defaults), `tfvars.sh` (`tfvars_get` cached hcl2json reads, `tfvars_has`/`tfvars_is_null` to tell an explicit `null` from a missing key, `tfvars_write` renders the wizard's `W_*` vars — `W_TF_SOURCE`, and `W_TF_VERSION`/`W_RUNNER_JSON` may be the literal `null`; plain strings go through `_tfvars_str`, so quotes/backslashes in names stay valid HCL; `projectOverrides`/`workspaceOverrides` are rendered from `W_PROJECT_OVERRIDES_JSON`/`W_WS_OVERRIDES_JSON` by `_tfvars_map` — `"key" = { attr = <json> }`, which hcl2json round-trips — so hand-written blocks survive a re-run of `init` (`wizard_run` reads them before writing; inner comments are lost); `tfWorkspaceIgnoreNames` is kept the same way (`W_IGNORE_NAMES_JSON`, never asked, shown in the review's scope row), and every other setting the wizard does not ask about (`tfProjects`, `cloudAuthVarPatterns`, hand-added keys) is appended verbatim from `W_PREV_JSON` under "Kept from the previous file"; `tfvars_missing_keys` (keys of `terraform.tfvars.example` the file lacks) and `tfvars_upgrade` (appends those paragraphs of the example — comment + default — under a dated header, `.bak` kept, touches nothing else; behind `init --upgrade`, prompt-free)), `scope.sh` (the run scope: `PROJECT_FILTER`/`WS_FILTER`/`WS_EXCLUDE`/`TAG_FILTER`/`TAG_EXCLUDE` arrays, `names_json`, `slug_of` (the transformer's project-slug rule in bash), `project_selected <seg>`, `ws_narrowed`/`ws_filter_json`/`ws_exclude_json` (tfvars `tfWorkspaceIgnoreNames` ∪ flags, memoized)/`ws_selected <name>` (bash `case` globs), `WS_SCOPE_JQ` + `ws_jq_args` → `WS_JQ_ARGS` for the jq sites, `scope_tags_json`/`scope_ignore_tags_json`, `scope_tfvar_args` → `SCOPE_TFVAR_ARGS` for apply, `scope_describe`, `scope_sha_input` for the apply phase hash), `tfc_api.sh` (`tfc_http`/`tfc_get_all` paginated, `tfc_list_{orgs,projects,workspaces}` — workspaces include `vcs_provider`/`vcs_url`/`vcs_identifier` —, `tfc_select_workspaces` (the module's name-glob / include-tags(all) / exclude-tags / exclude-name-globs filter, shared by wizard and preflight), `tfc_vcs_summary`/`tfc_vcs_kind_for`/`tfc_vcs_label_for`, `tfc_token` with provider precedence, `require_tfc_auth`), `sg_api.sh` (`_sg_api` 0/22/1 contract + `SG_HTTP_CODE`, `_sg_listall` paginated listall reader (the API pages at 50) behind `sg_list_workflows`/`sg_list_wfgrps`, `sg_set_vcs_triggers`, `sg_list_integrations`, `sg_integration_type`, `sg_vcs_kind_of` (connector type → `sourceConfigDestKind`), `sg_execution_preset` + `sg_preset_{desc,runner_desc,version_desc,runner_provided}` (the org's execution preset and how to describe it), `sg_*_exists`, `sg_list_workflows`, `sg_create_secret`, `sg_patch_workflow`, `wfgroup_*`), `wizard.sh` (`wizard_run`: 5 steps TFC → SG → projects → defaults → review; `wizard_projects` (with >1 selected project, from `W_SEL_PROJECTS_JSON`) asks whether the global connectors and `tfc-<project>` groups apply to all, else per project the cloud connector, VCS connector (kind + prefix follow) and group (default / existing via `sg_list_wfgrps` / typed), writing `projectOverrides`; "same as the default" deletes the key, unmanaged fields are kept, previous picks come first (`_w_first`); the defaults step also asks `stripCloudAuthVars`; the TFC step keeps the workspace list and derives, via `tfc_vcs_summary`, which VCS provider the selected workspaces use and the repositories' URL prefix — the SG step lists matching connectors first and takes the repo URL prefix from TFC (never from a previous tfvars written for another provider kind), warns on a kind mismatch, reads the org's execution preset and offers it as the runner choice; the defaults step asks whether to carry TFC versions or use the preset (`SGTerraformVersionSource`) and, for carry, which fallback (`SGDefaultTerraformVersion` or the preset = null); the review shows human-readable rows plus the `tfc-<project>` groups; `_w_select_lines` feeds menus line-by-line so names with spaces/globs survive), `preflight.sh` (`preflight_run <apply|import|all>`, once per process; ✓ lines are plain language and name the connector's role/kind, ✗/! lines name the tfvars field to fix; connector ids, runner groups and DPC kinds are collected from `SGDefault*`, `projectOverrides` and `workspaceOverrides`; besides existence it checks consistency — VCS kind vs. the connector's type, globally and per `projectOverrides` entry (fail), repo URL prefix vs. where the TFC repositories live or the kind's well-known host (warn), `projectOverrides` keys vs. the TFC project names (warn), `--project` values vs. the project names/slugs (fail; a `tfProjects` typo warns), and the workspace-selection preview applies the CLI scope (names, excludes, tags, projects) so its count is what the apply will export; the import context states the group policy and warns when a project's group changed since its last import; it also reports the org's execution preset next to the version/runner policy and warns when `carry` meets a preset that mounts a runner-provided binary), `state.sh` (`.sg/state.json`: `phases.<name>.input_sha`, `import.<seg>.{group,imported,updated,failed,tf_fallback,state_uploaded,state_failed}` (`group` anchors the never-move check; `updated` = PATCHed on a 409), `triggers.<seg>.{set,unchanged,failed,missing,sha}`, `secrets.<name>`; `run_phase` in migrate.sh skips unchanged phases), `report.sh` (`show_migration_summary` after apply — incl. a state-files row —, `show_import_plan` per-workflow table with columns sized to their content — it keeps the rows in `PLAN_ROWS_TSV` (name, group, action, version, runner, triggers, vars, secrets, segment, workspace, project) —, `write_run_result <outcome>` → `export/run-result.json` + `run-summary.md` from `PLAN_ROWS_TSV`, `PLAN_GROUP_ROWS` (plain copy of the group table, set by `plan_groups`), `PLAN_PROBLEMS`, the merged state and `CHECKLIST_OPEN`; per workflow `result` is planned/skipped/failed/updated/created, with the probe workflow — planned `create`, PATCHed on its second pass — reported as created), `errors.sh` (`explain_api_error` regex→hint table, `SG_API_HINTS`), `checklist.sh` (`create_secret_stubs` → SG secret `tfc-<wf>-<VAR>` = `CHANGE_ME`, referenced as `${secret::<name>}` via PATCH + payload patch; `write_checklist` → `export/post-import-checklist.md`, printed to the terminal as one ✓/! status line per section plus the imported groups with UI links — the file has the details; section 5 covers both the TFC state export and the SG state upload (`state_failed` from the import results); sets `CHECKLIST_OPEN`). Parallel jobs (`run_parallel`) run in subshells, so `do_import`/`do_set_triggers` write `.import-result.<seg>.json` / `.triggers-result.<seg>.json` into the export dir and the caller merges them into state. +- `scripts/migrate.sh` — the actual orchestrator (sources `scripts/lib/*.sh`, see below). Subcommands `init|preflight|apply|enrich|convert|validate|import|triggers|checklist|all|clean|completion` (no command prints the help menu; `enrich` runs in `all` unless `--no-variable-sets`). Flags beyond the basics: `--dry-run` (import, or all: the local export runs, then the plan is shown and nothing is created in SG), `--fresh` (ignore run state), the scope flags from `lib/scope.sh` — `--project NAME|SLUG`, `--workspace GLOB`, `--exclude-workspace GLOB`, `--tag NAME`, `--exclude-tag NAME`, all repeatable; tfvars holds the widest scope, include flags replace the tfvars list, exclude flags add to it; apply passes them as `-var` (`tfProjects`, `workspacenames`, `tfWorkspaceIgnoreNames`, `tfWorkspaceTags`, `tfWorkspaceIgnoreTags`), the later phases apply the same names/excludes to the payload entries (`ws_selected` in bash, `WS_SCOPE_JQ` + `WS_JQ_ARGS` in jq) and `--project` to the payload files by slug (`project_selected`); `--workspace '*'` alone is not a narrowing, so the unchanged-file skip still applies; a scope that matches nothing is an error —, `--tfvars FILE` (`TFVARS`, absolute; also `-var-file` for terraform and exported to the enrich script; `clean --all` only removes the module's own file), `--skip-preflight`, `--no-secret-stubs`, `--upgrade` (with `init`: `tfvars_upgrade`, append the settings an older tfvars lacks; preflight warns about them until then), and the run configuration flags `--set KEY=VALUE`, `--cloud-connector ID`, `--vcs-connector ID`, `--runner-group NAME|shared`, `--workflow-group NAME` (one-run overlay on the tfvars, see `scope.sh`; connector kinds looked up in SG; with `--project` they are that project's `projectOverrides`, else the `SGDefault*` values). `import`/`all` end by writing `export/run-result.json` + `run-summary.md` (`write_run_result`, outcomes planned/blocked/success/failed) for CI. Resolves each tool via `sg_resolve` (PATH first — the image installs them — else `sg_ensure_*` cache). `apply`/`all` first run `require_tfc_auth`, which resolves the token the tfe provider will use (`TFE_TOKEN`, `TF_TOKEN_<host>`, or the `terraform login` file for `tfHostname`) and verifies it with `GET /api/v2/account/details`, failing fast on a missing or rejected (expired) credential. Runs `convert` and `import` in parallel (`--concurrency`, default 4), raises `terraform apply -parallelism`, and retries `sg-cli` imports with backoff (`sg_retry`, `SG_RETRIES`). Import resolves each project's workflow group with `group_for` (legacy `.sg/workflow-groups.json` entry → the payload's `CLIConfiguration.WorkflowGroup.name` via `payload_group` → `tfc-<seg>`) and runs `plan_groups`: STATUS `reuse` (exists) / `create` (missing; created after the confirmation) / `missing!` (`--no-create-groups`) / `moved!`; it collects `PLAN_PROBLEMS` — a project whose workflows still live in the group recorded in `state.json` (or the default group) when the target changed (SG cannot move workflows), and two projects sharing a group with overlapping names — prints them as ✗ lines, still shows the per-workflow plan (also under `--dry-run`) and then dies. The skip set (`state_import_done`) is computed before the plan so the ACTION column can say `skip`/`update`/`create`. Groups are never PATCHed. Imports go through `import_bulk`: workflows with Terraform variables via `sg-cli workflow create --bulk`, workflows **without** via a direct `POST .../wfs/` (`sg_create_workflow` + `sg_upload_tfstate` in `sg_api.sh`) because sg-cli drops an empty `iacInputData.data` and the API rejects the workflow — a workaround (`TODO(sg-cli)` markers) until sg-cli ships with sg-sdk-go >= v1.5.7, which sends the empty object; the direct path prints sg-cli-style `Failed to create <name>: <code>: <body>` lines so the parsing below covers both. State files are the migrator's job on both paths: `import_bulk` tracks sg-cli's per-workflow upload lines (`State file uploaded successfully` / `Failed to upload state file for <wf>`), re-uploads the failures via `upload_state` → `sg_upload_tfstate` (adds the `x-ms-blob-type: BlockBlob` header Azure Blob requires; sg-cli omits it and also treats anything but a literal `HTTP/1.1 200 OK` as failure — second `TODO(sg-cli)`), and writes `[state] uploaded|failed|none <wf>` markers into the parsed output; `do_import` turns them into `state_uploaded`/`state_failed` in the result, a workflow without its state fails the file (and `state_import_done` re-tries it next run). Existing workflows come back from sg-cli as `Failed to create <wf>: 409: Workflow ID not unique` (its update branch matches the outdated text `Workflow name not unique` — third `TODO(sg-cli)`); `import_bulk` collects those, PATCHes them via `sg_update_workflow`, re-uploads their state and drops the failure line, and `sg_create_workflow` does the same fallback on the direct path (prints `updated`). Before the parallel import, `probe_import` imports one workflow alone (first selected of the first file) and dies unless the create and the state upload both succeeded — the fail-fast for environment problems. sg-cli exits 0 even when individual workflows fail, so `do_import` parses its output: a workflow rejected as above SG's managed Terraform ceiling (1.5.7, last MPL/FOSS release) is re-imported with `SGDefaultTerraformVersion` (read from `terraform.tfvars`; an explicit `null` there means the `terraformVersion` key is deleted instead, so the org's execution preset decides — `tfvars_is_null` tells an explicit null from a missing key), the payload patched in place, and the case logged to `export/terraform-version-fallbacks.log` plus a printed notice; any other per-workflow failure fails the run. `cmd_import` reads the org's execution preset (`sg_execution_preset`, `GET /orgs/{org}/` → `Settings.workflowDefaults`) and `preset_labels` (report.sh) turns it into the `preset (1.5.7)` / `preset (private:rg)` cells of the plan and the `SG_TF_FALLBACK_LABEL` used in messages. The trigger pass (`do_set_triggers` → `sg_set_vcs_triggers`) skips workflows that do not exist in SG, keeps the sha of each posted `{VCSConfig, VCSTriggers}` body in `triggers.<seg>.sha` and does not re-POST an unchanged, already-set workflow (`--fresh` does; the endpoint upserts anyway, a 4xx "already exists" from older builds counts as success), and `sg_api_post` returns 22 on 4xx so `sg_retry` (via `SG_NO_RETRY_RC`) does not retry definitive errors. `clean` removes local artifacts (`export/`, TF state, tool cache); `clean --all` also removes config. `completion bash|zsh` prints a completion script (`cmd_completion`; commands/options come from `SG_COMMANDS`/`SG_OPTIONS`, keep them in sync with the parser and the host flags); like `clean` it always runs natively. Output is concise by default — `apply` captures terraform's init/plan output and shows only progress + the final summary (or the full log on failure, `TF_IN_AUTOMATION=1`/`-no-color`); `-v`/`--verbose` (`SG_VERBOSE`) streams everything and un-gates the convert detail lines. Colored logging via `sg_step`/`sg_log`/`sg_success`/`sg_warn`/`sg_err`/`sg_row`; paths shown relative via `sg_rel` (all auto-off when not a TTY / `NO_COLOR`). Long steps show a live progress line (`sg_run_quiet` for terraform init/apply, `run_parallel <fn> <max> <label> <items>` for convert/import/triggers — animated only when `SG_ANIMATE=1`, i.e. colors are on; plain log lines otherwise) and report their duration (`sg_fmt_secs`). Phases start with `phase_begin` — inside `all` the header is numbered (`Phase 2/5: ...`, `PHASE_TOTAL` set in `main`) and `phase_took` gives the elapsed time; `run_parallel` prints a job's output as-is when it is a single line and adds the `── <file> ──` header only for longer or failed output (always with `-v`). The import confirmation goes through `sg_confirm` (so `SG_ANSWERS_FILE` works; a declined prompt is a neutral "cancelled" line, not an error) and `import`/`all` end with `finish_line` (outcome, total time, open checklist items from `CHECKLIST_OPEN`). A Ctrl-C is trapped to clear the progress line and print a resume hint. +- `scripts/lib/` — sourced libraries (a `lib/` gitignore rule is negated for this dir): `prompt.sh` (`sg_ask`/`sg_confirm`/`sg_select`; tty or `SG_ANSWERS_FILE` for tests; `SG_NONINTERACTIVE=1`/`-y`/no TTY → defaults), `tfvars.sh` (`tfvars_get` cached hcl2json reads — `tfvars_json` merges the run overlay (`TFVARS_OVERLAY_JSON`) over the file, and `tfvars_get`/`tfvars_get_json` keep a `false` value instead of treating it like a missing key —, `tfvars_has`/`tfvars_is_null` to tell an explicit `null` from a missing key, `tfvars_write` renders the wizard's `W_*` vars — `W_TF_SOURCE`, and `W_TF_VERSION`/`W_RUNNER_JSON` may be the literal `null`; plain strings go through `_tfvars_str`, so quotes/backslashes in names stay valid HCL; `projectOverrides`/`workspaceOverrides` are rendered from `W_PROJECT_OVERRIDES_JSON`/`W_WS_OVERRIDES_JSON` by `_tfvars_map` — `"key" = { attr = <json> }`, which hcl2json round-trips — so hand-written blocks survive a re-run of `init` (`wizard_run` reads them before writing; inner comments are lost); `tfWorkspaceIgnoreNames` is kept the same way (`W_IGNORE_NAMES_JSON`, never asked, shown in the review's scope row), and every other setting the wizard does not ask about (`tfProjects`, `cloudAuthVarPatterns`, hand-added keys) is appended verbatim from `W_PREV_JSON` under "Kept from the previous file"; `tfvars_missing_keys` (keys of `terraform.tfvars.example` the file lacks) and `tfvars_upgrade` (appends those paragraphs of the example — comment + default — under a dated header, `.bak` kept, touches nothing else; behind `init --upgrade`, prompt-free)), `scope.sh` (the run scope: `PROJECT_FILTER`/`WS_FILTER`/`WS_EXCLUDE`/`TAG_FILTER`/`TAG_EXCLUDE` arrays, `names_json`, `slug_of` (the transformer's project-slug rule in bash), `project_selected <seg>`, `ws_narrowed`/`ws_filter_json`/`ws_exclude_json` (tfvars `tfWorkspaceIgnoreNames` ∪ flags, memoized)/`ws_selected <name>` (bash `case` globs), `WS_SCOPE_JQ` + `ws_jq_args` → `WS_JQ_ARGS` for the jq sites, `scope_tags_json`/`scope_ignore_tags_json`, `scope_tfvar_args` → `SCOPE_TFVAR_ARGS` for apply, `scope_describe`, `scope_sha_input` for the apply phase hash; and the **run configuration overlay**: `--set KEY=VALUE` (`SET_VARS`, values via `_overlay_value`: HCL/JSON literals through hcl2json, bare text stays a string), `--cloud-connector`/`--vcs-connector`/`--runner-group`/`--workflow-group` → `overlay_build` looks the connector kinds up with `sg_list_integrations`/`sg_integration_type`/`sg_vcs_kind_of`, resolves `--project` values to raw TFC names (`_overlay_project_name`, fatal for an unknown project, as-given with a warning when TFC is unreachable) and fills `TFVARS_OVERLAY_JSON` — `projectOverrides[<name>]` fields with `--project`, `SGDefault*` keys without; `tfvars_json` deep-merges it over the file so every reader sees it, `scope_tfvar_args` passes each overlay key as `-var`, `scope_sha_input` includes it, `write_run_result` records it under `configuration`; `main` refuses the flags for `init`/`clean`/`completion` and a standalone `import` warns that they only reach the workflows through the export (`RAN_APPLY`)), `tfc_api.sh` (`tfc_http`/`tfc_get_all` paginated, `tfc_list_{orgs,projects,workspaces}` — workspaces include `vcs_provider`/`vcs_url`/`vcs_identifier` —, `tfc_select_workspaces` (the module's name-glob / include-tags(all) / exclude-tags / exclude-name-globs filter, shared by wizard and preflight), `tfc_vcs_summary`/`tfc_vcs_kind_for`/`tfc_vcs_label_for`, `tfc_token` with provider precedence, `require_tfc_auth`), `sg_api.sh` (`_sg_api` 0/22/1 contract + `SG_HTTP_CODE`, `_sg_listall` paginated listall reader (the API pages at 50) behind `sg_list_workflows`/`sg_list_wfgrps`, `sg_set_vcs_triggers`, `sg_list_integrations`, `sg_integration_type`, `sg_vcs_kind_of` (connector type → `sourceConfigDestKind`), `sg_execution_preset` + `sg_preset_{desc,runner_desc,version_desc,runner_provided}` (the org's execution preset and how to describe it), `sg_*_exists`, `sg_list_workflows`, `sg_create_secret`, `sg_patch_workflow`, `wfgroup_*`), `wizard.sh` (`wizard_run`: 5 steps TFC → SG → projects → defaults → review; `wizard_projects` (with >1 selected project, from `W_SEL_PROJECTS_JSON`) asks whether the global connectors and `tfc-<project>` groups apply to all, else per project the cloud connector, VCS connector (kind + prefix follow) and group (default / existing via `sg_list_wfgrps` / typed), writing `projectOverrides`; "same as the default" deletes the key, unmanaged fields are kept, previous picks come first (`_w_first`); the defaults step also asks `stripCloudAuthVars`; the TFC step keeps the workspace list and derives, via `tfc_vcs_summary`, which VCS provider the selected workspaces use and the repositories' URL prefix — the SG step lists matching connectors first and takes the repo URL prefix from TFC (never from a previous tfvars written for another provider kind), warns on a kind mismatch, reads the org's execution preset and offers it as the runner choice; the defaults step asks whether to carry TFC versions or use the preset (`SGTerraformVersionSource`) and, for carry, which fallback (`SGDefaultTerraformVersion` or the preset = null); the review shows human-readable rows plus the `tfc-<project>` groups; `_w_select_lines` feeds menus line-by-line so names with spaces/globs survive), `preflight.sh` (`preflight_run <apply|import|all>`, once per process; ✓ lines are plain language and name the connector's role/kind, ✗/! lines name the tfvars field to fix; connector ids, runner groups and DPC kinds are collected from `SGDefault*`, `projectOverrides` and `workspaceOverrides`; besides existence it checks consistency — VCS kind vs. the connector's type, globally and per `projectOverrides` entry (fail), repo URL prefix vs. where the TFC repositories live or the kind's well-known host (warn), `projectOverrides` keys vs. the TFC project names (warn), `--project` values vs. the project names/slugs (fail; a `tfProjects` typo warns), and the workspace-selection preview applies the CLI scope (names, excludes, tags, projects) so its count is what the apply will export; the import context states the group policy and warns when a project's group changed since its last import; it also reports the org's execution preset next to the version/runner policy and warns when `carry` meets a preset that mounts a runner-provided binary), `state.sh` (`.sg/state.json`: `phases.<name>.input_sha`, `import.<seg>.{group,imported,updated,failed,tf_fallback,state_uploaded,state_failed}` (`group` anchors the never-move check; `updated` = PATCHed on a 409), `triggers.<seg>.{set,unchanged,failed,missing,sha}`, `secrets.<name>`; `run_phase` in migrate.sh skips unchanged phases), `report.sh` (`show_migration_summary` after apply — incl. a state-files row —, `show_import_plan` per-workflow table with columns sized to their content — it keeps the rows in `PLAN_ROWS_TSV` (name, group, action, version, runner, triggers, vars, secrets, segment, workspace, project) —, `write_run_result <outcome>` → `export/run-result.json` + `run-summary.md` from `PLAN_ROWS_TSV`, `PLAN_GROUP_ROWS` (plain copy of the group table, set by `plan_groups`), `PLAN_PROBLEMS`, the merged state and `CHECKLIST_OPEN`; per workflow `result` is planned/skipped/failed/updated/created, with the probe workflow — planned `create`, PATCHed on its second pass — reported as created), `errors.sh` (`explain_api_error` regex→hint table, `SG_API_HINTS`), `checklist.sh` (`create_secret_stubs` → SG secret `tfc-<wf>-<VAR>` = `CHANGE_ME`, referenced as `${secret::<name>}` via PATCH + payload patch; `write_checklist` → `export/post-import-checklist.md`, printed to the terminal as one ✓/! status line per section plus the imported groups with UI links — the file has the details; section 5 covers both the TFC state export and the SG state upload (`state_failed` from the import results); sets `CHECKLIST_OPEN`). Parallel jobs (`run_parallel`) run in subshells, so `do_import`/`do_set_triggers` write `.import-result.<seg>.json` / `.triggers-result.<seg>.json` into the export dir and the caller merges them into state. - `scripts/tools.sh` — sourced by the other scripts (sets `SG_REPO_ROOT` to its parent dir). Provides `sg_resolve` (PATH-or-cache), `sg_ensure_{jq,hcl2json,yajsv,sgcli}` (pinned downloads into `.sg/cached/`; `sg-cli` is the Go binary), `sg_retry`, the color vars + log helpers. The Docker image bundles these tools at build time (`yajsv` built from source so arm64 works), so in-container runs never download. - **Variable sets** — `scripts/enrich_variable_sets.sh <tfc-org> <payload...>` (run by `cmd_enrich`, which reads `tfOrg`/`tfHostname` from `terraform.tfvars` via `hcl2json` — variable sets live in the **TFC** org, not `SG_ORG`). Lists all sets + vars via the TFC API (Bearer token from the `terraform login` creds file or `TFE_TOKEN`), computes per-workspace effective vars (scope: global/project/workspace; precedence: priority sets > workspace vars > non-priority sets), and merges them into each workflow (matched by workspace name parsed from `TfStateFilePath`). Sensitive set vars and key conflicts are reported. Skipped by `--no-variable-sets`. - **Workflow groups** — each TFC project maps to the SG workflow group the transformer wrote into its payload (`projectOverrides[<project>].workflowGroup`, default `tfc-<project-segment>`); an existing group is reused, a missing one created via `POST /api/v1/orgs/{org}/wfgrps/` (auth `Authorization: apikey <token>`) unless `--no-create-groups`. (Both the API calls and sg-cli honor the undocumented `SG_BASE_URL` for non-prod targets.) Groups are addressed by name, so no generated ID is tracked, and they are never PATCHed; workflows are never moved between groups (see `plan_groups`). `.sg/workflow-groups.json` (gitignored) is a deprecated per-segment override that still applies on top, with a warning. diff --git a/README.md b/README.md index 03cfcd3..ca6731b 100644 --- a/README.md +++ b/README.md @@ -34,6 +34,7 @@ That's it — no IDs to look up and no workflow-group mapping to fill in. `init` - Single phase: `./sg-migrate.sh preflight|apply|enrich|convert|validate|import|triggers|checklist`. Running `./sg-migrate.sh` with no command prints the help menu. - **Resume.** `all` remembers what it completed (`.sg/state.json`) and skips phases whose inputs have not changed, so after a failure you just re-run it; files already imported in full are skipped and files with failures are retried. `--fresh` redoes everything. - **Scope.** `terraform.tfvars` holds the widest selection (`workspacenames`, `tfWorkspaceTags`/`tfWorkspaceIgnoreTags`, `tfWorkspaceIgnoreNames`, `tfProjects`); flags narrow one run and every phase applies the same selection: `--project <name or slug>` (a TFC project, exported and imported on its own), `--workspace <glob>` (`team-*`, `*` = all; replaces `workspacenames`), `--exclude-workspace <glob>` (adds to `tfWorkspaceIgnoreNames`), `--tag <name>` / `--exclude-tag <name>` (export only). All repeatable — migrate one team first, then the rest, or drive the selection from a CI trigger. +- **Inline connectors and settings.** For a run driven from a pipeline, the connectors can be passed instead of edited into the tfvars: `--cloud-connector <id>`, `--vcs-connector <id>`, `--runner-group <name>|shared`, `--workflow-group <name>` and the generic `--set KEY=VALUE` (any transformer variable, HCL or JSON value). Only the connector id is typed; its kind (`AWS_RBAC`, `AZURE_OIDC`, `GITHUB_COM`, ...) is looked up in the org, and the VCS kind and repo URL prefix follow the connector. With `--project` they become that project's `projectOverrides` entry for the run, so every workspace of the project inherits them; without it they set the `SGDefault*` values. They are applied on top of `terraform.tfvars` for that run only and reach the workflows through the export (`all`, or `apply` then `import`); a later run without them PATCHes the workflows back to the tfvars values. `./sg-migrate.sh -y all --project Payments --workspace '*' --workflow-group payments-prod --cloud-connector aws-payments --vcs-connector github_payments` migrates one project into one group with one set of connectors. - **Dry run.** `./sg-migrate.sh import --dry-run` (or `all --dry-run`, which still runs the local export) prints the per-workflow plan and stops; nothing is created or changed in StackGuardian. The plan is also written to `export/run-result.json` and `export/run-summary.md`. - **Sensitive variables** (which TFC never exposes) are recreated as SG secrets with the value `CHANGE_ME` and referenced from the workflows as `${secret::<name>}`; the checklist lists each one to fill in. Opt out with `--no-secret-stubs`. - TFC **Variable Set** variables are merged into the payloads automatically (the `enrich` phase, via the TFC API); skip it with `--no-variable-sets`. @@ -43,7 +44,7 @@ That's it — no IDs to look up and no workflow-group mapping to fill in. `init` - Output is concise by default (terraform's plan/init noise is hidden; shown on error). Add `-v`/`--verbose` for full output. Known API errors come with a hint naming the `terraform.tfvars` field to fix. - Flags: `-y` skip the import prompt (required without a terminal, e.g. in CI), `--tfvars FILE` use a tfvars file kept elsewhere, `--concurrency N` parallel jobs, `--org NAME`, `--no-create-groups` require groups to pre-exist, `--skip-preflight`, `--build` rebuild the image, `--native`/`--local` force a local run even when Docker is available, `--mapping FILE` (deprecated group override map, see *Per-project settings*). - Tuning via env: `SG_RETRIES`, `SG_TF_PARALLELISM`, `SG_NATIVE=1`, `SG_TFVARS` (same as `--tfvars`), `SG_UI_URL` (base URL for the checklist's links, default `https://app.stackguardian.io`). -- **Running in CI.** Without a terminal every prompt takes its default: `init` cannot run the wizard (it only copies the example file), so generate `terraform.tfvars` once with `init` on a workstation, keep it next to the pipeline (it is gitignored here) and pass it with `--tfvars`; `import`/`all` need `-y`. Pick the scope per trigger with the flags above, run `all --dry-run` on a pull request and `-y all` on merge, and publish `export/run-summary.md` (e.g. `cat export/run-summary.md >> "$GITHUB_STEP_SUMMARY"`); `export/run-result.json` has the same data for scripts. Cache `.sg/` and `export/` between runs to keep the resume logic; without them every run re-exports and updates every workflow, which is correct but slower. `export/states/` holds real Terraform state — do not publish it as an artifact. +- **Running in CI.** Without a terminal every prompt takes its default: `init` cannot run the wizard (it only copies the example file), so generate `terraform.tfvars` once with `init` on a workstation, keep it next to the pipeline (it is gitignored here) and pass it with `--tfvars`; `import`/`all` need `-y`. Pick the scope per trigger with the flags above and the connectors with `--cloud-connector`/`--vcs-connector`/`--workflow-group` when they are trigger inputs too, run `all --dry-run` on a pull request and `-y all` on merge, and publish `export/run-summary.md` (e.g. `cat export/run-summary.md >> "$GITHUB_STEP_SUMMARY"`); `export/run-result.json` has the same data for scripts. Cache `.sg/` and `export/` between runs to keep the resume logic; without them every run re-exports and updates every workflow, which is correct but slower. `export/states/` holds real Terraform state — do not publish it as an artifact. - Tab completion for the current shell session: `source <(./sg-migrate.sh completion)` (bash/zsh detected; or pass `bash`/`zsh`). `init` prints this line. - TFC auth: set `TFE_TOKEN` (recommended — a long-lived token avoids re-running `terraform login`); otherwise the `terraform login` credentials file is mounted read-only into the container. Tokens are only ever read from the environment; `init` never writes them to disk. diff --git a/scripts/lib/report.sh b/scripts/lib/report.sh index c72cbf4..710685c 100644 --- a/scripts/lib/report.sh +++ b/scripts/lib/report.sh @@ -192,11 +192,12 @@ write_run_result() { state_read | "$JQ_BIN" --arg cmd "$PROG ${SG_RUN_ARGS:-}" --arg at "$(state_now)" --argjson took "$((SECONDS - RUN_T0))" \ --arg org "$ORG" --arg url "$SG_BASE_URL" --argjson dry "$([ "${DRY_RUN:-0}" -eq 1 ] && echo true || echo false)" \ --arg outcome "$outcome" --argjson scope "$scope" --argjson groups "${PLAN_GROUP_ROWS:-[]}" \ + --argjson overlay "${TFVARS_OVERLAY_JSON:-{\}}" --arg overlay_desc "$(declare -F overlay_describe >/dev/null && overlay_describe || true)" \ --argjson problems "$problems" --argjson rows "$rows" --argjson open "${CHECKLIST_OPEN:-0}" ' . as $st | { command: $cmd, at: $at, tookSeconds: $took, org: $org, apiUrl: $url, dryRun: $dry, outcome: $outcome, - scope: $scope, groups: $groups, problems: $problems, + scope: $scope, configuration: {description: $overlay_desc, overlay: $overlay}, groups: $groups, problems: $problems, workflows: [ $rows[] | . as $r | ($st.import[$r.segment] // {}) as $imp | ($st.triggers[$r.segment] // {}) as $tr | . + { @@ -224,6 +225,7 @@ write_run_result() { "- Org: `\(.org)` (\(.apiUrl))", "- Command: `\(.command)`", "- Finished: \(.at) after \(.tookSeconds)s", (if .dryRun then "- Dry run: nothing was created or changed in StackGuardian" else empty end), (if (.scope | [.[]] | add | length) > 0 then "- Scope: \(.scope | to_entries | map(select(.value | length > 0) | "\(.key) \(.value | join(", "))") | join("; "))" else empty end), + (if (.configuration.description // "") != "" then "- Run configuration: \(.configuration.description)" else empty end), "", "| Workflow | Group | Plan | Result | Terraform | State | Triggers |", "|---|---|---|---|---|---|---|", (.workflows[] | "| \(.name) | \(.group) | \(.plan) | \(.result) | \(.terraformVersion)\(if .tfFallback then " (fallback)" else "" end) | \(.state // "-") | \(.triggers // "-") |"), diff --git a/scripts/lib/scope.sh b/scripts/lib/scope.sh index cc96072..7d7ecc8 100644 --- a/scripts/lib/scope.sh +++ b/scripts/lib/scope.sh @@ -131,8 +131,14 @@ ws_jq_args() { WS_JQ_ARGS=(--argjson inc "$(ws_filter_json)" --argjson exc "$(ws # CLI scope to terraform apply (nothing when no flag is set). SCOPE_TFVAR_ARGS=() scope_tfvar_args() { + local k SCOPE_TFVAR_ARGS=() _scope_jq + # The configuration overlay first (merged values, see overlay_build); a + # scope flag on the same variable wins because terraform takes the last -var. + while IFS= read -r k; do + [ -n "$k" ] && SCOPE_TFVAR_ARGS+=(-var "$k=$(tfvars_get_json ".$k")") + done < <(printf '%s' "${TFVARS_OVERLAY_JSON:-{\}}" | "$JQ_BIN" -r 'keys[]') [ "${#PROJECT_FILTER[@]}" -gt 0 ] && SCOPE_TFVAR_ARGS+=(-var "tfProjects=$(names_json "${PROJECT_FILTER[@]}")") [ "${#WS_FILTER[@]}" -gt 0 ] && SCOPE_TFVAR_ARGS+=(-var "workspacenames=$(names_json "${WS_FILTER[@]}")") [ "${#WS_EXCLUDE[@]}" -gt 0 ] && SCOPE_TFVAR_ARGS+=(-var "tfWorkspaceIgnoreNames=$(ws_exclude_json)") @@ -154,4 +160,142 @@ scope_describe() { # scope_sha_input — the CLI scope as a string for the apply phase hash, so a # run with a different selection re-runs the export. -scope_sha_input() { printf '%s|%s|%s|%s|%s' "${PROJECT_FILTER[*]-}" "$(ws_filter_json)" "$(ws_exclude_json)" "$(scope_tags_json)" "$(scope_ignore_tags_json)"; } +scope_sha_input() { printf '%s|%s|%s|%s|%s|%s' "${PROJECT_FILTER[*]-}" "$(ws_filter_json)" "$(ws_exclude_json)" "$(scope_tags_json)" "$(scope_ignore_tags_json)" "${TFVARS_OVERLAY_JSON:-}"; } + +# --- run configuration overlay ------------------------------------------------ +# Connector and setting flags apply to one run on top of terraform.tfvars: +# --set KEY=VALUE any transformer variable (HCL/JSON value; bare text = string) +# --cloud-connector ID DeploymentPlatformConfig; the kind is looked up in SG +# --vcs-connector ID vcsAuthIntegrationID + sourceConfigDestKind (looked up) + repo prefix +# --runner-group NAME|shared RunnerConstraints +# --workflow-group NAME the project's workflow group (needs --project) +# With --project the connector flags become that project's projectOverrides +# entry, so every workspace of the project inherits them; without it they set +# the SGDefault* values. tfvars_json returns the file merged with +# TFVARS_OVERLAY_JSON, so preflight, the plan, enrich and the import see the +# same values, and apply receives them as -var. Nothing is written to the +# tfvars: a later run without the flags PATCHes the workflows back to it. +SET_VARS=() +CLOUD_CONNECTOR="" +VCS_CONNECTOR="" +RUNNER_GROUP="" +WORKFLOW_GROUP="" +TFVARS_OVERLAY_JSON='{}' + +overlay_requested() { [ "${#SET_VARS[@]}" -gt 0 ] || [ -n "$CLOUD_CONNECTOR$VCS_CONNECTOR$RUNNER_GROUP$WORKFLOW_GROUP" ]; } + +# _overlay_value <text> — a --set value as JSON. Literals ([..], {..}, "..", +# true/false/null, numbers) go through hcl2json; anything else is a string, so +# /integrations/x or aws-prod never turn into HCL expressions. +_overlay_value() { + case "$1" in + \[* | \{* | \"* | true | false | null) + printf 'x = %s\n' "$1" | "$(sg_resolve hcl2json sg_ensure_hcl2json)" 2>/dev/null | "$JQ_BIN" -c '.x' + return + ;; + esac + case "$1" in + '' | *[!0-9.-]*) "$JQ_BIN" -cn --arg v "$1" '$v' ;; + *) printf '%s' "$1" ;; + esac +} + +# _overlay_project_name <value> — the raw TFC project name for a --project +# value (name or slug), via the TFC API: projectOverrides is keyed by the +# name. A value matching no project is fatal; when TFC cannot be reached the +# value is used as given, with a warning (fine when it is the exact name). +_overlay_project_name() { + local slug hit + _scope_jq + slug="$(slug_of "$1")" + if [ -z "${_OVERLAY_PROJECTS+x}" ]; then + _OVERLAY_PROJECTS="$(tfc_list_projects "$(tfvars_get .tfOrg)" 2>/dev/null)" || _OVERLAY_PROJECTS="" + fi + if [ -z "$_OVERLAY_PROJECTS" ]; then + sg_warn "could not list the TFC projects to resolve --project '$1' — using it as the project name" + printf '%s' "$1" + return + fi + hit="$(printf '%s' "$_OVERLAY_PROJECTS" | "$JQ_BIN" -r --arg s "$slug" '[.[] | select((.name | ascii_downcase | gsub("[^a-z0-9-]+"; "-")) == $s) | .name] | first // empty')" + [ -n "$hit" ] || die "--project '$1' matches no TFC project in '$(tfvars_get .tfOrg)' (projects: $(printf '%s' "$_OVERLAY_PROJECTS" | "$JQ_BIN" -r '[.[].name] | join(", ")'))" + printf '%s' "$hit" +} + +# overlay_build — turn the flags into TFVARS_OVERLAY_JSON (dies on a bad flag). +# Needs ORG/SG_API_TOKEN for the connector lookups and TFC auth to resolve a +# project slug (falls back to the value as given). +overlay_build() { + overlay_requested || return 0 + _scope_jq + local o='{}' kv k v ints ctype kind fields='{}' p name rc + for kv in ${SET_VARS[@]+"${SET_VARS[@]}"}; do + k="${kv%%=*}" + v="${kv#*=}" + { [ "$k" != "$kv" ] && [ -n "$k" ]; } || die "--set expects KEY=VALUE, got '$kv'" + tfvars_variable_names | grep -qx -- "$k" || die "--set: '$k' is not a setting of the transformer (see transformer/terraform-cloud/variables.tf)" + v="$(_overlay_value "$v")" + [ -n "$v" ] || die "--set $k: the value is not valid HCL/JSON" + o="$("$JQ_BIN" -c --arg k "$k" --argjson v "$v" '.[$k] = $v' <<<"$o")" + done + if [ -n "$CLOUD_CONNECTOR$VCS_CONNECTOR" ]; then + { [ -n "${SG_API_TOKEN:-}" ] && [ -n "${ORG:-}" ]; } || die "--cloud-connector / --vcs-connector need SG_API_TOKEN and SG_ORG (the connector kind is looked up in the org)" + ints="$(sg_list_integrations)" || die "could not list the connectors of org '$ORG' (HTTP ${SG_HTTP_CODE:-?}) — check SG_API_TOKEN" + fi + if [ -n "$CLOUD_CONNECTOR" ]; then + ctype="$(sg_integration_type "$ints" "$CLOUD_CONNECTOR")" + [ -n "$ctype" ] || die "cloud connector '$CLOUD_CONNECTOR' not found in org '$ORG' (available: $(printf '%s' "$ints" | "$JQ_BIN" -r '[.[] | select(.type | test("^(AWS|AZURE|GCP)_")) | .name] | join(", ")'))" + case "$ctype" in AWS_* | AZURE_* | GCP_*) ;; *) die "'$CLOUD_CONNECTOR' is a $ctype connector, not a cloud connector" ;; esac + fields="$("$JQ_BIN" -c --arg k "$ctype" --arg i "/integrations/${CLOUD_CONNECTOR#/integrations/}" '.DeploymentPlatformConfig = [{kind: $k, config: {integrationId: $i}}]' <<<"$fields")" + OVERLAY_CLOUD_KIND="$ctype" + fi + if [ -n "$VCS_CONNECTOR" ]; then + ctype="$(sg_integration_type "$ints" "$VCS_CONNECTOR")" + [ -n "$ctype" ] || die "VCS connector '$VCS_CONNECTOR' not found in org '$ORG' (available: $(printf '%s' "$ints" | "$JQ_BIN" -r '[.[] | select(.type | test("^(AWS|AZURE|GCP)_") | not) | .name] | join(", ")'))" + kind="$(sg_vcs_kind_of "$ctype")" + [ -n "$kind" ] || die "'$VCS_CONNECTOR' is a $ctype connector; the migrator cannot map that to a VCS kind — pass --set SGDefaultSourceConfigDestKind=<GITHUB_COM|GITLAB_COM|BITBUCKET_ORG|AZURE_DEVOPS|GIT_OTHER> as well" + fields="$("$JQ_BIN" -c --arg i "/integrations/${VCS_CONNECTOR#/integrations/}" --arg k "$kind" '.vcsAuthIntegrationID = $i | .sourceConfigDestKind = $k' <<<"$fields")" + # A different provider than the tfvars default needs its own repo prefix. + if [ "$kind" != "$(tfvars_get .SGDefaultSourceConfigDestKind)" ]; then + fields="$("$JQ_BIN" -c --arg p "$(_w_repo_prefix_for "$kind")" '.vcsRepoPrefix = $p' <<<"$fields")" + fi + OVERLAY_VCS_KIND="$kind" + fi + if [ -n "$RUNNER_GROUP" ]; then + if [ "$RUNNER_GROUP" = "shared" ]; then rc='{"type":"shared"}'; else rc="$("$JQ_BIN" -cn --arg n "$RUNNER_GROUP" '{type: "private", names: [$n]}')"; fi + fields="$("$JQ_BIN" -c --argjson r "$rc" '.RunnerConstraints = $r' <<<"$fields")" + fi + if [ -n "$WORKFLOW_GROUP" ]; then + [ "${#PROJECT_FILTER[@]}" -gt 0 ] || die "--workflow-group needs --project: a workflow group belongs to a TFC project" + fields="$("$JQ_BIN" -c --arg g "$WORKFLOW_GROUP" '.workflowGroup = $g' <<<"$fields")" + fi + if [ "$fields" != "{}" ]; then + if [ "${#PROJECT_FILTER[@]}" -gt 0 ]; then + for p in "${PROJECT_FILTER[@]}"; do + name="$(_overlay_project_name "$p")" + o="$("$JQ_BIN" -c --arg n "$name" --argjson f "$fields" '.projectOverrides[$n] = ((.projectOverrides[$n] // {}) + $f)' <<<"$o")" + done + else + o="$("$JQ_BIN" -c --argjson f "$fields" '. + ($f | with_entries(.key |= ({ + DeploymentPlatformConfig: "SGDefaultDeploymentPlatformConfig", vcsAuthIntegrationID: "SGDefaultVCSAuthIntegrationID", + sourceConfigDestKind: "SGDefaultSourceConfigDestKind", vcsRepoPrefix: "SGDefaultIACVCSRepoPrefix", + RunnerConstraints: "SGDefaultRunnerConstraints"}[.])))' <<<"$o")" + fi + fi + TFVARS_OVERLAY_JSON="$o" + tfvars_invalidate + sg_log "run configuration: $(overlay_describe)" +} + +# overlay_describe — one line for the log and the run result. +overlay_describe() { + local out="" kv + [ -n "$CLOUD_CONNECTOR" ] && out="cloud connector ${CLOUD_CONNECTOR#/integrations/} (${OVERLAY_CLOUD_KIND:-?})" + [ -n "$VCS_CONNECTOR" ] && out="${out:+$out, }VCS connector ${VCS_CONNECTOR#/integrations/} (${OVERLAY_VCS_KIND:-?})" + [ -n "$RUNNER_GROUP" ] && out="${out:+$out, }runners $RUNNER_GROUP" + [ -n "$WORKFLOW_GROUP" ] && out="${out:+$out, }workflow group $WORKFLOW_GROUP" + if [ -n "$out" ]; then + if [ "${#PROJECT_FILTER[@]}" -gt 0 ]; then out="$out for project(s) ${PROJECT_FILTER[*]}"; else out="$out as the defaults"; fi + fi + for kv in ${SET_VARS[@]+"${SET_VARS[@]}"}; do out="${out:+$out, }$kv"; done + printf '%s' "$out" +} diff --git a/scripts/lib/tfvars.sh b/scripts/lib/tfvars.sh index 5b41db8..6480a74 100644 --- a/scripts/lib/tfvars.sh +++ b/scripts/lib/tfvars.sh @@ -6,15 +6,23 @@ _TFVARS_JSON="" _TFVARS_JSON_FOR="" -# tfvars_json — the whole tfvars file as JSON (empty object when missing). +# tfvars_json — the whole tfvars file as JSON (empty object when missing), +# merged with the run's configuration overlay (TFVARS_OVERLAY_JSON from +# lib/scope.sh: --set / --cloud-connector / ...), so every reader sees the +# values the run actually uses. Objects merge deeply, so an overlay entry for +# one project keeps the other projectOverrides of the file. tfvars_json() { - if [ -z "$_TFVARS_JSON" ] || [ "$_TFVARS_JSON_FOR" != "$TFVARS" ]; then + local overlay="${TFVARS_OVERLAY_JSON:-{\}}" + if [ -z "$_TFVARS_JSON" ] || [ "$_TFVARS_JSON_FOR" != "$TFVARS|$overlay" ]; then if [ -f "$TFVARS" ]; then _TFVARS_JSON="$("$(sg_resolve hcl2json sg_ensure_hcl2json)" "$TFVARS" 2>/dev/null || echo '{}')" else _TFVARS_JSON='{}' fi - _TFVARS_JSON_FOR="$TFVARS" + if [ "$overlay" != "{}" ]; then + _TFVARS_JSON="$(printf '%s' "$_TFVARS_JSON" | "$(sg_resolve jq sg_ensure_jq)" -c --argjson o "$overlay" '. * $o')" + fi + _TFVARS_JSON_FOR="$TFVARS|$overlay" fi printf '%s' "$_TFVARS_JSON" } @@ -23,13 +31,15 @@ tfvars_json() { # JSON, e.g. tfvars_get '.tfOrg'. Prints the default when null/missing. tfvars_get() { local expr="$1" def="${2-}" v - v="$(tfvars_json | "$(sg_resolve jq sg_ensure_jq)" -r "$expr // empty" 2>/dev/null || true)" + # Not `// empty`: jq's // also swallows false, and false is a real value here + # (stripCloudAuthVars = false, exportStateFiles = false). + v="$(tfvars_json | "$(sg_resolve jq sg_ensure_jq)" -r "[$expr][0] | if . == false then \"false\" else (. // empty) end" 2>/dev/null || true)" printf '%s' "${v:-$def}" } # tfvars_get_json <jq-expr> — compact JSON value of an expression (or null). tfvars_get_json() { - tfvars_json | "$(sg_resolve jq sg_ensure_jq)" -c "$1 // null" 2>/dev/null || echo null + tfvars_json | "$(sg_resolve jq sg_ensure_jq)" -c "[$1][0] | if . == false then false else (. // null) end" 2>/dev/null || echo null } # tfvars_has <key> — exit 0 when the top-level key is present in the file (an diff --git a/scripts/migrate.sh b/scripts/migrate.sh index db2baa8..e8cb28c 100755 --- a/scripts/migrate.sh +++ b/scripts/migrate.sh @@ -54,6 +54,9 @@ SG_BASE_URL="${SG_BASE_URL:-https://api.app.stackguardian.io}" ASSUME_YES=0 PURGE=0 UPGRADE=0 +# Set once the export ran (or was found up to date) in this process: the run +# configuration flags reach the workflows through the payload the export writes. +RAN_APPLY=0 CREATE_GROUPS=1 ENRICH_VARSETS=1 VCS_TRIGGERS=1 @@ -146,6 +149,15 @@ Options: --exclude-tag NAME Leave workspaces carrying the tag out (repeatable; adds to tfWorkspaceIgnoreTags) --all With 'clean': also remove config (terraform.tfvars, mapping, .sg) --upgrade With 'init': append missing settings to an existing terraform.tfvars + +Run configuration (on top of terraform.tfvars, for this run only; with --project +they apply to that project's workflows, otherwise as the defaults): + --cloud-connector ID Cloud connector (/integrations/<name>); its kind is looked up in SG + --vcs-connector ID VCS connector; kind and repo URL prefix follow the connector + --runner-group NAME Private runner group for the workflows ("shared" = SG runners) + --workflow-group NAME Workflow group for the project's workflows (needs --project) + --set KEY=VALUE Any transformer variable, e.g. --set SGDefaultTerraformVersion=null + (repeatable; HCL/JSON value, bare text is a string) -v, --verbose Show full terraform/tool output (default: concise) -y, --yes Skip the import confirmation prompt -h, --help Show this help @@ -603,6 +615,7 @@ tf_apply() { cd "$TRANSFORMER_DIR" && export PATH="$TF_PATH" TF_IN_AUTOMATION=1 cmd_apply() { phase_begin "apply (terraform)" + RAN_APPLY=1 command -v terraform >/dev/null 2>&1 || die "terraform not found on PATH" [ -f "$TFVARS" ] || die "Missing $(sg_rel "$TFVARS"). Run: $PROG init" # terraform auto-loads terraform.tfvars from the module dir on top of -var-file. @@ -1001,6 +1014,9 @@ cmd_import() { export SG_API_TOKEN SG_BASE_URL JQ_BIN="$(sg_resolve jq sg_ensure_jq)" preflight_run import + if overlay_requested && [ "$RAN_APPLY" -ne 1 ]; then + sg_warn "the run configuration flags shape the export and were not applied to the existing payload files — run '$PROG all' (or 'apply', then 'import') for them to reach the workflows" + fi payload_files [ "${#PF[@]}" -gt 0 ] || die "No payload files in $(sg_rel "$EXPORT_DIR")." @@ -1120,7 +1136,7 @@ finish_line() { # Single source of truth for shell completion (keep in sync with the parser below # and the host-only flags in sg-migrate.sh). SG_COMMANDS="init preflight apply enrich convert validate import triggers checklist all clean completion update" -SG_OPTIONS="--org --export-dir --tfvars --mapping --concurrency --no-create-groups --no-variable-sets --no-vcs-triggers --skip-preflight --dry-run --no-secret-stubs --fresh --project --workspace --exclude-workspace --tag --exclude-tag --all --upgrade -v --verbose -y --yes -h --help --native --local --build" +SG_OPTIONS="--org --export-dir --tfvars --mapping --concurrency --no-create-groups --no-variable-sets --no-vcs-triggers --skip-preflight --dry-run --no-secret-stubs --fresh --project --workspace --exclude-workspace --tag --exclude-tag --all --upgrade --set --cloud-connector --vcs-connector --runner-group --workflow-group -v --verbose -y --yes -h --help --native --local --build" # cmd_completion <bash|zsh> — print a completion script for sg-migrate.sh / # migrate.sh to stdout. Both shells fall back to the basename when the command @@ -1144,7 +1160,7 @@ _sg_migrate() { case "\$prev" in --export-dir) COMPREPLY=(\$(compgen -d -- "\$cur")); return ;; --mapping | --tfvars) COMPREPLY=(\$(compgen -f -- "\$cur")); return ;; - --org | --concurrency | --project | --workspace | --exclude-workspace | --tag | --exclude-tag) COMPREPLY=(); return ;; + --org | --concurrency | --project | --workspace | --exclude-workspace | --tag | --exclude-tag | --set | --cloud-connector | --vcs-connector | --runner-group | --workflow-group) COMPREPLY=(); return ;; completion) COMPREPLY=(\$(compgen -W "bash zsh" -- "\$cur")); return ;; esac for w in "\${COMP_WORDS[@]:1:COMP_CWORD-1}"; do @@ -1204,6 +1220,11 @@ _sg_migrate() { '*--exclude-tag[Leave workspaces carrying this tag out]:tag' \\ '--all[With clean: also remove config]' \\ '--upgrade[With init: append missing settings to terraform.tfvars]' \\ + '*--set[Transformer variable for this run (KEY=VALUE)]:setting' \\ + '--cloud-connector[Cloud connector for this run (kind looked up in SG)]:id' \\ + '--vcs-connector[VCS connector for this run (kind looked up in SG)]:id' \\ + '--runner-group[Private runner group for this run (shared = SG runners)]:name' \\ + '--workflow-group[Workflow group for the --project workflows]:name' \\ '(-v --verbose)'{-v,--verbose}'[Show full terraform/tool output]' \\ '(-y --yes)'{-y,--yes}'[Skip the import confirmation prompt]' \\ '(-h --help)'{-h,--help}'[Show help]' \\ @@ -1294,6 +1315,31 @@ main() { -v | --verbose) VERBOSE=1 ;; --all) PURGE=1 ;; --upgrade) UPGRADE=1 ;; + --set) + SET_VARS+=("$2") + shift + ;; + --set=*) SET_VARS+=("${1#*=}") ;; + --cloud-connector) + CLOUD_CONNECTOR="$2" + shift + ;; + --cloud-connector=*) CLOUD_CONNECTOR="${1#*=}" ;; + --vcs-connector) + VCS_CONNECTOR="$2" + shift + ;; + --vcs-connector=*) VCS_CONNECTOR="${1#*=}" ;; + --runner-group) + RUNNER_GROUP="$2" + shift + ;; + --runner-group=*) RUNNER_GROUP="${1#*=}" ;; + --workflow-group) + WORKFLOW_GROUP="$2" + shift + ;; + --workflow-group=*) WORKFLOW_GROUP="${1#*=}" ;; -h | --help) usage exit 0 @@ -1323,6 +1369,17 @@ main() { fi export SG_VERBOSE="$VERBOSE" + # The run configuration flags (lib/scope.sh) shape a run, not the file. + case "$CMD" in + init | clean | completion) + overlay_requested && die "--set / --cloud-connector / --vcs-connector / --runner-group / --workflow-group apply to a run (apply, import, all), not to '$CMD' — edit $(sg_rel "$TFVARS") instead" + ;; + *) + export SG_API_TOKEN SG_BASE_URL + overlay_build + ;; + esac + case "$CMD" in init) cmd_init standalone ;; clean) cmd_clean ;; @@ -1363,6 +1420,7 @@ main() { if [ "$ENRICH_VARSETS" -eq 1 ]; then run_phase enrich "$apply_sha" cmd_enrich; fi run_phase convert "$(payload_sha)" cmd_convert run_phase validate "$(payload_sha)" cmd_validate + RAN_APPLY=1 # the export ran, or its inputs (flags included) were unchanged cmd_import ;; esac From df28fc7deec212d9c600cbe00c2c6ef45f3e7541 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adis=20Halilovi=C4=87?= <adis.halilovic@stackguardian.io> Date: Mon, 14 Sep 2026 17:47:07 +0200 Subject: [PATCH 57/71] feat: --region eu|us picks the StackGuardian API and UI hosts together The API host was only reachable through the undocumented SG_BASE_URL and the checklist's UI links always pointed at app.stackguardian.io. --region (or SG_REGION) sets both: eu = api.app/app.stackguardian.io (default), us = api.us/us.stackguardian.io. An explicit SG_BASE_URL or SG_UI_URL still wins, init remembers the region in .sg/state.json (older files with only sg_base_url keep working), the wrapper forwards SG_REGION, and the run result records region, apiUrl and uiUrl. --- CLAUDE.md | 3 +- README.md | 1 + scripts/lib/report.sh | 4 +- scripts/lib/wizard.sh | 6 +-- scripts/migrate.sh | 92 ++++++++++++++++++++++++++++++++++--------- sg-migrate.sh | 2 +- 6 files changed, 82 insertions(+), 26 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index e3e3752..4ed7868 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -24,10 +24,11 @@ The migration is a user-driven pipeline, not a single program: The five phases are wrapped by an orchestrator so users don't run them by hand: - `sg-migrate.sh` (host entrypoint, repo root) — runs `scripts/migrate.sh` **inside the Docker image** (`Dockerfile`), bind-mounting the repo at `/app` and forwarding `SG_API_TOKEN`/`SG_ORG`/`SG_BASE_URL`/`TFE_TOKEN`/`TF_TOKEN_*`. `--tfvars FILE` (or `SG_TFVARS`) is a host path: the wrapper strips the flag, resolves it and passes it on as `SG_TFVARS` — rewritten to `/app/<relative>` when the file is inside the checkout, otherwise mounted read-only at `/tmp/sg-run.tfvars`. For TFC auth it prefers a long-lived `TFE_TOKEN` (env); otherwise it mounts `~/.terraform.d/credentials.tfrc.json` read-only. Runs natively instead when `--native`/`--local` is passed, `SG_NATIVE=1` is set, the command is `clean`/`completion`/help (or no command is given), or Docker is absent. Exports `SG_PROG` so `migrate.sh` shows `./sg-migrate.sh` in its usage/hints. `update` is host-only too (`cmd_update` in the wrapper, never passed to `migrate.sh`): refuses a non-git checkout, a detached HEAD or dirty tracked files, then `git pull --ff-only` and rebuilds the image only when the `Dockerfile` changed between the old and new HEAD — this is how customers pick up fixes from a plain clone (untracked config/output survive). `migrate.sh` lists `update` in its usage/`SG_COMMANDS` and rejects it with a host-only hint. `.gitattributes` forces LF so a Git-for-Windows clone does not CRLF the scripts. -- `scripts/migrate.sh` — the actual orchestrator (sources `scripts/lib/*.sh`, see below). Subcommands `init|preflight|apply|enrich|convert|validate|import|triggers|checklist|all|clean|completion` (no command prints the help menu; `enrich` runs in `all` unless `--no-variable-sets`). Flags beyond the basics: `--dry-run` (import, or all: the local export runs, then the plan is shown and nothing is created in SG), `--fresh` (ignore run state), the scope flags from `lib/scope.sh` — `--project NAME|SLUG`, `--workspace GLOB`, `--exclude-workspace GLOB`, `--tag NAME`, `--exclude-tag NAME`, all repeatable; tfvars holds the widest scope, include flags replace the tfvars list, exclude flags add to it; apply passes them as `-var` (`tfProjects`, `workspacenames`, `tfWorkspaceIgnoreNames`, `tfWorkspaceTags`, `tfWorkspaceIgnoreTags`), the later phases apply the same names/excludes to the payload entries (`ws_selected` in bash, `WS_SCOPE_JQ` + `WS_JQ_ARGS` in jq) and `--project` to the payload files by slug (`project_selected`); `--workspace '*'` alone is not a narrowing, so the unchanged-file skip still applies; a scope that matches nothing is an error —, `--tfvars FILE` (`TFVARS`, absolute; also `-var-file` for terraform and exported to the enrich script; `clean --all` only removes the module's own file), `--skip-preflight`, `--no-secret-stubs`, `--upgrade` (with `init`: `tfvars_upgrade`, append the settings an older tfvars lacks; preflight warns about them until then), and the run configuration flags `--set KEY=VALUE`, `--cloud-connector ID`, `--vcs-connector ID`, `--runner-group NAME|shared`, `--workflow-group NAME` (one-run overlay on the tfvars, see `scope.sh`; connector kinds looked up in SG; with `--project` they are that project's `projectOverrides`, else the `SGDefault*` values). `import`/`all` end by writing `export/run-result.json` + `run-summary.md` (`write_run_result`, outcomes planned/blocked/success/failed) for CI. Resolves each tool via `sg_resolve` (PATH first — the image installs them — else `sg_ensure_*` cache). `apply`/`all` first run `require_tfc_auth`, which resolves the token the tfe provider will use (`TFE_TOKEN`, `TF_TOKEN_<host>`, or the `terraform login` file for `tfHostname`) and verifies it with `GET /api/v2/account/details`, failing fast on a missing or rejected (expired) credential. Runs `convert` and `import` in parallel (`--concurrency`, default 4), raises `terraform apply -parallelism`, and retries `sg-cli` imports with backoff (`sg_retry`, `SG_RETRIES`). Import resolves each project's workflow group with `group_for` (legacy `.sg/workflow-groups.json` entry → the payload's `CLIConfiguration.WorkflowGroup.name` via `payload_group` → `tfc-<seg>`) and runs `plan_groups`: STATUS `reuse` (exists) / `create` (missing; created after the confirmation) / `missing!` (`--no-create-groups`) / `moved!`; it collects `PLAN_PROBLEMS` — a project whose workflows still live in the group recorded in `state.json` (or the default group) when the target changed (SG cannot move workflows), and two projects sharing a group with overlapping names — prints them as ✗ lines, still shows the per-workflow plan (also under `--dry-run`) and then dies. The skip set (`state_import_done`) is computed before the plan so the ACTION column can say `skip`/`update`/`create`. Groups are never PATCHed. Imports go through `import_bulk`: workflows with Terraform variables via `sg-cli workflow create --bulk`, workflows **without** via a direct `POST .../wfs/` (`sg_create_workflow` + `sg_upload_tfstate` in `sg_api.sh`) because sg-cli drops an empty `iacInputData.data` and the API rejects the workflow — a workaround (`TODO(sg-cli)` markers) until sg-cli ships with sg-sdk-go >= v1.5.7, which sends the empty object; the direct path prints sg-cli-style `Failed to create <name>: <code>: <body>` lines so the parsing below covers both. State files are the migrator's job on both paths: `import_bulk` tracks sg-cli's per-workflow upload lines (`State file uploaded successfully` / `Failed to upload state file for <wf>`), re-uploads the failures via `upload_state` → `sg_upload_tfstate` (adds the `x-ms-blob-type: BlockBlob` header Azure Blob requires; sg-cli omits it and also treats anything but a literal `HTTP/1.1 200 OK` as failure — second `TODO(sg-cli)`), and writes `[state] uploaded|failed|none <wf>` markers into the parsed output; `do_import` turns them into `state_uploaded`/`state_failed` in the result, a workflow without its state fails the file (and `state_import_done` re-tries it next run). Existing workflows come back from sg-cli as `Failed to create <wf>: 409: Workflow ID not unique` (its update branch matches the outdated text `Workflow name not unique` — third `TODO(sg-cli)`); `import_bulk` collects those, PATCHes them via `sg_update_workflow`, re-uploads their state and drops the failure line, and `sg_create_workflow` does the same fallback on the direct path (prints `updated`). Before the parallel import, `probe_import` imports one workflow alone (first selected of the first file) and dies unless the create and the state upload both succeeded — the fail-fast for environment problems. sg-cli exits 0 even when individual workflows fail, so `do_import` parses its output: a workflow rejected as above SG's managed Terraform ceiling (1.5.7, last MPL/FOSS release) is re-imported with `SGDefaultTerraformVersion` (read from `terraform.tfvars`; an explicit `null` there means the `terraformVersion` key is deleted instead, so the org's execution preset decides — `tfvars_is_null` tells an explicit null from a missing key), the payload patched in place, and the case logged to `export/terraform-version-fallbacks.log` plus a printed notice; any other per-workflow failure fails the run. `cmd_import` reads the org's execution preset (`sg_execution_preset`, `GET /orgs/{org}/` → `Settings.workflowDefaults`) and `preset_labels` (report.sh) turns it into the `preset (1.5.7)` / `preset (private:rg)` cells of the plan and the `SG_TF_FALLBACK_LABEL` used in messages. The trigger pass (`do_set_triggers` → `sg_set_vcs_triggers`) skips workflows that do not exist in SG, keeps the sha of each posted `{VCSConfig, VCSTriggers}` body in `triggers.<seg>.sha` and does not re-POST an unchanged, already-set workflow (`--fresh` does; the endpoint upserts anyway, a 4xx "already exists" from older builds counts as success), and `sg_api_post` returns 22 on 4xx so `sg_retry` (via `SG_NO_RETRY_RC`) does not retry definitive errors. `clean` removes local artifacts (`export/`, TF state, tool cache); `clean --all` also removes config. `completion bash|zsh` prints a completion script (`cmd_completion`; commands/options come from `SG_COMMANDS`/`SG_OPTIONS`, keep them in sync with the parser and the host flags); like `clean` it always runs natively. Output is concise by default — `apply` captures terraform's init/plan output and shows only progress + the final summary (or the full log on failure, `TF_IN_AUTOMATION=1`/`-no-color`); `-v`/`--verbose` (`SG_VERBOSE`) streams everything and un-gates the convert detail lines. Colored logging via `sg_step`/`sg_log`/`sg_success`/`sg_warn`/`sg_err`/`sg_row`; paths shown relative via `sg_rel` (all auto-off when not a TTY / `NO_COLOR`). Long steps show a live progress line (`sg_run_quiet` for terraform init/apply, `run_parallel <fn> <max> <label> <items>` for convert/import/triggers — animated only when `SG_ANIMATE=1`, i.e. colors are on; plain log lines otherwise) and report their duration (`sg_fmt_secs`). Phases start with `phase_begin` — inside `all` the header is numbered (`Phase 2/5: ...`, `PHASE_TOTAL` set in `main`) and `phase_took` gives the elapsed time; `run_parallel` prints a job's output as-is when it is a single line and adds the `── <file> ──` header only for longer or failed output (always with `-v`). The import confirmation goes through `sg_confirm` (so `SG_ANSWERS_FILE` works; a declined prompt is a neutral "cancelled" line, not an error) and `import`/`all` end with `finish_line` (outcome, total time, open checklist items from `CHECKLIST_OPEN`). A Ctrl-C is trapped to clear the progress line and print a resume hint. +- `scripts/migrate.sh` — the actual orchestrator (sources `scripts/lib/*.sh`, see below). Subcommands `init|preflight|apply|enrich|convert|validate|import|triggers|checklist|all|clean|completion` (no command prints the help menu; `enrich` runs in `all` unless `--no-variable-sets`). Flags beyond the basics: `--dry-run` (import, or all: the local export runs, then the plan is shown and nothing is created in SG), `--fresh` (ignore run state), the scope flags from `lib/scope.sh` — `--project NAME|SLUG`, `--workspace GLOB`, `--exclude-workspace GLOB`, `--tag NAME`, `--exclude-tag NAME`, all repeatable; tfvars holds the widest scope, include flags replace the tfvars list, exclude flags add to it; apply passes them as `-var` (`tfProjects`, `workspacenames`, `tfWorkspaceIgnoreNames`, `tfWorkspaceTags`, `tfWorkspaceIgnoreTags`), the later phases apply the same names/excludes to the payload entries (`ws_selected` in bash, `WS_SCOPE_JQ` + `WS_JQ_ARGS` in jq) and `--project` to the payload files by slug (`project_selected`); `--workspace '*'` alone is not a narrowing, so the unchanged-file skip still applies; a scope that matches nothing is an error —, `--tfvars FILE` (`TFVARS`, absolute; also `-var-file` for terraform and exported to the enrich script; `clean --all` only removes the module's own file), `--skip-preflight`, `--no-secret-stubs`, `--upgrade` (with `init`: `tfvars_upgrade`, append the settings an older tfvars lacks; preflight warns about them until then), and the run configuration flags `--set KEY=VALUE`, `--cloud-connector ID`, `--vcs-connector ID`, `--runner-group NAME|shared`, `--workflow-group NAME` (one-run overlay on the tfvars, see `scope.sh`; connector kinds looked up in SG; with `--project` they are that project's `projectOverrides`, else the `SGDefault*` values). `import`/`all` end by writing `export/run-result.json` + `run-summary.md` (`write_run_result`, outcomes planned/blocked/success/failed) for CI. Resolves each tool via `sg_resolve` (PATH first — the image installs them — else `sg_ensure_*` cache). `apply`/`all` first run `require_tfc_auth`, which resolves the token the tfe provider will use (`TFE_TOKEN`, `TF_TOKEN_<host>`, or the `terraform login` file for `tfHostname`) and verifies it with `GET /api/v2/account/details`, failing fast on a missing or rejected (expired) credential. Runs `convert` and `import` in parallel (`--concurrency`, default 4), raises `terraform apply -parallelism`, and retries `sg-cli` imports with backoff (`sg_retry`, `SG_RETRIES`). Import resolves each project's workflow group with `group_for` (legacy `.sg/workflow-groups.json` entry → the payload's `CLIConfiguration.WorkflowGroup.name` via `payload_group` → `tfc-<seg>`) and runs `plan_groups`: STATUS `reuse` (exists) / `create` (missing; created after the confirmation) / `missing!` (`--no-create-groups`) / `moved!`; it collects `PLAN_PROBLEMS` — a project whose workflows still live in the group recorded in `state.json` (or the default group) when the target changed (SG cannot move workflows), and two projects sharing a group with overlapping names — prints them as ✗ lines, still shows the per-workflow plan (also under `--dry-run`) and then dies. The skip set (`state_import_done`) is computed before the plan so the ACTION column can say `skip`/`update`/`create`. Groups are never PATCHed. Imports go through `import_bulk`: workflows with Terraform variables via `sg-cli workflow create --bulk`, workflows **without** via a direct `POST .../wfs/` (`sg_create_workflow` + `sg_upload_tfstate` in `sg_api.sh`) because sg-cli drops an empty `iacInputData.data` and the API rejects the workflow — a workaround (`TODO(sg-cli)` markers) until sg-cli ships with sg-sdk-go >= v1.5.7, which sends the empty object; the direct path prints sg-cli-style `Failed to create <name>: <code>: <body>` lines so the parsing below covers both. State files are the migrator's job on both paths: `import_bulk` tracks sg-cli's per-workflow upload lines (`State file uploaded successfully` / `Failed to upload state file for <wf>`), re-uploads the failures via `upload_state` → `sg_upload_tfstate` (adds the `x-ms-blob-type: BlockBlob` header Azure Blob requires; sg-cli omits it and also treats anything but a literal `HTTP/1.1 200 OK` as failure — second `TODO(sg-cli)`), and writes `[state] uploaded|failed|none <wf>` markers into the parsed output; `do_import` turns them into `state_uploaded`/`state_failed` in the result, a workflow without its state fails the file (and `state_import_done` re-tries it next run). Existing workflows come back from sg-cli as `Failed to create <wf>: 409: Workflow ID not unique` (its update branch matches the outdated text `Workflow name not unique` — third `TODO(sg-cli)`); `import_bulk` collects those, PATCHes them via `sg_update_workflow`, re-uploads their state and drops the failure line, and `sg_create_workflow` does the same fallback on the direct path (prints `updated`). Before the parallel import, `probe_import` imports one workflow alone (first selected of the first file) and dies unless the create and the state upload both succeeded — the fail-fast for environment problems. sg-cli exits 0 even when individual workflows fail, so `do_import` parses its output: a workflow rejected as above SG's managed Terraform ceiling (1.5.7, last MPL/FOSS release) is re-imported with `SGDefaultTerraformVersion` (read from `terraform.tfvars`; an explicit `null` there means the `terraformVersion` key is deleted instead, so the org's execution preset decides — `tfvars_is_null` tells an explicit null from a missing key), the payload patched in place, and the case logged to `export/terraform-version-fallbacks.log` plus a printed notice; any other per-workflow failure fails the run. `cmd_import` reads the org's execution preset (`sg_execution_preset`, `GET /orgs/{org}/` → `Settings.workflowDefaults`) and `preset_labels` (report.sh) turns it into the `preset (1.5.7)` / `preset (private:rg)` cells of the plan and the `SG_TF_FALLBACK_LABEL` used in messages. The trigger pass (`do_set_triggers` → `sg_set_vcs_triggers`) skips workflows that do not exist in SG, keeps the sha of each posted `{VCSConfig, VCSTriggers}` body in `triggers.<seg>.sha` and does not re-POST an unchanged, already-set workflow (`--fresh` does; the endpoint upserts anyway, a 4xx "already exists" from older builds counts as success), and `sg_api_post` returns 22 on 4xx so `sg_retry` (via `SG_NO_RETRY_RC`) does not retry definitive errors. `clean` removes local artifacts (`export/`, TF state, tool cache); `clean --all` also removes config. `completion bash|zsh` prints a completion script (`cmd_completion`; commands + descriptions come from `SG_COMMAND_DESCS` (`SG_COMMANDS` is derived from it), options from `SG_OPTIONS`; keep them in sync with the parser and the host flags; the zsh script also works autoloaded from `$fpath` as `_sg-migrate.sh`); like `clean` it always runs natively. Output is concise by default — `apply` captures terraform's init/plan output and shows only progress + the final summary (or the full log on failure, `TF_IN_AUTOMATION=1`/`-no-color`); `-v`/`--verbose` (`SG_VERBOSE`) streams everything and un-gates the convert detail lines. Colored logging via `sg_step`/`sg_log`/`sg_success`/`sg_warn`/`sg_err`/`sg_row`; paths shown relative via `sg_rel` (all auto-off when not a TTY / `NO_COLOR`). Long steps show a live progress line (`sg_run_quiet` for terraform init/apply, `run_parallel <fn> <max> <label> <items>` for convert/import/triggers — animated only when `SG_ANIMATE=1`, i.e. colors are on; plain log lines otherwise) and report their duration (`sg_fmt_secs`). Phases start with `phase_begin` — inside `all` the header is numbered (`Phase 2/5: ...`, `PHASE_TOTAL` set in `main`) and `phase_took` gives the elapsed time; `run_parallel` prints a job's output as-is when it is a single line and adds the `── <file> ──` header only for longer or failed output (always with `-v`). The import confirmation goes through `sg_confirm` (so `SG_ANSWERS_FILE` works; a declined prompt is a neutral "cancelled" line, not an error) and `import`/`all` end with `finish_line` (outcome, total time, open checklist items from `CHECKLIST_OPEN`). A Ctrl-C is trapped to clear the progress line and print a resume hint. - `scripts/lib/` — sourced libraries (a `lib/` gitignore rule is negated for this dir): `prompt.sh` (`sg_ask`/`sg_confirm`/`sg_select`; tty or `SG_ANSWERS_FILE` for tests; `SG_NONINTERACTIVE=1`/`-y`/no TTY → defaults), `tfvars.sh` (`tfvars_get` cached hcl2json reads — `tfvars_json` merges the run overlay (`TFVARS_OVERLAY_JSON`) over the file, and `tfvars_get`/`tfvars_get_json` keep a `false` value instead of treating it like a missing key —, `tfvars_has`/`tfvars_is_null` to tell an explicit `null` from a missing key, `tfvars_write` renders the wizard's `W_*` vars — `W_TF_SOURCE`, and `W_TF_VERSION`/`W_RUNNER_JSON` may be the literal `null`; plain strings go through `_tfvars_str`, so quotes/backslashes in names stay valid HCL; `projectOverrides`/`workspaceOverrides` are rendered from `W_PROJECT_OVERRIDES_JSON`/`W_WS_OVERRIDES_JSON` by `_tfvars_map` — `"key" = { attr = <json> }`, which hcl2json round-trips — so hand-written blocks survive a re-run of `init` (`wizard_run` reads them before writing; inner comments are lost); `tfWorkspaceIgnoreNames` is kept the same way (`W_IGNORE_NAMES_JSON`, never asked, shown in the review's scope row), and every other setting the wizard does not ask about (`tfProjects`, `cloudAuthVarPatterns`, hand-added keys) is appended verbatim from `W_PREV_JSON` under "Kept from the previous file"; `tfvars_missing_keys` (keys of `terraform.tfvars.example` the file lacks) and `tfvars_upgrade` (appends those paragraphs of the example — comment + default — under a dated header, `.bak` kept, touches nothing else; behind `init --upgrade`, prompt-free)), `scope.sh` (the run scope: `PROJECT_FILTER`/`WS_FILTER`/`WS_EXCLUDE`/`TAG_FILTER`/`TAG_EXCLUDE` arrays, `names_json`, `slug_of` (the transformer's project-slug rule in bash), `project_selected <seg>`, `ws_narrowed`/`ws_filter_json`/`ws_exclude_json` (tfvars `tfWorkspaceIgnoreNames` ∪ flags, memoized)/`ws_selected <name>` (bash `case` globs), `WS_SCOPE_JQ` + `ws_jq_args` → `WS_JQ_ARGS` for the jq sites, `scope_tags_json`/`scope_ignore_tags_json`, `scope_tfvar_args` → `SCOPE_TFVAR_ARGS` for apply, `scope_describe`, `scope_sha_input` for the apply phase hash; and the **run configuration overlay**: `--set KEY=VALUE` (`SET_VARS`, values via `_overlay_value`: HCL/JSON literals through hcl2json, bare text stays a string), `--cloud-connector`/`--vcs-connector`/`--runner-group`/`--workflow-group` → `overlay_build` looks the connector kinds up with `sg_list_integrations`/`sg_integration_type`/`sg_vcs_kind_of`, resolves `--project` values to raw TFC names (`_overlay_project_name`, fatal for an unknown project, as-given with a warning when TFC is unreachable) and fills `TFVARS_OVERLAY_JSON` — `projectOverrides[<name>]` fields with `--project`, `SGDefault*` keys without; `tfvars_json` deep-merges it over the file so every reader sees it, `scope_tfvar_args` passes each overlay key as `-var`, `scope_sha_input` includes it, `write_run_result` records it under `configuration`; `main` refuses the flags for `init`/`clean`/`completion` and a standalone `import` warns that they only reach the workflows through the export (`RAN_APPLY`)), `tfc_api.sh` (`tfc_http`/`tfc_get_all` paginated, `tfc_list_{orgs,projects,workspaces}` — workspaces include `vcs_provider`/`vcs_url`/`vcs_identifier` —, `tfc_select_workspaces` (the module's name-glob / include-tags(all) / exclude-tags / exclude-name-globs filter, shared by wizard and preflight), `tfc_vcs_summary`/`tfc_vcs_kind_for`/`tfc_vcs_label_for`, `tfc_token` with provider precedence, `require_tfc_auth`), `sg_api.sh` (`_sg_api` 0/22/1 contract + `SG_HTTP_CODE`, `_sg_listall` paginated listall reader (the API pages at 50) behind `sg_list_workflows`/`sg_list_wfgrps`, `sg_set_vcs_triggers`, `sg_list_integrations`, `sg_integration_type`, `sg_vcs_kind_of` (connector type → `sourceConfigDestKind`), `sg_execution_preset` + `sg_preset_{desc,runner_desc,version_desc,runner_provided}` (the org's execution preset and how to describe it), `sg_*_exists`, `sg_list_workflows`, `sg_create_secret`, `sg_patch_workflow`, `wfgroup_*`), `wizard.sh` (`wizard_run`: 5 steps TFC → SG → projects → defaults → review; `wizard_projects` (with >1 selected project, from `W_SEL_PROJECTS_JSON`) asks whether the global connectors and `tfc-<project>` groups apply to all, else per project the cloud connector, VCS connector (kind + prefix follow) and group (default / existing via `sg_list_wfgrps` / typed), writing `projectOverrides`; "same as the default" deletes the key, unmanaged fields are kept, previous picks come first (`_w_first`); the defaults step also asks `stripCloudAuthVars`; the TFC step keeps the workspace list and derives, via `tfc_vcs_summary`, which VCS provider the selected workspaces use and the repositories' URL prefix — the SG step lists matching connectors first and takes the repo URL prefix from TFC (never from a previous tfvars written for another provider kind), warns on a kind mismatch, reads the org's execution preset and offers it as the runner choice; the defaults step asks whether to carry TFC versions or use the preset (`SGTerraformVersionSource`) and, for carry, which fallback (`SGDefaultTerraformVersion` or the preset = null); the review shows human-readable rows plus the `tfc-<project>` groups; `_w_select_lines` feeds menus line-by-line so names with spaces/globs survive), `preflight.sh` (`preflight_run <apply|import|all>`, once per process; ✓ lines are plain language and name the connector's role/kind, ✗/! lines name the tfvars field to fix; connector ids, runner groups and DPC kinds are collected from `SGDefault*`, `projectOverrides` and `workspaceOverrides`; besides existence it checks consistency — VCS kind vs. the connector's type, globally and per `projectOverrides` entry (fail), repo URL prefix vs. where the TFC repositories live or the kind's well-known host (warn), `projectOverrides` keys vs. the TFC project names (warn), `--project` values vs. the project names/slugs (fail; a `tfProjects` typo warns), and the workspace-selection preview applies the CLI scope (names, excludes, tags, projects) so its count is what the apply will export; the import context states the group policy and warns when a project's group changed since its last import; it also reports the org's execution preset next to the version/runner policy and warns when `carry` meets a preset that mounts a runner-provided binary), `state.sh` (`.sg/state.json`: `phases.<name>.input_sha`, `import.<seg>.{group,imported,updated,failed,tf_fallback,state_uploaded,state_failed}` (`group` anchors the never-move check; `updated` = PATCHed on a 409), `triggers.<seg>.{set,unchanged,failed,missing,sha}`, `secrets.<name>`; `run_phase` in migrate.sh skips unchanged phases), `report.sh` (`show_migration_summary` after apply — incl. a state-files row —, `show_import_plan` per-workflow table with columns sized to their content — it keeps the rows in `PLAN_ROWS_TSV` (name, group, action, version, runner, triggers, vars, secrets, segment, workspace, project) —, `write_run_result <outcome>` → `export/run-result.json` + `run-summary.md` from `PLAN_ROWS_TSV`, `PLAN_GROUP_ROWS` (plain copy of the group table, set by `plan_groups`), `PLAN_PROBLEMS`, the merged state and `CHECKLIST_OPEN`; per workflow `result` is planned/skipped/failed/updated/created, with the probe workflow — planned `create`, PATCHed on its second pass — reported as created), `errors.sh` (`explain_api_error` regex→hint table, `SG_API_HINTS`), `checklist.sh` (`create_secret_stubs` → SG secret `tfc-<wf>-<VAR>` = `CHANGE_ME`, referenced as `${secret::<name>}` via PATCH + payload patch; `write_checklist` → `export/post-import-checklist.md`, printed to the terminal as one ✓/! status line per section plus the imported groups with UI links — the file has the details; section 5 covers both the TFC state export and the SG state upload (`state_failed` from the import results); sets `CHECKLIST_OPEN`). Parallel jobs (`run_parallel`) run in subshells, so `do_import`/`do_set_triggers` write `.import-result.<seg>.json` / `.triggers-result.<seg>.json` into the export dir and the caller merges them into state. - `scripts/tools.sh` — sourced by the other scripts (sets `SG_REPO_ROOT` to its parent dir). Provides `sg_resolve` (PATH-or-cache), `sg_ensure_{jq,hcl2json,yajsv,sgcli}` (pinned downloads into `.sg/cached/`; `sg-cli` is the Go binary), `sg_retry`, the color vars + log helpers. The Docker image bundles these tools at build time (`yajsv` built from source so arm64 works), so in-container runs never download. - **Variable sets** — `scripts/enrich_variable_sets.sh <tfc-org> <payload...>` (run by `cmd_enrich`, which reads `tfOrg`/`tfHostname` from `terraform.tfvars` via `hcl2json` — variable sets live in the **TFC** org, not `SG_ORG`). Lists all sets + vars via the TFC API (Bearer token from the `terraform login` creds file or `TFE_TOKEN`), computes per-workspace effective vars (scope: global/project/workspace; precedence: priority sets > workspace vars > non-priority sets), and merges them into each workflow (matched by workspace name parsed from `TfStateFilePath`). Sensitive set vars and key conflicts are reported. Skipped by `--no-variable-sets`. +- **Regions** — `--region eu|us` / `SG_REGION` (`region_urls`/`region_apply` in migrate.sh) pick `SG_BASE_URL` and `SG_UI_URL` together: eu = `api.app`/`app.stackguardian.io` (default), us = `api.us`/`us.stackguardian.io`, and the undocumented internal `dash` = `testapi.qa`/`dash.qa.stackguardian.io` (never mention it in user-facing docs). An explicit `SG_BASE_URL`/`SG_UI_URL` wins over the region. `init` stores `config.sg_region` in `.sg/state.json` (older files only have `sg_base_url`, still honoured); the plan header and `run-result.json` (`region`, `apiUrl`, `uiUrl`) show what a run used. The wrapper forwards `SG_REGION`. - **Workflow groups** — each TFC project maps to the SG workflow group the transformer wrote into its payload (`projectOverrides[<project>].workflowGroup`, default `tfc-<project-segment>`); an existing group is reused, a missing one created via `POST /api/v1/orgs/{org}/wfgrps/` (auth `Authorization: apikey <token>`) unless `--no-create-groups`. (Both the API calls and sg-cli honor the undocumented `SG_BASE_URL` for non-prod targets.) Groups are addressed by name, so no generated ID is tracked, and they are never PATCHed; workflows are never moved between groups (see `plan_groups`). `.sg/workflow-groups.json` (gitignored) is a deprecated per-segment override that still applies on top, with a warning. ### The transformer (`transformer/terraform-cloud/`) diff --git a/README.md b/README.md index ca6731b..4b22016 100644 --- a/README.md +++ b/README.md @@ -30,6 +30,7 @@ export SG_ORG=<your SG org> That's it — no IDs to look up and no workflow-group mapping to fill in. `init` lists what the tokens can see (TFC organisations and workspaces, SG VCS/cloud connectors and runner groups, the org's execution preset) and writes `terraform.tfvars` from your picks — it also reads which VCS provider your TFC workspaces are connected to, lists the matching connectors first and takes the repository URL prefix from TFC. With more than one TFC project it asks whether the same connectors and groups apply to every project, or lets you pick a cloud connector, a VCS connector and the workflow group per project (see *Per-project settings* below). `all` verifies every reference **before** running terraform (preflight — including that the VCS connector, the VCS kind and the repo URL prefix agree with each other and with the TFC repositories), prints a migration summary after the export, shows a per-workflow import plan (create/update, Terraform version, runner, triggers, secrets), and ends with a **post-import checklist** of what still needs a human. Phases are numbered, long steps show a live progress line, and every phase reports how long it took. Each TFC project is imported into the SG workflow group assigned to it — `tfc-<project>` by default, or the group you picked for that project — **reusing the group when it exists and creating it otherwise**. Workflows are never moved between groups: when a project's workflows already live in another group, the plan stops and says so. +- **Region.** StackGuardian runs in the EU (`api.app.stackguardian.io` / `app.stackguardian.io`, the default) and the US (`api.us.stackguardian.io` / `us.stackguardian.io`). Pass `--region us` (or set `SG_REGION=us`) on `init` and it is remembered for every later run; the import plan and `export/run-result.json` show which API the run used. `SG_BASE_URL` / `SG_UI_URL` override the region's hosts for private or test environments. - **Updating.** Clone the repo (don't fork it or download the release zip) and run `./sg-migrate.sh update` to pull the latest version; it fast-forwards the checkout and rebuilds the Docker image only if the `Dockerfile` changed. Your `terraform.tfvars`, `export/` and `.sg/` are never tracked, so they survive every update. To stay on a fixed release instead, `git checkout v1.2.2` (then `git checkout master` to follow the latest again). - Single phase: `./sg-migrate.sh preflight|apply|enrich|convert|validate|import|triggers|checklist`. Running `./sg-migrate.sh` with no command prints the help menu. - **Resume.** `all` remembers what it completed (`.sg/state.json`) and skips phases whose inputs have not changed, so after a failure you just re-run it; files already imported in full are skipped and files with failures are retried. `--fresh` redoes everything. diff --git a/scripts/lib/report.sh b/scripts/lib/report.sh index 710685c..188f8d8 100644 --- a/scripts/lib/report.sh +++ b/scripts/lib/report.sh @@ -190,13 +190,13 @@ write_run_result() { --argjson xt "$([ "${#TAG_EXCLUDE[@]}" -gt 0 ] && names_json "${TAG_EXCLUDE[@]}" || echo '[]')" \ '{projects: $p, workspaces: $w, excludeWorkspaces: $x, tags: $t, excludeTags: $xt}')" state_read | "$JQ_BIN" --arg cmd "$PROG ${SG_RUN_ARGS:-}" --arg at "$(state_now)" --argjson took "$((SECONDS - RUN_T0))" \ - --arg org "$ORG" --arg url "$SG_BASE_URL" --argjson dry "$([ "${DRY_RUN:-0}" -eq 1 ] && echo true || echo false)" \ + --arg org "$ORG" --arg url "$SG_BASE_URL" --arg region "${SG_REGION:-}" --arg ui "${SG_UI_URL:-}" --argjson dry "$([ "${DRY_RUN:-0}" -eq 1 ] && echo true || echo false)" \ --arg outcome "$outcome" --argjson scope "$scope" --argjson groups "${PLAN_GROUP_ROWS:-[]}" \ --argjson overlay "${TFVARS_OVERLAY_JSON:-{\}}" --arg overlay_desc "$(declare -F overlay_describe >/dev/null && overlay_describe || true)" \ --argjson problems "$problems" --argjson rows "$rows" --argjson open "${CHECKLIST_OPEN:-0}" ' . as $st | { - command: $cmd, at: $at, tookSeconds: $took, org: $org, apiUrl: $url, dryRun: $dry, outcome: $outcome, + command: $cmd, at: $at, tookSeconds: $took, org: $org, region: $region, apiUrl: $url, uiUrl: $ui, dryRun: $dry, outcome: $outcome, scope: $scope, configuration: {description: $overlay_desc, overlay: $overlay}, groups: $groups, problems: $problems, workflows: [ $rows[] | . as $r | ($st.import[$r.segment] // {}) as $imp | ($st.triggers[$r.segment] // {}) as $tr diff --git a/scripts/lib/wizard.sh b/scripts/lib/wizard.sh index 38625be..3e6e545 100644 --- a/scripts/lib/wizard.sh +++ b/scripts/lib/wizard.sh @@ -648,8 +648,8 @@ wizard_run() { sg_err "the generated $(sg_rel "$TFVARS") is not valid HCL — this is a bug in the wizard; the file was kept for inspection" return 1 fi - # Remember the SG org (and API host) for later phases, so users don't have to - # export SG_ORG again in a new shell. Tokens are never stored. - state_update '.config = ((.config // {}) + {sg_org: $o, sg_base_url: $u})' --arg o "$ORG" --arg u "$SG_BASE_URL" + # Remember the SG org, region and API host for later phases, so users don't + # have to export SG_ORG / SG_REGION again in a new shell. Tokens are never stored. + state_update '.config = ((.config // {}) + {sg_org: $o, sg_region: $r, sg_base_url: $u})' --arg o "$ORG" --arg r "${SG_REGION:-eu}" --arg u "$SG_BASE_URL" sg_success "wrote $(sg_rel "$TFVARS")$kept" } diff --git a/scripts/migrate.sh b/scripts/migrate.sh index e8cb28c..01dc302 100755 --- a/scripts/migrate.sh +++ b/scripts/migrate.sh @@ -7,6 +7,9 @@ set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# Captured before the libraries load: lib/checklist.sh gives SG_UI_URL a +# default, and region_apply must know whether the user set it explicitly. +SG_UI_URL_SET="${SG_UI_URL:-}" # shellcheck source=tools.sh source "$SCRIPT_DIR/tools.sh" # shellcheck source=lib/prompt.sh @@ -49,8 +52,31 @@ PROG="${SG_PROG:-$0}" EXPORT_DIR="${SG_EXPORT_DIR:-$SG_REPO_ROOT/export}" MAPPING="${SG_WFGROUP_MAP:-$SG_REPO_ROOT/.sg/workflow-groups.json}" ORG="${SG_ORG:-}" +# StackGuardian region: --region / SG_REGION picks the API and UI hosts (eu is +# the default; us; dash = the internal QA environment). An explicit SG_BASE_URL +# or SG_UI_URL always wins over the region's host. Applied by region_apply once +# the flags are parsed and the state consulted. +SG_REGION="${SG_REGION:-}" SG_BASE_URL_SET="${SG_BASE_URL:-}" SG_BASE_URL="${SG_BASE_URL:-https://api.app.stackguardian.io}" +SG_UI_URL="${SG_UI_URL:-https://app.stackguardian.io}" +# region_urls <region> — "<api-url> <ui-url>", exit 1 for an unknown region. +region_urls() { + case "$1" in + eu) printf 'https://api.app.stackguardian.io https://app.stackguardian.io' ;; + us) printf 'https://api.us.stackguardian.io https://us.stackguardian.io' ;; + dash) printf 'https://testapi.qa.stackguardian.io https://dash.qa.stackguardian.io' ;; # internal QA + *) return 1 ;; + esac +} +region_apply() { + local urls + SG_REGION="${SG_REGION:-eu}" + urls="$(region_urls "$SG_REGION")" || die "unknown --region '$SG_REGION' (eu or us)" + [ -z "$SG_BASE_URL_SET" ] && SG_BASE_URL="${urls% *}" + [ -z "$SG_UI_URL_SET" ] && SG_UI_URL="${urls#* }" + export SG_BASE_URL SG_UI_URL +} ASSUME_YES=0 PURGE=0 UPGRADE=0 @@ -83,7 +109,14 @@ RUN_T0=$SECONDS # shell without SG_ORG still works; flags and env always win. if [ -z "$ORG" ] && [ -f "$STATE_FILE" ]; then ORG="$(state_read | "$(sg_resolve jq sg_ensure_jq)" -r '.config.sg_org // empty' 2>/dev/null || true)" - [ -n "$ORG" ] && [ -z "${SG_BASE_URL_SET:-}" ] && SG_BASE_URL="$(state_read | "$(sg_resolve jq sg_ensure_jq)" -r --arg d "$SG_BASE_URL" '.config.sg_base_url // $d' 2>/dev/null || echo "$SG_BASE_URL")" + if [ -n "$ORG" ] && [ -z "$SG_REGION" ] && [ -z "$SG_BASE_URL_SET" ]; then + # The region init ran with; older state files only carry the API host. + SG_REGION="$(state_read | "$(sg_resolve jq sg_ensure_jq)" -r '.config.sg_region // empty' 2>/dev/null || true)" + if [ -z "$SG_REGION" ]; then + SG_BASE_URL_SET="$(state_read | "$(sg_resolve jq sg_ensure_jq)" -r '.config.sg_base_url // empty' 2>/dev/null || true)" + [ -n "$SG_BASE_URL_SET" ] && SG_BASE_URL="$SG_BASE_URL_SET" + fi + fi fi @@ -123,6 +156,8 @@ in another group the plan stops and says so. Options: --org NAME StackGuardian org for import (or set SG_ORG) + --region eu|us StackGuardian region: eu = api.app/app.stackguardian.io (default), + us = api.us/us.stackguardian.io (or set SG_REGION; init remembers it) --export-dir DIR Payload/state output dir (default: ./export) --tfvars FILE Use this tfvars file instead of transformer/terraform-cloud/terraform.tfvars (or set SG_TFVARS); the file the transformer, enrich and preflight read @@ -168,7 +203,9 @@ Environment: SG_RETRIES Import retry attempts on failure (default: 4) SG_TF_PARALLELISM terraform apply -parallelism (default: 20) SG_TFVARS tfvars file to use (same as --tfvars) - SG_UI_URL StackGuardian UI base for checklist links (default: https://app.stackguardian.io) + SG_REGION StackGuardian region (same as --region) + SG_BASE_URL StackGuardian API base URL; overrides the region's host + SG_UI_URL StackGuardian UI base for checklist links; overrides the region's host EOF } @@ -1134,13 +1171,28 @@ finish_line() { } # Single source of truth for shell completion (keep in sync with the parser below -# and the host-only flags in sg-migrate.sh). -SG_COMMANDS="init preflight apply enrich convert validate import triggers checklist all clean completion update" -SG_OPTIONS="--org --export-dir --tfvars --mapping --concurrency --no-create-groups --no-variable-sets --no-vcs-triggers --skip-preflight --dry-run --no-secret-stubs --fresh --project --workspace --exclude-workspace --tag --exclude-tag --all --upgrade --set --cloud-connector --vcs-connector --runner-group --workflow-group -v --verbose -y --yes -h --help --native --local --build" +# and the host-only flags in sg-migrate.sh). One "cmd:description" per line; the +# zsh script shows the descriptions, SG_COMMANDS is derived from the first column. +SG_COMMAND_DESCS="init:Guided setup — generates terraform.tfvars +preflight:Verify tokens and every id in terraform.tfvars before running +apply:Run the transformer (terraform apply) +enrich:Merge TFC Variable Set variables into the payloads +convert:Convert HCL-string variables to JSON +validate:Validate payloads against the SG schema +import:Import payloads to StackGuardian, then register VCS triggers +triggers:Register VCS triggers for already-imported workflows +checklist:Write the post-import checklist (and create secret stubs) +all:preflight -> apply -> enrich -> convert -> validate -> import -> checklist +clean:Remove local working artifacts +completion:Print a shell completion script +update:Pull the latest migrator version (git) and rebuild the image if needed" +SG_COMMANDS="$(printf '%s\n' "$SG_COMMAND_DESCS" | cut -d: -f1 | tr '\n' ' ' | sed 's/ $//')" +SG_OPTIONS="--org --region --export-dir --tfvars --mapping --concurrency --no-create-groups --no-variable-sets --no-vcs-triggers --skip-preflight --dry-run --no-secret-stubs --fresh --project --workspace --exclude-workspace --tag --exclude-tag --all --upgrade --set --cloud-connector --vcs-connector --runner-group --workflow-group -v --verbose -y --yes -h --help --native --local --build" # cmd_completion <bash|zsh> — print a completion script for sg-migrate.sh / # migrate.sh to stdout. Both shells fall back to the basename when the command -# is invoked by path, so ./sg-migrate.sh completes too. +# is invoked by path, so ./sg-migrate.sh completes too. The zsh script works +# both sourced and saved into a \$fpath dir as _sg-migrate.sh. cmd_completion() { local shell="${1:-$(current_shell)}" case "$shell" in @@ -1160,6 +1212,7 @@ _sg_migrate() { case "\$prev" in --export-dir) COMPREPLY=(\$(compgen -d -- "\$cur")); return ;; --mapping | --tfvars) COMPREPLY=(\$(compgen -f -- "\$cur")); return ;; + --region) COMPREPLY=(\$(compgen -W "eu us" -- "\$cur")); return ;; --org | --concurrency | --project | --workspace | --exclude-workspace | --tag | --exclude-tag | --set | --cloud-connector | --vcs-connector | --runner-group | --workflow-group) COMPREPLY=(); return ;; completion) COMPREPLY=(\$(compgen -W "bash zsh" -- "\$cur")); return ;; esac @@ -1187,21 +1240,11 @@ fi _sg_migrate() { local -a cmds cmds=( - 'init:Guided setup — generates terraform.tfvars' - 'preflight:Verify tokens and every id in terraform.tfvars before running' - 'apply:Run the transformer (terraform apply)' - 'enrich:Merge TFC Variable Set variables into the payloads' - 'convert:Convert HCL-string variables to JSON' - 'validate:Validate payloads against the SG schema' - 'import:Import payloads to StackGuardian, then register VCS triggers' - 'triggers:Register VCS triggers for already-imported workflows' - 'checklist:Write the post-import checklist (and create secret stubs)' - 'all:apply -> enrich -> convert -> validate -> import' - 'clean:Remove local working artifacts' - 'completion:Print a shell completion script' +$(printf '%s\n' "$SG_COMMAND_DESCS" | sed "s/.*/ '&'/") ) _arguments -s \\ '--org[StackGuardian org for import]:org' \\ + '--region[StackGuardian region]:region:(eu us)' \\ '--export-dir[Payload/state output dir]:dir:_files -/' \\ '--tfvars[tfvars file to use instead of terraform.tfvars]:file:_files' \\ '--mapping[Project-segment -> group override map]:file:_files' \\ @@ -1237,7 +1280,12 @@ _sg_migrate() { shell) [[ "\${words[CURRENT-1]}" == completion ]] && _values 'shell' bash zsh ;; esac } -compdef _sg_migrate sg-migrate.sh migrate.sh +# Sourced (source <(... completion)): register. Autoloaded from a file in \$fpath +# (installed as _sg-migrate.sh): this run *is* the completion call, so complete now. +case "\${funcstack[1]}" in + _*) _sg_migrate "\$@" ;; + *) compdef _sg_migrate sg-migrate.sh migrate.sh ;; +esac ZSH ;; *) die "usage: $PROG completion [bash|zsh] (default: the shell you are running)" ;; @@ -1260,6 +1308,11 @@ main() { shift ;; --org=*) ORG="${1#*=}" ;; + --region) + SG_REGION="$2" + shift + ;; + --region=*) SG_REGION="${1#*=}" ;; --export-dir) EXPORT_DIR="$2" shift @@ -1368,6 +1421,7 @@ main() { exit 0 fi export SG_VERBOSE="$VERBOSE" + region_apply # The run configuration flags (lib/scope.sh) shape a run, not the file. case "$CMD" in diff --git a/sg-migrate.sh b/sg-migrate.sh index e57cb0e..43c9e67 100755 --- a/sg-migrate.sh +++ b/sg-migrate.sh @@ -128,7 +128,7 @@ fi DOCKER_ARGS=(--rm -i -v "$SCRIPT_DIR:/app" -w /app - -e SG_API_TOKEN -e SG_ORG -e SG_BASE_URL -e SG_CONCURRENCY -e SG_RETRIES -e SG_TF_PARALLELISM + -e SG_API_TOKEN -e SG_ORG -e SG_REGION -e SG_BASE_URL -e SG_CONCURRENCY -e SG_RETRIES -e SG_TF_PARALLELISM -e TFE_TOKEN -e SG_PROG -e SG_SHELL -e SG_NONINTERACTIVE -e SG_UI_URL) # Interactive TTY only when attached to one (so the confirmation prompt works, From 9e458c105403911a110b2aa7a02065d97c5bfeb7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adis=20Halilovi=C4=87?= <adis.halilovic@stackguardian.io> Date: Mon, 14 Sep 2026 17:59:41 +0200 Subject: [PATCH 58/71] feat: init asks which TFC projects to migrate and always writes tfProjects The project comes before the workspaces: with more than one project the wizard lists them with their workspace counts and asks for all of them or some by name or slug (unknown names are re-asked, a previous choice is the default), the workspace questions then apply within those projects, and tfProjects is written to the file ([] = every project) instead of only surviving as a carried-over key. Also fixes a stray 'jq: invalid JSON text passed to --argjson' from the paginated workflow list when the API answers with an empty or non-JSON body for a group that does not exist yet. --- CLAUDE.md | 2 +- README.md | 2 +- scripts/lib/sg_api.sh | 9 +++++--- scripts/lib/tfvars.sh | 6 ++++- scripts/lib/wizard.sh | 52 +++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 65 insertions(+), 6 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 4ed7868..a2d5399 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -25,7 +25,7 @@ The five phases are wrapped by an orchestrator so users don't run them by hand: - `sg-migrate.sh` (host entrypoint, repo root) — runs `scripts/migrate.sh` **inside the Docker image** (`Dockerfile`), bind-mounting the repo at `/app` and forwarding `SG_API_TOKEN`/`SG_ORG`/`SG_BASE_URL`/`TFE_TOKEN`/`TF_TOKEN_*`. `--tfvars FILE` (or `SG_TFVARS`) is a host path: the wrapper strips the flag, resolves it and passes it on as `SG_TFVARS` — rewritten to `/app/<relative>` when the file is inside the checkout, otherwise mounted read-only at `/tmp/sg-run.tfvars`. For TFC auth it prefers a long-lived `TFE_TOKEN` (env); otherwise it mounts `~/.terraform.d/credentials.tfrc.json` read-only. Runs natively instead when `--native`/`--local` is passed, `SG_NATIVE=1` is set, the command is `clean`/`completion`/help (or no command is given), or Docker is absent. Exports `SG_PROG` so `migrate.sh` shows `./sg-migrate.sh` in its usage/hints. `update` is host-only too (`cmd_update` in the wrapper, never passed to `migrate.sh`): refuses a non-git checkout, a detached HEAD or dirty tracked files, then `git pull --ff-only` and rebuilds the image only when the `Dockerfile` changed between the old and new HEAD — this is how customers pick up fixes from a plain clone (untracked config/output survive). `migrate.sh` lists `update` in its usage/`SG_COMMANDS` and rejects it with a host-only hint. `.gitattributes` forces LF so a Git-for-Windows clone does not CRLF the scripts. - `scripts/migrate.sh` — the actual orchestrator (sources `scripts/lib/*.sh`, see below). Subcommands `init|preflight|apply|enrich|convert|validate|import|triggers|checklist|all|clean|completion` (no command prints the help menu; `enrich` runs in `all` unless `--no-variable-sets`). Flags beyond the basics: `--dry-run` (import, or all: the local export runs, then the plan is shown and nothing is created in SG), `--fresh` (ignore run state), the scope flags from `lib/scope.sh` — `--project NAME|SLUG`, `--workspace GLOB`, `--exclude-workspace GLOB`, `--tag NAME`, `--exclude-tag NAME`, all repeatable; tfvars holds the widest scope, include flags replace the tfvars list, exclude flags add to it; apply passes them as `-var` (`tfProjects`, `workspacenames`, `tfWorkspaceIgnoreNames`, `tfWorkspaceTags`, `tfWorkspaceIgnoreTags`), the later phases apply the same names/excludes to the payload entries (`ws_selected` in bash, `WS_SCOPE_JQ` + `WS_JQ_ARGS` in jq) and `--project` to the payload files by slug (`project_selected`); `--workspace '*'` alone is not a narrowing, so the unchanged-file skip still applies; a scope that matches nothing is an error —, `--tfvars FILE` (`TFVARS`, absolute; also `-var-file` for terraform and exported to the enrich script; `clean --all` only removes the module's own file), `--skip-preflight`, `--no-secret-stubs`, `--upgrade` (with `init`: `tfvars_upgrade`, append the settings an older tfvars lacks; preflight warns about them until then), and the run configuration flags `--set KEY=VALUE`, `--cloud-connector ID`, `--vcs-connector ID`, `--runner-group NAME|shared`, `--workflow-group NAME` (one-run overlay on the tfvars, see `scope.sh`; connector kinds looked up in SG; with `--project` they are that project's `projectOverrides`, else the `SGDefault*` values). `import`/`all` end by writing `export/run-result.json` + `run-summary.md` (`write_run_result`, outcomes planned/blocked/success/failed) for CI. Resolves each tool via `sg_resolve` (PATH first — the image installs them — else `sg_ensure_*` cache). `apply`/`all` first run `require_tfc_auth`, which resolves the token the tfe provider will use (`TFE_TOKEN`, `TF_TOKEN_<host>`, or the `terraform login` file for `tfHostname`) and verifies it with `GET /api/v2/account/details`, failing fast on a missing or rejected (expired) credential. Runs `convert` and `import` in parallel (`--concurrency`, default 4), raises `terraform apply -parallelism`, and retries `sg-cli` imports with backoff (`sg_retry`, `SG_RETRIES`). Import resolves each project's workflow group with `group_for` (legacy `.sg/workflow-groups.json` entry → the payload's `CLIConfiguration.WorkflowGroup.name` via `payload_group` → `tfc-<seg>`) and runs `plan_groups`: STATUS `reuse` (exists) / `create` (missing; created after the confirmation) / `missing!` (`--no-create-groups`) / `moved!`; it collects `PLAN_PROBLEMS` — a project whose workflows still live in the group recorded in `state.json` (or the default group) when the target changed (SG cannot move workflows), and two projects sharing a group with overlapping names — prints them as ✗ lines, still shows the per-workflow plan (also under `--dry-run`) and then dies. The skip set (`state_import_done`) is computed before the plan so the ACTION column can say `skip`/`update`/`create`. Groups are never PATCHed. Imports go through `import_bulk`: workflows with Terraform variables via `sg-cli workflow create --bulk`, workflows **without** via a direct `POST .../wfs/` (`sg_create_workflow` + `sg_upload_tfstate` in `sg_api.sh`) because sg-cli drops an empty `iacInputData.data` and the API rejects the workflow — a workaround (`TODO(sg-cli)` markers) until sg-cli ships with sg-sdk-go >= v1.5.7, which sends the empty object; the direct path prints sg-cli-style `Failed to create <name>: <code>: <body>` lines so the parsing below covers both. State files are the migrator's job on both paths: `import_bulk` tracks sg-cli's per-workflow upload lines (`State file uploaded successfully` / `Failed to upload state file for <wf>`), re-uploads the failures via `upload_state` → `sg_upload_tfstate` (adds the `x-ms-blob-type: BlockBlob` header Azure Blob requires; sg-cli omits it and also treats anything but a literal `HTTP/1.1 200 OK` as failure — second `TODO(sg-cli)`), and writes `[state] uploaded|failed|none <wf>` markers into the parsed output; `do_import` turns them into `state_uploaded`/`state_failed` in the result, a workflow without its state fails the file (and `state_import_done` re-tries it next run). Existing workflows come back from sg-cli as `Failed to create <wf>: 409: Workflow ID not unique` (its update branch matches the outdated text `Workflow name not unique` — third `TODO(sg-cli)`); `import_bulk` collects those, PATCHes them via `sg_update_workflow`, re-uploads their state and drops the failure line, and `sg_create_workflow` does the same fallback on the direct path (prints `updated`). Before the parallel import, `probe_import` imports one workflow alone (first selected of the first file) and dies unless the create and the state upload both succeeded — the fail-fast for environment problems. sg-cli exits 0 even when individual workflows fail, so `do_import` parses its output: a workflow rejected as above SG's managed Terraform ceiling (1.5.7, last MPL/FOSS release) is re-imported with `SGDefaultTerraformVersion` (read from `terraform.tfvars`; an explicit `null` there means the `terraformVersion` key is deleted instead, so the org's execution preset decides — `tfvars_is_null` tells an explicit null from a missing key), the payload patched in place, and the case logged to `export/terraform-version-fallbacks.log` plus a printed notice; any other per-workflow failure fails the run. `cmd_import` reads the org's execution preset (`sg_execution_preset`, `GET /orgs/{org}/` → `Settings.workflowDefaults`) and `preset_labels` (report.sh) turns it into the `preset (1.5.7)` / `preset (private:rg)` cells of the plan and the `SG_TF_FALLBACK_LABEL` used in messages. The trigger pass (`do_set_triggers` → `sg_set_vcs_triggers`) skips workflows that do not exist in SG, keeps the sha of each posted `{VCSConfig, VCSTriggers}` body in `triggers.<seg>.sha` and does not re-POST an unchanged, already-set workflow (`--fresh` does; the endpoint upserts anyway, a 4xx "already exists" from older builds counts as success), and `sg_api_post` returns 22 on 4xx so `sg_retry` (via `SG_NO_RETRY_RC`) does not retry definitive errors. `clean` removes local artifacts (`export/`, TF state, tool cache); `clean --all` also removes config. `completion bash|zsh` prints a completion script (`cmd_completion`; commands + descriptions come from `SG_COMMAND_DESCS` (`SG_COMMANDS` is derived from it), options from `SG_OPTIONS`; keep them in sync with the parser and the host flags; the zsh script also works autoloaded from `$fpath` as `_sg-migrate.sh`); like `clean` it always runs natively. Output is concise by default — `apply` captures terraform's init/plan output and shows only progress + the final summary (or the full log on failure, `TF_IN_AUTOMATION=1`/`-no-color`); `-v`/`--verbose` (`SG_VERBOSE`) streams everything and un-gates the convert detail lines. Colored logging via `sg_step`/`sg_log`/`sg_success`/`sg_warn`/`sg_err`/`sg_row`; paths shown relative via `sg_rel` (all auto-off when not a TTY / `NO_COLOR`). Long steps show a live progress line (`sg_run_quiet` for terraform init/apply, `run_parallel <fn> <max> <label> <items>` for convert/import/triggers — animated only when `SG_ANIMATE=1`, i.e. colors are on; plain log lines otherwise) and report their duration (`sg_fmt_secs`). Phases start with `phase_begin` — inside `all` the header is numbered (`Phase 2/5: ...`, `PHASE_TOTAL` set in `main`) and `phase_took` gives the elapsed time; `run_parallel` prints a job's output as-is when it is a single line and adds the `── <file> ──` header only for longer or failed output (always with `-v`). The import confirmation goes through `sg_confirm` (so `SG_ANSWERS_FILE` works; a declined prompt is a neutral "cancelled" line, not an error) and `import`/`all` end with `finish_line` (outcome, total time, open checklist items from `CHECKLIST_OPEN`). A Ctrl-C is trapped to clear the progress line and print a resume hint. -- `scripts/lib/` — sourced libraries (a `lib/` gitignore rule is negated for this dir): `prompt.sh` (`sg_ask`/`sg_confirm`/`sg_select`; tty or `SG_ANSWERS_FILE` for tests; `SG_NONINTERACTIVE=1`/`-y`/no TTY → defaults), `tfvars.sh` (`tfvars_get` cached hcl2json reads — `tfvars_json` merges the run overlay (`TFVARS_OVERLAY_JSON`) over the file, and `tfvars_get`/`tfvars_get_json` keep a `false` value instead of treating it like a missing key —, `tfvars_has`/`tfvars_is_null` to tell an explicit `null` from a missing key, `tfvars_write` renders the wizard's `W_*` vars — `W_TF_SOURCE`, and `W_TF_VERSION`/`W_RUNNER_JSON` may be the literal `null`; plain strings go through `_tfvars_str`, so quotes/backslashes in names stay valid HCL; `projectOverrides`/`workspaceOverrides` are rendered from `W_PROJECT_OVERRIDES_JSON`/`W_WS_OVERRIDES_JSON` by `_tfvars_map` — `"key" = { attr = <json> }`, which hcl2json round-trips — so hand-written blocks survive a re-run of `init` (`wizard_run` reads them before writing; inner comments are lost); `tfWorkspaceIgnoreNames` is kept the same way (`W_IGNORE_NAMES_JSON`, never asked, shown in the review's scope row), and every other setting the wizard does not ask about (`tfProjects`, `cloudAuthVarPatterns`, hand-added keys) is appended verbatim from `W_PREV_JSON` under "Kept from the previous file"; `tfvars_missing_keys` (keys of `terraform.tfvars.example` the file lacks) and `tfvars_upgrade` (appends those paragraphs of the example — comment + default — under a dated header, `.bak` kept, touches nothing else; behind `init --upgrade`, prompt-free)), `scope.sh` (the run scope: `PROJECT_FILTER`/`WS_FILTER`/`WS_EXCLUDE`/`TAG_FILTER`/`TAG_EXCLUDE` arrays, `names_json`, `slug_of` (the transformer's project-slug rule in bash), `project_selected <seg>`, `ws_narrowed`/`ws_filter_json`/`ws_exclude_json` (tfvars `tfWorkspaceIgnoreNames` ∪ flags, memoized)/`ws_selected <name>` (bash `case` globs), `WS_SCOPE_JQ` + `ws_jq_args` → `WS_JQ_ARGS` for the jq sites, `scope_tags_json`/`scope_ignore_tags_json`, `scope_tfvar_args` → `SCOPE_TFVAR_ARGS` for apply, `scope_describe`, `scope_sha_input` for the apply phase hash; and the **run configuration overlay**: `--set KEY=VALUE` (`SET_VARS`, values via `_overlay_value`: HCL/JSON literals through hcl2json, bare text stays a string), `--cloud-connector`/`--vcs-connector`/`--runner-group`/`--workflow-group` → `overlay_build` looks the connector kinds up with `sg_list_integrations`/`sg_integration_type`/`sg_vcs_kind_of`, resolves `--project` values to raw TFC names (`_overlay_project_name`, fatal for an unknown project, as-given with a warning when TFC is unreachable) and fills `TFVARS_OVERLAY_JSON` — `projectOverrides[<name>]` fields with `--project`, `SGDefault*` keys without; `tfvars_json` deep-merges it over the file so every reader sees it, `scope_tfvar_args` passes each overlay key as `-var`, `scope_sha_input` includes it, `write_run_result` records it under `configuration`; `main` refuses the flags for `init`/`clean`/`completion` and a standalone `import` warns that they only reach the workflows through the export (`RAN_APPLY`)), `tfc_api.sh` (`tfc_http`/`tfc_get_all` paginated, `tfc_list_{orgs,projects,workspaces}` — workspaces include `vcs_provider`/`vcs_url`/`vcs_identifier` —, `tfc_select_workspaces` (the module's name-glob / include-tags(all) / exclude-tags / exclude-name-globs filter, shared by wizard and preflight), `tfc_vcs_summary`/`tfc_vcs_kind_for`/`tfc_vcs_label_for`, `tfc_token` with provider precedence, `require_tfc_auth`), `sg_api.sh` (`_sg_api` 0/22/1 contract + `SG_HTTP_CODE`, `_sg_listall` paginated listall reader (the API pages at 50) behind `sg_list_workflows`/`sg_list_wfgrps`, `sg_set_vcs_triggers`, `sg_list_integrations`, `sg_integration_type`, `sg_vcs_kind_of` (connector type → `sourceConfigDestKind`), `sg_execution_preset` + `sg_preset_{desc,runner_desc,version_desc,runner_provided}` (the org's execution preset and how to describe it), `sg_*_exists`, `sg_list_workflows`, `sg_create_secret`, `sg_patch_workflow`, `wfgroup_*`), `wizard.sh` (`wizard_run`: 5 steps TFC → SG → projects → defaults → review; `wizard_projects` (with >1 selected project, from `W_SEL_PROJECTS_JSON`) asks whether the global connectors and `tfc-<project>` groups apply to all, else per project the cloud connector, VCS connector (kind + prefix follow) and group (default / existing via `sg_list_wfgrps` / typed), writing `projectOverrides`; "same as the default" deletes the key, unmanaged fields are kept, previous picks come first (`_w_first`); the defaults step also asks `stripCloudAuthVars`; the TFC step keeps the workspace list and derives, via `tfc_vcs_summary`, which VCS provider the selected workspaces use and the repositories' URL prefix — the SG step lists matching connectors first and takes the repo URL prefix from TFC (never from a previous tfvars written for another provider kind), warns on a kind mismatch, reads the org's execution preset and offers it as the runner choice; the defaults step asks whether to carry TFC versions or use the preset (`SGTerraformVersionSource`) and, for carry, which fallback (`SGDefaultTerraformVersion` or the preset = null); the review shows human-readable rows plus the `tfc-<project>` groups; `_w_select_lines` feeds menus line-by-line so names with spaces/globs survive), `preflight.sh` (`preflight_run <apply|import|all>`, once per process; ✓ lines are plain language and name the connector's role/kind, ✗/! lines name the tfvars field to fix; connector ids, runner groups and DPC kinds are collected from `SGDefault*`, `projectOverrides` and `workspaceOverrides`; besides existence it checks consistency — VCS kind vs. the connector's type, globally and per `projectOverrides` entry (fail), repo URL prefix vs. where the TFC repositories live or the kind's well-known host (warn), `projectOverrides` keys vs. the TFC project names (warn), `--project` values vs. the project names/slugs (fail; a `tfProjects` typo warns), and the workspace-selection preview applies the CLI scope (names, excludes, tags, projects) so its count is what the apply will export; the import context states the group policy and warns when a project's group changed since its last import; it also reports the org's execution preset next to the version/runner policy and warns when `carry` meets a preset that mounts a runner-provided binary), `state.sh` (`.sg/state.json`: `phases.<name>.input_sha`, `import.<seg>.{group,imported,updated,failed,tf_fallback,state_uploaded,state_failed}` (`group` anchors the never-move check; `updated` = PATCHed on a 409), `triggers.<seg>.{set,unchanged,failed,missing,sha}`, `secrets.<name>`; `run_phase` in migrate.sh skips unchanged phases), `report.sh` (`show_migration_summary` after apply — incl. a state-files row —, `show_import_plan` per-workflow table with columns sized to their content — it keeps the rows in `PLAN_ROWS_TSV` (name, group, action, version, runner, triggers, vars, secrets, segment, workspace, project) —, `write_run_result <outcome>` → `export/run-result.json` + `run-summary.md` from `PLAN_ROWS_TSV`, `PLAN_GROUP_ROWS` (plain copy of the group table, set by `plan_groups`), `PLAN_PROBLEMS`, the merged state and `CHECKLIST_OPEN`; per workflow `result` is planned/skipped/failed/updated/created, with the probe workflow — planned `create`, PATCHed on its second pass — reported as created), `errors.sh` (`explain_api_error` regex→hint table, `SG_API_HINTS`), `checklist.sh` (`create_secret_stubs` → SG secret `tfc-<wf>-<VAR>` = `CHANGE_ME`, referenced as `${secret::<name>}` via PATCH + payload patch; `write_checklist` → `export/post-import-checklist.md`, printed to the terminal as one ✓/! status line per section plus the imported groups with UI links — the file has the details; section 5 covers both the TFC state export and the SG state upload (`state_failed` from the import results); sets `CHECKLIST_OPEN`). Parallel jobs (`run_parallel`) run in subshells, so `do_import`/`do_set_triggers` write `.import-result.<seg>.json` / `.triggers-result.<seg>.json` into the export dir and the caller merges them into state. +- `scripts/lib/` — sourced libraries (a `lib/` gitignore rule is negated for this dir): `prompt.sh` (`sg_ask`/`sg_confirm`/`sg_select`; tty or `SG_ANSWERS_FILE` for tests; `SG_NONINTERACTIVE=1`/`-y`/no TTY → defaults), `tfvars.sh` (`tfvars_get` cached hcl2json reads — `tfvars_json` merges the run overlay (`TFVARS_OVERLAY_JSON`) over the file, and `tfvars_get`/`tfvars_get_json` keep a `false` value instead of treating it like a missing key —, `tfvars_has`/`tfvars_is_null` to tell an explicit `null` from a missing key, `tfvars_write` renders the wizard's `W_*` vars — `W_TF_SOURCE`, and `W_TF_VERSION`/`W_RUNNER_JSON` may be the literal `null`; plain strings go through `_tfvars_str`, so quotes/backslashes in names stay valid HCL; `projectOverrides`/`workspaceOverrides` are rendered from `W_PROJECT_OVERRIDES_JSON`/`W_WS_OVERRIDES_JSON` by `_tfvars_map` — `"key" = { attr = <json> }`, which hcl2json round-trips — so hand-written blocks survive a re-run of `init` (`wizard_run` reads them before writing; inner comments are lost); `tfWorkspaceIgnoreNames` is kept the same way (`W_IGNORE_NAMES_JSON`, never asked, shown in the review's scope row), and every other setting the wizard does not ask about (`tfProjects`, `cloudAuthVarPatterns`, hand-added keys) is appended verbatim from `W_PREV_JSON` under "Kept from the previous file"; `tfvars_missing_keys` (keys of `terraform.tfvars.example` the file lacks) and `tfvars_upgrade` (appends those paragraphs of the example — comment + default — under a dated header, `.bak` kept, touches nothing else; behind `init --upgrade`, prompt-free)), `scope.sh` (the run scope: `PROJECT_FILTER`/`WS_FILTER`/`WS_EXCLUDE`/`TAG_FILTER`/`TAG_EXCLUDE` arrays, `names_json`, `slug_of` (the transformer's project-slug rule in bash), `project_selected <seg>`, `ws_narrowed`/`ws_filter_json`/`ws_exclude_json` (tfvars `tfWorkspaceIgnoreNames` ∪ flags, memoized)/`ws_selected <name>` (bash `case` globs), `WS_SCOPE_JQ` + `ws_jq_args` → `WS_JQ_ARGS` for the jq sites, `scope_tags_json`/`scope_ignore_tags_json`, `scope_tfvar_args` → `SCOPE_TFVAR_ARGS` for apply, `scope_describe`, `scope_sha_input` for the apply phase hash; and the **run configuration overlay**: `--set KEY=VALUE` (`SET_VARS`, values via `_overlay_value`: HCL/JSON literals through hcl2json, bare text stays a string), `--cloud-connector`/`--vcs-connector`/`--runner-group`/`--workflow-group` → `overlay_build` looks the connector kinds up with `sg_list_integrations`/`sg_integration_type`/`sg_vcs_kind_of`, resolves `--project` values to raw TFC names (`_overlay_project_name`, fatal for an unknown project, as-given with a warning when TFC is unreachable) and fills `TFVARS_OVERLAY_JSON` — `projectOverrides[<name>]` fields with `--project`, `SGDefault*` keys without; `tfvars_json` deep-merges it over the file so every reader sees it, `scope_tfvar_args` passes each overlay key as `-var`, `scope_sha_input` includes it, `write_run_result` records it under `configuration`; `main` refuses the flags for `init`/`clean`/`completion` and a standalone `import` warns that they only reach the workflows through the export (`RAN_APPLY`)), `tfc_api.sh` (`tfc_http`/`tfc_get_all` paginated, `tfc_list_{orgs,projects,workspaces}` — workspaces include `vcs_provider`/`vcs_url`/`vcs_identifier` —, `tfc_select_workspaces` (the module's name-glob / include-tags(all) / exclude-tags / exclude-name-globs filter, shared by wizard and preflight), `tfc_vcs_summary`/`tfc_vcs_kind_for`/`tfc_vcs_label_for`, `tfc_token` with provider precedence, `require_tfc_auth`), `sg_api.sh` (`_sg_api` 0/22/1 contract + `SG_HTTP_CODE`, `_sg_listall` paginated listall reader (the API pages at 50) behind `sg_list_workflows`/`sg_list_wfgrps`, `sg_set_vcs_triggers`, `sg_list_integrations`, `sg_integration_type`, `sg_vcs_kind_of` (connector type → `sourceConfigDestKind`), `sg_execution_preset` + `sg_preset_{desc,runner_desc,version_desc,runner_provided}` (the org's execution preset and how to describe it), `sg_*_exists`, `sg_list_workflows`, `sg_create_secret`, `sg_patch_workflow`, `wfgroup_*`), `wizard.sh` (`wizard_run`: 5 steps TFC → SG → projects → defaults → review; `wizard_projects` (with >1 selected project, from `W_SEL_PROJECTS_JSON`) asks whether the global connectors and `tfc-<project>` groups apply to all, else per project the cloud connector, VCS connector (kind + prefix follow) and group (default / existing via `sg_list_wfgrps` / typed), writing `projectOverrides`; "same as the default" deletes the key, unmanaged fields are kept, previous picks come first (`_w_first`); the defaults step also asks `stripCloudAuthVars`; the TFC step first asks which TFC projects to migrate (with >1 project: `all` or `pick` by name/slug, resolved to raw names, unknown names re-asked; a previous non-empty `tfProjects` makes `pick` the first item; free text without discovery) → `W_PROJECTS_JSON`, rendered as `tfProjects` (`[]` = all) and applied to the workspace selection before the scope question; it keeps the workspace list and derives, via `tfc_vcs_summary`, which VCS provider the selected workspaces use and the repositories' URL prefix — the SG step lists matching connectors first and takes the repo URL prefix from TFC (never from a previous tfvars written for another provider kind), warns on a kind mismatch, reads the org's execution preset and offers it as the runner choice; the defaults step asks whether to carry TFC versions or use the preset (`SGTerraformVersionSource`) and, for carry, which fallback (`SGDefaultTerraformVersion` or the preset = null); the review shows human-readable rows plus the `tfc-<project>` groups; `_w_select_lines` feeds menus line-by-line so names with spaces/globs survive), `preflight.sh` (`preflight_run <apply|import|all>`, once per process; ✓ lines are plain language and name the connector's role/kind, ✗/! lines name the tfvars field to fix; connector ids, runner groups and DPC kinds are collected from `SGDefault*`, `projectOverrides` and `workspaceOverrides`; besides existence it checks consistency — VCS kind vs. the connector's type, globally and per `projectOverrides` entry (fail), repo URL prefix vs. where the TFC repositories live or the kind's well-known host (warn), `projectOverrides` keys vs. the TFC project names (warn), `--project` values vs. the project names/slugs (fail; a `tfProjects` typo warns), and the workspace-selection preview applies the CLI scope (names, excludes, tags, projects) so its count is what the apply will export; the import context states the group policy and warns when a project's group changed since its last import; it also reports the org's execution preset next to the version/runner policy and warns when `carry` meets a preset that mounts a runner-provided binary), `state.sh` (`.sg/state.json`: `phases.<name>.input_sha`, `import.<seg>.{group,imported,updated,failed,tf_fallback,state_uploaded,state_failed}` (`group` anchors the never-move check; `updated` = PATCHed on a 409), `triggers.<seg>.{set,unchanged,failed,missing,sha}`, `secrets.<name>`; `run_phase` in migrate.sh skips unchanged phases), `report.sh` (`show_migration_summary` after apply — incl. a state-files row —, `show_import_plan` per-workflow table with columns sized to their content — it keeps the rows in `PLAN_ROWS_TSV` (name, group, action, version, runner, triggers, vars, secrets, segment, workspace, project) —, `write_run_result <outcome>` → `export/run-result.json` + `run-summary.md` from `PLAN_ROWS_TSV`, `PLAN_GROUP_ROWS` (plain copy of the group table, set by `plan_groups`), `PLAN_PROBLEMS`, the merged state and `CHECKLIST_OPEN`; per workflow `result` is planned/skipped/failed/updated/created, with the probe workflow — planned `create`, PATCHed on its second pass — reported as created), `errors.sh` (`explain_api_error` regex→hint table, `SG_API_HINTS`), `checklist.sh` (`create_secret_stubs` → SG secret `tfc-<wf>-<VAR>` = `CHANGE_ME`, referenced as `${secret::<name>}` via PATCH + payload patch; `write_checklist` → `export/post-import-checklist.md`, printed to the terminal as one ✓/! status line per section plus the imported groups with UI links — the file has the details; section 5 covers both the TFC state export and the SG state upload (`state_failed` from the import results); sets `CHECKLIST_OPEN`). Parallel jobs (`run_parallel`) run in subshells, so `do_import`/`do_set_triggers` write `.import-result.<seg>.json` / `.triggers-result.<seg>.json` into the export dir and the caller merges them into state. - `scripts/tools.sh` — sourced by the other scripts (sets `SG_REPO_ROOT` to its parent dir). Provides `sg_resolve` (PATH-or-cache), `sg_ensure_{jq,hcl2json,yajsv,sgcli}` (pinned downloads into `.sg/cached/`; `sg-cli` is the Go binary), `sg_retry`, the color vars + log helpers. The Docker image bundles these tools at build time (`yajsv` built from source so arm64 works), so in-container runs never download. - **Variable sets** — `scripts/enrich_variable_sets.sh <tfc-org> <payload...>` (run by `cmd_enrich`, which reads `tfOrg`/`tfHostname` from `terraform.tfvars` via `hcl2json` — variable sets live in the **TFC** org, not `SG_ORG`). Lists all sets + vars via the TFC API (Bearer token from the `terraform login` creds file or `TFE_TOKEN`), computes per-workspace effective vars (scope: global/project/workspace; precedence: priority sets > workspace vars > non-priority sets), and merges them into each workflow (matched by workspace name parsed from `TfStateFilePath`). Sensitive set vars and key conflicts are reported. Skipped by `--no-variable-sets`. - **Regions** — `--region eu|us` / `SG_REGION` (`region_urls`/`region_apply` in migrate.sh) pick `SG_BASE_URL` and `SG_UI_URL` together: eu = `api.app`/`app.stackguardian.io` (default), us = `api.us`/`us.stackguardian.io`, and the undocumented internal `dash` = `testapi.qa`/`dash.qa.stackguardian.io` (never mention it in user-facing docs). An explicit `SG_BASE_URL`/`SG_UI_URL` wins over the region. `init` stores `config.sg_region` in `.sg/state.json` (older files only have `sg_base_url`, still honoured); the plan header and `run-result.json` (`region`, `apiUrl`, `uiUrl`) show what a run used. The wrapper forwards `SG_REGION`. diff --git a/README.md b/README.md index 4b22016..ad0850f 100644 --- a/README.md +++ b/README.md @@ -28,7 +28,7 @@ export SG_ORG=<your SG org> ./sg-migrate.sh all # preflight -> apply -> enrich -> convert -> validate -> import (shows a plan, asks before importing) ``` -That's it — no IDs to look up and no workflow-group mapping to fill in. `init` lists what the tokens can see (TFC organisations and workspaces, SG VCS/cloud connectors and runner groups, the org's execution preset) and writes `terraform.tfvars` from your picks — it also reads which VCS provider your TFC workspaces are connected to, lists the matching connectors first and takes the repository URL prefix from TFC. With more than one TFC project it asks whether the same connectors and groups apply to every project, or lets you pick a cloud connector, a VCS connector and the workflow group per project (see *Per-project settings* below). `all` verifies every reference **before** running terraform (preflight — including that the VCS connector, the VCS kind and the repo URL prefix agree with each other and with the TFC repositories), prints a migration summary after the export, shows a per-workflow import plan (create/update, Terraform version, runner, triggers, secrets), and ends with a **post-import checklist** of what still needs a human. Phases are numbered, long steps show a live progress line, and every phase reports how long it took. Each TFC project is imported into the SG workflow group assigned to it — `tfc-<project>` by default, or the group you picked for that project — **reusing the group when it exists and creating it otherwise**. Workflows are never moved between groups: when a project's workflows already live in another group, the plan stops and says so. +That's it — no IDs to look up and no workflow-group mapping to fill in. `init` lists what the tokens can see (TFC organisations, projects and workspaces, SG VCS/cloud connectors and runner groups, the org's execution preset), asks which TFC projects to migrate (all, or some by name — `tfProjects`, `[]` = all) and then which of their workspaces, and writes `terraform.tfvars` from your picks — it also reads which VCS provider your TFC workspaces are connected to, lists the matching connectors first and takes the repository URL prefix from TFC. With more than one TFC project it asks whether the same connectors and groups apply to every project, or lets you pick a cloud connector, a VCS connector and the workflow group per project (see *Per-project settings* below). `all` verifies every reference **before** running terraform (preflight — including that the VCS connector, the VCS kind and the repo URL prefix agree with each other and with the TFC repositories), prints a migration summary after the export, shows a per-workflow import plan (create/update, Terraform version, runner, triggers, secrets), and ends with a **post-import checklist** of what still needs a human. Phases are numbered, long steps show a live progress line, and every phase reports how long it took. Each TFC project is imported into the SG workflow group assigned to it — `tfc-<project>` by default, or the group you picked for that project — **reusing the group when it exists and creating it otherwise**. Workflows are never moved between groups: when a project's workflows already live in another group, the plan stops and says so. - **Region.** StackGuardian runs in the EU (`api.app.stackguardian.io` / `app.stackguardian.io`, the default) and the US (`api.us.stackguardian.io` / `us.stackguardian.io`). Pass `--region us` (or set `SG_REGION=us`) on `init` and it is remembered for every later run; the import plan and `export/run-result.json` show which API the run used. `SG_BASE_URL` / `SG_UI_URL` override the region's hosts for private or test environments. - **Updating.** Clone the repo (don't fork it or download the release zip) and run `./sg-migrate.sh update` to pull the latest version; it fast-forwards the checkout and rebuilds the Docker image only if the `Dockerfile` changed. Your `terraform.tfvars`, `export/` and `.sg/` are never tracked, so they survive every update. To stay on a fixed release instead, `git checkout v1.2.2` (then `git checkout master` to follow the latest again). diff --git a/scripts/lib/sg_api.sh b/scripts/lib/sg_api.sh index 5c3ae23..b306656 100644 --- a/scripts/lib/sg_api.sh +++ b/scripts/lib/sg_api.sh @@ -112,9 +112,12 @@ _sg_listall() { jqb="$(sg_resolve jq sg_ensure_jq)" while :; do body="$(sg_api_get "$(sg_org_url)/${path}?limit=100${key:+&lastevaluatedkey=$key}" 2>/dev/null)" || break - page="$(printf '%s' "$body" | "$jqb" -c '[(if (.msg | type) == "array" then .msg elif (.data | type) == "array" then .data elif (.data.Workflows? | type) == "array" then .data.Workflows elif type == "array" then . else [] end)[] | '"$expr"' | select(. != null and . != "")]' 2>/dev/null)" || page='[]' - acc="$("$jqb" -nc --argjson a "$acc" --argjson b "$page" '$a + $b')" - key="$(printf '%s' "$body" | "$jqb" -r '.lastevaluatedkey // empty' 2>/dev/null | "$jqb" -sRr @uri)" + # -s: an empty or non-JSON body (a 200 for a group that does not exist yet) + # must yield [] rather than an empty string that --argjson rejects. + page="$(printf '%s' "$body" | "$jqb" -sc '(.[0] // {}) as $b | [(if ($b | type) == "array" then $b elif ($b.msg | type) == "array" then $b.msg elif ($b.data | type) == "array" then $b.data elif ($b.data.Workflows? | type) == "array" then $b.data.Workflows else [] end)[] | '"$expr"' | select(. != null and . != "")]' 2>/dev/null)" || page='[]' + [ -n "$page" ] || page='[]' + acc="$("$jqb" -nc --argjson a "$acc" --argjson b "$page" '$a + $b' 2>/dev/null)" || acc="$page" + key="$(printf '%s' "$body" | "$jqb" -sr '(.[0] // {}) | if (.lastevaluatedkey | type) == "string" then .lastevaluatedkey else empty end' 2>/dev/null | "$jqb" -sRr @uri)" [ -n "$key" ] || break done printf '%s' "$acc" diff --git a/scripts/lib/tfvars.sh b/scripts/lib/tfvars.sh index 6480a74..c127c59 100644 --- a/scripts/lib/tfvars.sh +++ b/scripts/lib/tfvars.sh @@ -117,7 +117,7 @@ _tfvars_str() { # wizard (lists/objects are passed as compact JSON, which HCL accepts). Keeps # the same order and comments as terraform.tfvars.example so the file stays # hand-editable afterwards. -# W_TFORG W_TFHOST W_WSNAMES_JSON W_TAGS_JSON W_IGNORE_TAGS_JSON W_IGNORE_NAMES_JSON W_EXPORT_STATE +# W_TFORG W_TFHOST W_PROJECTS_JSON W_WSNAMES_JSON W_TAGS_JSON W_IGNORE_TAGS_JSON W_IGNORE_NAMES_JSON W_EXPORT_STATE # W_APPROVERS_JSON W_REPO_PREFIX W_VCS_INTEGRATION W_DPC_JSON W_RUNNER_JSON # W_DEST_KIND W_TF_SOURCE W_TF_VERSION W_TRIGGERS W_IGNORE_PATTERNS_JSON # W_STRIP_CLOUD W_PROJECT_OVERRIDES_JSON W_WS_OVERRIDES_JSON @@ -156,6 +156,10 @@ tfWorkspaceIgnoreTags = $W_IGNORE_TAGS_JSON # the filters above. sg-migrate.sh --exclude-workspace adds to this list for one run. tfWorkspaceIgnoreNames = ${W_IGNORE_NAMES_JSON:-[]} +# Only these TFC projects, by name ([] = every project; each project becomes one +# workflow group). sg-migrate.sh --project narrows a single run further. +tfProjects = ${W_PROJECTS_JSON:-[]} + # Directory to export Terraform files to exportPath = "export" diff --git a/scripts/lib/wizard.sh b/scripts/lib/wizard.sh index 3e6e545..1a63334 100644 --- a/scripts/lib/wizard.sh +++ b/scripts/lib/wizard.sh @@ -117,6 +117,7 @@ wizard_tfc() { W_TFC_VCS_OTHER=0 W_SEL_PROJECTS_JSON='[]' W_SEL_WS_JSON='[]' + W_PROJECTS_JSON='[]' projects='[]' if [ "$W_TFC_DISCOVERY" -eq 1 ] && ws="$(tfc_list_workspaces "$W_TFORG" 2>/dev/null)"; then W_TFC_WS_JSON="$ws" @@ -129,6 +130,46 @@ wizard_tfc() { else alltags="" fi + + # Projects first: a TFC project becomes one SG workflow group, and the + # workspace questions below apply within the chosen projects. Written as + # tfProjects ([] = every project); --project narrows a run further. + local prev_projects pick names n + prev_projects="$(tfvars_get_json .tfProjects)" + [ "$prev_projects" = "null" ] && prev_projects='[]' + if [ "${prn:-0}" -gt 1 ]; then + n=0 + while IFS=$'\t' read -r name cnt; do + [ -n "$name" ] || continue + n=$((n + 1)) + sg_dim " $n) $name ($cnt workspace(s))" + done <<<"$(printf '%s' "$W_TFC_WS_JSON" | "$jqb" -r --argjson pr "$projects" ' + ($pr | map({key: .id, value: .name}) | from_entries) as $names + | group_by(.project) | sort_by(-length) | .[] | "\($names[.[0].project] // .[0].project)\t\(length)"')" + if [ "$prev_projects" = "[]" ]; then + pick="$(sg_select "Which TFC projects should be migrated?" "all|every project ($prn)" "pick|some of them, by name")" || return 1 + else + pick="$(sg_select "Which TFC projects should be migrated?" "pick|some of them, by name (currently: $(_w_csv "$prev_projects"))" "all|every project ($prn)")" || return 1 + fi + if [ "$pick" = "pick" ]; then + while :; do + names="$(sg_ask_required "Project names (comma-separated; name or slug)")" || return 1 + # Resolve to the raw project names; unknown entries are re-asked. + W_PROJECTS_JSON="$(printf '%s' "$projects" | "$jqb" -c --argjson want "$(_w_csv_json "$names")" ' + map({name, slug: (.name | ascii_downcase | gsub("[^a-z0-9-]+"; "-"))}) as $pr + | [$want[] | . as $w | ($w | ascii_downcase | gsub("[^a-z0-9-]+"; "-")) as $s + | ($pr[] | select(.slug == $s) | .name) // ("?" + $w)]')" + if printf '%s' "$W_PROJECTS_JSON" | "$jqb" -e 'any(.[]; startswith("?"))' >/dev/null; then + sg_warn "no such project: $(printf '%s' "$W_PROJECTS_JSON" | "$jqb" -r '[.[] | select(startswith("?")) | .[1:]] | join(", ")') — projects: $(printf '%s' "$projects" | "$jqb" -r '[.[].name] | join(", ")')" + continue + fi + break + done + fi + elif [ "$W_TFC_DISCOVERY" -eq 0 ]; then + names="$(sg_ask "TFC projects to migrate (comma-separated names; empty = every project)" "$(_w_csv "$prev_projects")")" || return 1 + [ -n "$names" ] && W_PROJECTS_JSON="$(_w_csv_json "$names")" + fi scope="$(sg_select "Which workspaces should be migrated?" \ "all|every workspace in the organisation" \ "tags|only workspaces carrying certain tags" \ @@ -155,6 +196,12 @@ wizard_tfc() { # What the selection looks like (drives the review and the SG step's hints). if [ -n "$W_TFC_WS_JSON" ]; then sel="$(tfc_select_workspaces "$W_TFC_WS_JSON" "$W_WSNAMES_JSON" "$W_TAGS_JSON" "$W_IGNORE_TAGS_JSON" "${W_IGNORE_NAMES_JSON:-[]}")" + # Within the chosen projects only (tfProjects), like the transformer does. + if [ "$W_PROJECTS_JSON" != "[]" ]; then + sel="$(printf '%s' "$sel" | "$jqb" -c --argjson pr "$projects" --argjson want "$W_PROJECTS_JSON" ' + ($pr | map(select(.name as $n | $want | index($n) != null) | .id)) as $ids + | map(select(.project as $p | $ids | index($p) != null))')" + fi W_WS_TOTAL="$wsn" W_WS_COUNT="$(printf '%s' "$sel" | "$jqb" 'length')" W_WS_ABOVE_CEILING="$(printf '%s' "$sel" | "$jqb" '[.[] | select((.terraform_version // "") | test("^[0-9]+\\.[0-9]+\\.[0-9]+$")) | select(((.terraform_version | split(".") | map(tonumber)) as $v | ($v[0] > 1) or ($v[0] == 1 and $v[1] > 5) or ($v[0] == 1 and $v[1] == 5 and $v[2] > 7)))] | length')" @@ -546,6 +593,11 @@ wizard_review() { if [ -n "${W_WS_COUNT:-}" ]; then if [ "${W_SCOPE:-all}" = "all" ] && [ "${W_IGNORE_NAMES_JSON:-[]}" = "[]" ]; then scope="$scope ($W_WS_COUNT)"; else scope="$scope — $W_WS_COUNT of $W_WS_TOTAL match"; fi fi + if [ "${W_PROJECTS_JSON:-[]}" = "[]" ]; then + sg_row "TFC projects" "all (tfProjects = [])" + else + sg_row "TFC projects" "$(_w_csv "$W_PROJECTS_JSON")" + fi sg_row "Workspaces" "$scope" if [ -n "${W_PROJECT_ROWS:-}" ]; then while IFS='|' read -r k line; do [ -n "$k" ] && sg_row "Project '$k'" "$line"; done <<<"$W_PROJECT_ROWS" From 2e5a0ec81f109d807cf5d0884089b448b8257103 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adis=20Halilovi=C4=87?= <adis.halilovic@stackguardian.io> Date: Mon, 14 Sep 2026 18:05:30 +0200 Subject: [PATCH 59/71] fix: cloudAuthVarPatterns is an advanced setting; preflight and init --upgrade leave it alone Every user saw a warning that their file predates cloudAuthVarPatterns, a list nobody needs to edit for a normal migration. It stays documented in variables.tf and the example, --set and a hand edit still work, but it is no longer reported as missing or appended. --- scripts/lib/preflight.sh | 2 +- scripts/lib/tfvars.sh | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/scripts/lib/preflight.sh b/scripts/lib/preflight.sh index bc17b36..7af4e00 100644 --- a/scripts/lib/preflight.sh +++ b/scripts/lib/preflight.sh @@ -400,7 +400,7 @@ preflight_run() { local missing missing="$(tfvars_missing_keys | tr '\n' ' ')" if [ -n "$missing" ]; then - pf_warn "$(sg_rel "$TFVARS") predates these settings (defaults apply): ${missing% }— '$PROG init --upgrade' appends them with their defaults and comments" + pf_warn "$(sg_rel "$TFVARS") predates these settings (defaults apply): ${missing% } — '$PROG init --upgrade' appends them with their defaults and comments" fi case "$ctx" in apply) diff --git a/scripts/lib/tfvars.sh b/scripts/lib/tfvars.sh index c127c59..20d757e 100644 --- a/scripts/lib/tfvars.sh +++ b/scripts/lib/tfvars.sh @@ -263,6 +263,11 @@ tfvars_variables_tf() { printf '%s' "${TFVARS_VARIABLES_TF:-$SG_REPO_ROOT/transf # tfvars_variable_names — every variable the module declares, in file order. tfvars_variable_names() { sed -nE 's/^variable "([^"]+)".*/\1/p' "$(tfvars_variables_tf)"; } +# Advanced settings nobody needs in a normal file: never reported as missing +# and never appended by init --upgrade (they stay documented in variables.tf +# and the example, and --set / a hand edit still work). +TFVARS_ADVANCED_KEYS="cloudAuthVarPatterns" + # _tfvars_variable_attr <name> <attr> — a single-line attribute of a variable # block (`default = []`, `description = "..."`); empty when absent or when the # value spans several lines (a `{`/`[` with nothing after it). @@ -285,6 +290,7 @@ tfvars_missing_keys() { local have have="$(tfvars_json | "$(sg_resolve jq sg_ensure_jq)" -r 'keys[]')" tfvars_variable_names | while IFS= read -r k; do + case " $TFVARS_ADVANCED_KEYS " in *" $k "*) continue ;; esac grep -qx -- "$k" <<<"$have" && continue grep -qE "^# ?$k *=" "$TFVARS" 2>/dev/null && continue printf '%s\n' "$k" From cd9663f6fcc240194bcb103836f9193eec7720a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adis=20Halilovi=C4=87?= <adis.halilovic@stackguardian.io> Date: Mon, 14 Sep 2026 18:11:40 +0200 Subject: [PATCH 60/71] fix: existing workflows are PATCHed directly instead of failing a create first The plan already knew a workflow existed (ACTION update), yet the import still sent it through sg-cli's create and waited for the 409 to switch to PATCH, printing a failure line on every re-run. import_bulk now lists the group once, updates the known workflows straight away (state re-uploaded) and only creates the rest; the 409 handling stays as the safety net for a race between plan and import. Update failures print 'Failed to update' and are parsed like create failures. Also drops the empty 'run scope:' line when only configuration flags are given. --- CLAUDE.md | 2 +- README.md | 2 +- scripts/migrate.sh | 76 +++++++++++++++++++++++++++++++++++----------- 3 files changed, 60 insertions(+), 20 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index a2d5399..acbfa9c 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -24,7 +24,7 @@ The migration is a user-driven pipeline, not a single program: The five phases are wrapped by an orchestrator so users don't run them by hand: - `sg-migrate.sh` (host entrypoint, repo root) — runs `scripts/migrate.sh` **inside the Docker image** (`Dockerfile`), bind-mounting the repo at `/app` and forwarding `SG_API_TOKEN`/`SG_ORG`/`SG_BASE_URL`/`TFE_TOKEN`/`TF_TOKEN_*`. `--tfvars FILE` (or `SG_TFVARS`) is a host path: the wrapper strips the flag, resolves it and passes it on as `SG_TFVARS` — rewritten to `/app/<relative>` when the file is inside the checkout, otherwise mounted read-only at `/tmp/sg-run.tfvars`. For TFC auth it prefers a long-lived `TFE_TOKEN` (env); otherwise it mounts `~/.terraform.d/credentials.tfrc.json` read-only. Runs natively instead when `--native`/`--local` is passed, `SG_NATIVE=1` is set, the command is `clean`/`completion`/help (or no command is given), or Docker is absent. Exports `SG_PROG` so `migrate.sh` shows `./sg-migrate.sh` in its usage/hints. `update` is host-only too (`cmd_update` in the wrapper, never passed to `migrate.sh`): refuses a non-git checkout, a detached HEAD or dirty tracked files, then `git pull --ff-only` and rebuilds the image only when the `Dockerfile` changed between the old and new HEAD — this is how customers pick up fixes from a plain clone (untracked config/output survive). `migrate.sh` lists `update` in its usage/`SG_COMMANDS` and rejects it with a host-only hint. `.gitattributes` forces LF so a Git-for-Windows clone does not CRLF the scripts. -- `scripts/migrate.sh` — the actual orchestrator (sources `scripts/lib/*.sh`, see below). Subcommands `init|preflight|apply|enrich|convert|validate|import|triggers|checklist|all|clean|completion` (no command prints the help menu; `enrich` runs in `all` unless `--no-variable-sets`). Flags beyond the basics: `--dry-run` (import, or all: the local export runs, then the plan is shown and nothing is created in SG), `--fresh` (ignore run state), the scope flags from `lib/scope.sh` — `--project NAME|SLUG`, `--workspace GLOB`, `--exclude-workspace GLOB`, `--tag NAME`, `--exclude-tag NAME`, all repeatable; tfvars holds the widest scope, include flags replace the tfvars list, exclude flags add to it; apply passes them as `-var` (`tfProjects`, `workspacenames`, `tfWorkspaceIgnoreNames`, `tfWorkspaceTags`, `tfWorkspaceIgnoreTags`), the later phases apply the same names/excludes to the payload entries (`ws_selected` in bash, `WS_SCOPE_JQ` + `WS_JQ_ARGS` in jq) and `--project` to the payload files by slug (`project_selected`); `--workspace '*'` alone is not a narrowing, so the unchanged-file skip still applies; a scope that matches nothing is an error —, `--tfvars FILE` (`TFVARS`, absolute; also `-var-file` for terraform and exported to the enrich script; `clean --all` only removes the module's own file), `--skip-preflight`, `--no-secret-stubs`, `--upgrade` (with `init`: `tfvars_upgrade`, append the settings an older tfvars lacks; preflight warns about them until then), and the run configuration flags `--set KEY=VALUE`, `--cloud-connector ID`, `--vcs-connector ID`, `--runner-group NAME|shared`, `--workflow-group NAME` (one-run overlay on the tfvars, see `scope.sh`; connector kinds looked up in SG; with `--project` they are that project's `projectOverrides`, else the `SGDefault*` values). `import`/`all` end by writing `export/run-result.json` + `run-summary.md` (`write_run_result`, outcomes planned/blocked/success/failed) for CI. Resolves each tool via `sg_resolve` (PATH first — the image installs them — else `sg_ensure_*` cache). `apply`/`all` first run `require_tfc_auth`, which resolves the token the tfe provider will use (`TFE_TOKEN`, `TF_TOKEN_<host>`, or the `terraform login` file for `tfHostname`) and verifies it with `GET /api/v2/account/details`, failing fast on a missing or rejected (expired) credential. Runs `convert` and `import` in parallel (`--concurrency`, default 4), raises `terraform apply -parallelism`, and retries `sg-cli` imports with backoff (`sg_retry`, `SG_RETRIES`). Import resolves each project's workflow group with `group_for` (legacy `.sg/workflow-groups.json` entry → the payload's `CLIConfiguration.WorkflowGroup.name` via `payload_group` → `tfc-<seg>`) and runs `plan_groups`: STATUS `reuse` (exists) / `create` (missing; created after the confirmation) / `missing!` (`--no-create-groups`) / `moved!`; it collects `PLAN_PROBLEMS` — a project whose workflows still live in the group recorded in `state.json` (or the default group) when the target changed (SG cannot move workflows), and two projects sharing a group with overlapping names — prints them as ✗ lines, still shows the per-workflow plan (also under `--dry-run`) and then dies. The skip set (`state_import_done`) is computed before the plan so the ACTION column can say `skip`/`update`/`create`. Groups are never PATCHed. Imports go through `import_bulk`: workflows with Terraform variables via `sg-cli workflow create --bulk`, workflows **without** via a direct `POST .../wfs/` (`sg_create_workflow` + `sg_upload_tfstate` in `sg_api.sh`) because sg-cli drops an empty `iacInputData.data` and the API rejects the workflow — a workaround (`TODO(sg-cli)` markers) until sg-cli ships with sg-sdk-go >= v1.5.7, which sends the empty object; the direct path prints sg-cli-style `Failed to create <name>: <code>: <body>` lines so the parsing below covers both. State files are the migrator's job on both paths: `import_bulk` tracks sg-cli's per-workflow upload lines (`State file uploaded successfully` / `Failed to upload state file for <wf>`), re-uploads the failures via `upload_state` → `sg_upload_tfstate` (adds the `x-ms-blob-type: BlockBlob` header Azure Blob requires; sg-cli omits it and also treats anything but a literal `HTTP/1.1 200 OK` as failure — second `TODO(sg-cli)`), and writes `[state] uploaded|failed|none <wf>` markers into the parsed output; `do_import` turns them into `state_uploaded`/`state_failed` in the result, a workflow without its state fails the file (and `state_import_done` re-tries it next run). Existing workflows come back from sg-cli as `Failed to create <wf>: 409: Workflow ID not unique` (its update branch matches the outdated text `Workflow name not unique` — third `TODO(sg-cli)`); `import_bulk` collects those, PATCHes them via `sg_update_workflow`, re-uploads their state and drops the failure line, and `sg_create_workflow` does the same fallback on the direct path (prints `updated`). Before the parallel import, `probe_import` imports one workflow alone (first selected of the first file) and dies unless the create and the state upload both succeeded — the fail-fast for environment problems. sg-cli exits 0 even when individual workflows fail, so `do_import` parses its output: a workflow rejected as above SG's managed Terraform ceiling (1.5.7, last MPL/FOSS release) is re-imported with `SGDefaultTerraformVersion` (read from `terraform.tfvars`; an explicit `null` there means the `terraformVersion` key is deleted instead, so the org's execution preset decides — `tfvars_is_null` tells an explicit null from a missing key), the payload patched in place, and the case logged to `export/terraform-version-fallbacks.log` plus a printed notice; any other per-workflow failure fails the run. `cmd_import` reads the org's execution preset (`sg_execution_preset`, `GET /orgs/{org}/` → `Settings.workflowDefaults`) and `preset_labels` (report.sh) turns it into the `preset (1.5.7)` / `preset (private:rg)` cells of the plan and the `SG_TF_FALLBACK_LABEL` used in messages. The trigger pass (`do_set_triggers` → `sg_set_vcs_triggers`) skips workflows that do not exist in SG, keeps the sha of each posted `{VCSConfig, VCSTriggers}` body in `triggers.<seg>.sha` and does not re-POST an unchanged, already-set workflow (`--fresh` does; the endpoint upserts anyway, a 4xx "already exists" from older builds counts as success), and `sg_api_post` returns 22 on 4xx so `sg_retry` (via `SG_NO_RETRY_RC`) does not retry definitive errors. `clean` removes local artifacts (`export/`, TF state, tool cache); `clean --all` also removes config. `completion bash|zsh` prints a completion script (`cmd_completion`; commands + descriptions come from `SG_COMMAND_DESCS` (`SG_COMMANDS` is derived from it), options from `SG_OPTIONS`; keep them in sync with the parser and the host flags; the zsh script also works autoloaded from `$fpath` as `_sg-migrate.sh`); like `clean` it always runs natively. Output is concise by default — `apply` captures terraform's init/plan output and shows only progress + the final summary (or the full log on failure, `TF_IN_AUTOMATION=1`/`-no-color`); `-v`/`--verbose` (`SG_VERBOSE`) streams everything and un-gates the convert detail lines. Colored logging via `sg_step`/`sg_log`/`sg_success`/`sg_warn`/`sg_err`/`sg_row`; paths shown relative via `sg_rel` (all auto-off when not a TTY / `NO_COLOR`). Long steps show a live progress line (`sg_run_quiet` for terraform init/apply, `run_parallel <fn> <max> <label> <items>` for convert/import/triggers — animated only when `SG_ANIMATE=1`, i.e. colors are on; plain log lines otherwise) and report their duration (`sg_fmt_secs`). Phases start with `phase_begin` — inside `all` the header is numbered (`Phase 2/5: ...`, `PHASE_TOTAL` set in `main`) and `phase_took` gives the elapsed time; `run_parallel` prints a job's output as-is when it is a single line and adds the `── <file> ──` header only for longer or failed output (always with `-v`). The import confirmation goes through `sg_confirm` (so `SG_ANSWERS_FILE` works; a declined prompt is a neutral "cancelled" line, not an error) and `import`/`all` end with `finish_line` (outcome, total time, open checklist items from `CHECKLIST_OPEN`). A Ctrl-C is trapped to clear the progress line and print a resume hint. +- `scripts/migrate.sh` — the actual orchestrator (sources `scripts/lib/*.sh`, see below). Subcommands `init|preflight|apply|enrich|convert|validate|import|triggers|checklist|all|clean|completion` (no command prints the help menu; `enrich` runs in `all` unless `--no-variable-sets`). Flags beyond the basics: `--dry-run` (import, or all: the local export runs, then the plan is shown and nothing is created in SG), `--fresh` (ignore run state), the scope flags from `lib/scope.sh` — `--project NAME|SLUG`, `--workspace GLOB`, `--exclude-workspace GLOB`, `--tag NAME`, `--exclude-tag NAME`, all repeatable; tfvars holds the widest scope, include flags replace the tfvars list, exclude flags add to it; apply passes them as `-var` (`tfProjects`, `workspacenames`, `tfWorkspaceIgnoreNames`, `tfWorkspaceTags`, `tfWorkspaceIgnoreTags`), the later phases apply the same names/excludes to the payload entries (`ws_selected` in bash, `WS_SCOPE_JQ` + `WS_JQ_ARGS` in jq) and `--project` to the payload files by slug (`project_selected`); `--workspace '*'` alone is not a narrowing, so the unchanged-file skip still applies; a scope that matches nothing is an error —, `--tfvars FILE` (`TFVARS`, absolute; also `-var-file` for terraform and exported to the enrich script; `clean --all` only removes the module's own file), `--skip-preflight`, `--no-secret-stubs`, `--upgrade` (with `init`: `tfvars_upgrade`, append the settings an older tfvars lacks; preflight warns about them until then), and the run configuration flags `--set KEY=VALUE`, `--cloud-connector ID`, `--vcs-connector ID`, `--runner-group NAME|shared`, `--workflow-group NAME` (one-run overlay on the tfvars, see `scope.sh`; connector kinds looked up in SG; with `--project` they are that project's `projectOverrides`, else the `SGDefault*` values). `import`/`all` end by writing `export/run-result.json` + `run-summary.md` (`write_run_result`, outcomes planned/blocked/success/failed) for CI. Resolves each tool via `sg_resolve` (PATH first — the image installs them — else `sg_ensure_*` cache). `apply`/`all` first run `require_tfc_auth`, which resolves the token the tfe provider will use (`TFE_TOKEN`, `TF_TOKEN_<host>`, or the `terraform login` file for `tfHostname`) and verifies it with `GET /api/v2/account/details`, failing fast on a missing or rejected (expired) credential. Runs `convert` and `import` in parallel (`--concurrency`, default 4), raises `terraform apply -parallelism`, and retries `sg-cli` imports with backoff (`sg_retry`, `SG_RETRIES`). Import resolves each project's workflow group with `group_for` (legacy `.sg/workflow-groups.json` entry → the payload's `CLIConfiguration.WorkflowGroup.name` via `payload_group` → `tfc-<seg>`) and runs `plan_groups`: STATUS `reuse` (exists) / `create` (missing; created after the confirmation) / `missing!` (`--no-create-groups`) / `moved!`; it collects `PLAN_PROBLEMS` — a project whose workflows still live in the group recorded in `state.json` (or the default group) when the target changed (SG cannot move workflows), and two projects sharing a group with overlapping names — prints them as ✗ lines, still shows the per-workflow plan (also under `--dry-run`) and then dies. The skip set (`state_import_done`) is computed before the plan so the ACTION column can say `skip`/`update`/`create`. Groups are never PATCHed. Imports go through `import_bulk`: it first lists the group (`sg_list_workflows`) and PATCHes every workflow that already exists (`sg_update_workflow`, `[updated]` marker, state re-uploaded, failures as `Failed to update <wf>: ...`), so the plan's `update` rows never hit a create; the remaining, new workflows with Terraform variables go via `sg-cli workflow create --bulk`, workflows **without** via a direct `POST .../wfs/` (`sg_create_workflow` + `sg_upload_tfstate` in `sg_api.sh`) because sg-cli drops an empty `iacInputData.data` and the API rejects the workflow — a workaround (`TODO(sg-cli)` markers) until sg-cli ships with sg-sdk-go >= v1.5.7, which sends the empty object; the direct path prints sg-cli-style `Failed to create <name>: <code>: <body>` lines so the parsing below covers both. State files are the migrator's job on both paths: `import_bulk` tracks sg-cli's per-workflow upload lines (`State file uploaded successfully` / `Failed to upload state file for <wf>`), re-uploads the failures via `upload_state` → `sg_upload_tfstate` (adds the `x-ms-blob-type: BlockBlob` header Azure Blob requires; sg-cli omits it and also treats anything but a literal `HTTP/1.1 200 OK` as failure — second `TODO(sg-cli)`), and writes `[state] uploaded|failed|none <wf>` markers into the parsed output; `do_import` turns them into `state_uploaded`/`state_failed` in the result, a workflow without its state fails the file (and `state_import_done` re-tries it next run). Existing workflows come back from sg-cli as `Failed to create <wf>: 409: Workflow ID not unique` (its update branch matches the outdated text `Workflow name not unique` — third `TODO(sg-cli)`); `import_bulk` collects those, PATCHes them via `sg_update_workflow`, re-uploads their state and drops the failure line, and `sg_create_workflow` does the same fallback on the direct path (prints `updated`). Before the parallel import, `probe_import` imports one workflow alone (first selected of the first file) and dies unless the create and the state upload both succeeded — the fail-fast for environment problems. sg-cli exits 0 even when individual workflows fail, so `do_import` parses its output: a workflow rejected as above SG's managed Terraform ceiling (1.5.7, last MPL/FOSS release) is re-imported with `SGDefaultTerraformVersion` (read from `terraform.tfvars`; an explicit `null` there means the `terraformVersion` key is deleted instead, so the org's execution preset decides — `tfvars_is_null` tells an explicit null from a missing key), the payload patched in place, and the case logged to `export/terraform-version-fallbacks.log` plus a printed notice; any other per-workflow failure fails the run. `cmd_import` reads the org's execution preset (`sg_execution_preset`, `GET /orgs/{org}/` → `Settings.workflowDefaults`) and `preset_labels` (report.sh) turns it into the `preset (1.5.7)` / `preset (private:rg)` cells of the plan and the `SG_TF_FALLBACK_LABEL` used in messages. The trigger pass (`do_set_triggers` → `sg_set_vcs_triggers`) skips workflows that do not exist in SG, keeps the sha of each posted `{VCSConfig, VCSTriggers}` body in `triggers.<seg>.sha` and does not re-POST an unchanged, already-set workflow (`--fresh` does; the endpoint upserts anyway, a 4xx "already exists" from older builds counts as success), and `sg_api_post` returns 22 on 4xx so `sg_retry` (via `SG_NO_RETRY_RC`) does not retry definitive errors. `clean` removes local artifacts (`export/`, TF state, tool cache); `clean --all` also removes config. `completion bash|zsh` prints a completion script (`cmd_completion`; commands + descriptions come from `SG_COMMAND_DESCS` (`SG_COMMANDS` is derived from it), options from `SG_OPTIONS`; keep them in sync with the parser and the host flags; the zsh script also works autoloaded from `$fpath` as `_sg-migrate.sh`); like `clean` it always runs natively. Output is concise by default — `apply` captures terraform's init/plan output and shows only progress + the final summary (or the full log on failure, `TF_IN_AUTOMATION=1`/`-no-color`); `-v`/`--verbose` (`SG_VERBOSE`) streams everything and un-gates the convert detail lines. Colored logging via `sg_step`/`sg_log`/`sg_success`/`sg_warn`/`sg_err`/`sg_row`; paths shown relative via `sg_rel` (all auto-off when not a TTY / `NO_COLOR`). Long steps show a live progress line (`sg_run_quiet` for terraform init/apply, `run_parallel <fn> <max> <label> <items>` for convert/import/triggers — animated only when `SG_ANIMATE=1`, i.e. colors are on; plain log lines otherwise) and report their duration (`sg_fmt_secs`). Phases start with `phase_begin` — inside `all` the header is numbered (`Phase 2/5: ...`, `PHASE_TOTAL` set in `main`) and `phase_took` gives the elapsed time; `run_parallel` prints a job's output as-is when it is a single line and adds the `── <file> ──` header only for longer or failed output (always with `-v`). The import confirmation goes through `sg_confirm` (so `SG_ANSWERS_FILE` works; a declined prompt is a neutral "cancelled" line, not an error) and `import`/`all` end with `finish_line` (outcome, total time, open checklist items from `CHECKLIST_OPEN`). A Ctrl-C is trapped to clear the progress line and print a resume hint. - `scripts/lib/` — sourced libraries (a `lib/` gitignore rule is negated for this dir): `prompt.sh` (`sg_ask`/`sg_confirm`/`sg_select`; tty or `SG_ANSWERS_FILE` for tests; `SG_NONINTERACTIVE=1`/`-y`/no TTY → defaults), `tfvars.sh` (`tfvars_get` cached hcl2json reads — `tfvars_json` merges the run overlay (`TFVARS_OVERLAY_JSON`) over the file, and `tfvars_get`/`tfvars_get_json` keep a `false` value instead of treating it like a missing key —, `tfvars_has`/`tfvars_is_null` to tell an explicit `null` from a missing key, `tfvars_write` renders the wizard's `W_*` vars — `W_TF_SOURCE`, and `W_TF_VERSION`/`W_RUNNER_JSON` may be the literal `null`; plain strings go through `_tfvars_str`, so quotes/backslashes in names stay valid HCL; `projectOverrides`/`workspaceOverrides` are rendered from `W_PROJECT_OVERRIDES_JSON`/`W_WS_OVERRIDES_JSON` by `_tfvars_map` — `"key" = { attr = <json> }`, which hcl2json round-trips — so hand-written blocks survive a re-run of `init` (`wizard_run` reads them before writing; inner comments are lost); `tfWorkspaceIgnoreNames` is kept the same way (`W_IGNORE_NAMES_JSON`, never asked, shown in the review's scope row), and every other setting the wizard does not ask about (`tfProjects`, `cloudAuthVarPatterns`, hand-added keys) is appended verbatim from `W_PREV_JSON` under "Kept from the previous file"; `tfvars_missing_keys` (keys of `terraform.tfvars.example` the file lacks) and `tfvars_upgrade` (appends those paragraphs of the example — comment + default — under a dated header, `.bak` kept, touches nothing else; behind `init --upgrade`, prompt-free)), `scope.sh` (the run scope: `PROJECT_FILTER`/`WS_FILTER`/`WS_EXCLUDE`/`TAG_FILTER`/`TAG_EXCLUDE` arrays, `names_json`, `slug_of` (the transformer's project-slug rule in bash), `project_selected <seg>`, `ws_narrowed`/`ws_filter_json`/`ws_exclude_json` (tfvars `tfWorkspaceIgnoreNames` ∪ flags, memoized)/`ws_selected <name>` (bash `case` globs), `WS_SCOPE_JQ` + `ws_jq_args` → `WS_JQ_ARGS` for the jq sites, `scope_tags_json`/`scope_ignore_tags_json`, `scope_tfvar_args` → `SCOPE_TFVAR_ARGS` for apply, `scope_describe`, `scope_sha_input` for the apply phase hash; and the **run configuration overlay**: `--set KEY=VALUE` (`SET_VARS`, values via `_overlay_value`: HCL/JSON literals through hcl2json, bare text stays a string), `--cloud-connector`/`--vcs-connector`/`--runner-group`/`--workflow-group` → `overlay_build` looks the connector kinds up with `sg_list_integrations`/`sg_integration_type`/`sg_vcs_kind_of`, resolves `--project` values to raw TFC names (`_overlay_project_name`, fatal for an unknown project, as-given with a warning when TFC is unreachable) and fills `TFVARS_OVERLAY_JSON` — `projectOverrides[<name>]` fields with `--project`, `SGDefault*` keys without; `tfvars_json` deep-merges it over the file so every reader sees it, `scope_tfvar_args` passes each overlay key as `-var`, `scope_sha_input` includes it, `write_run_result` records it under `configuration`; `main` refuses the flags for `init`/`clean`/`completion` and a standalone `import` warns that they only reach the workflows through the export (`RAN_APPLY`)), `tfc_api.sh` (`tfc_http`/`tfc_get_all` paginated, `tfc_list_{orgs,projects,workspaces}` — workspaces include `vcs_provider`/`vcs_url`/`vcs_identifier` —, `tfc_select_workspaces` (the module's name-glob / include-tags(all) / exclude-tags / exclude-name-globs filter, shared by wizard and preflight), `tfc_vcs_summary`/`tfc_vcs_kind_for`/`tfc_vcs_label_for`, `tfc_token` with provider precedence, `require_tfc_auth`), `sg_api.sh` (`_sg_api` 0/22/1 contract + `SG_HTTP_CODE`, `_sg_listall` paginated listall reader (the API pages at 50) behind `sg_list_workflows`/`sg_list_wfgrps`, `sg_set_vcs_triggers`, `sg_list_integrations`, `sg_integration_type`, `sg_vcs_kind_of` (connector type → `sourceConfigDestKind`), `sg_execution_preset` + `sg_preset_{desc,runner_desc,version_desc,runner_provided}` (the org's execution preset and how to describe it), `sg_*_exists`, `sg_list_workflows`, `sg_create_secret`, `sg_patch_workflow`, `wfgroup_*`), `wizard.sh` (`wizard_run`: 5 steps TFC → SG → projects → defaults → review; `wizard_projects` (with >1 selected project, from `W_SEL_PROJECTS_JSON`) asks whether the global connectors and `tfc-<project>` groups apply to all, else per project the cloud connector, VCS connector (kind + prefix follow) and group (default / existing via `sg_list_wfgrps` / typed), writing `projectOverrides`; "same as the default" deletes the key, unmanaged fields are kept, previous picks come first (`_w_first`); the defaults step also asks `stripCloudAuthVars`; the TFC step first asks which TFC projects to migrate (with >1 project: `all` or `pick` by name/slug, resolved to raw names, unknown names re-asked; a previous non-empty `tfProjects` makes `pick` the first item; free text without discovery) → `W_PROJECTS_JSON`, rendered as `tfProjects` (`[]` = all) and applied to the workspace selection before the scope question; it keeps the workspace list and derives, via `tfc_vcs_summary`, which VCS provider the selected workspaces use and the repositories' URL prefix — the SG step lists matching connectors first and takes the repo URL prefix from TFC (never from a previous tfvars written for another provider kind), warns on a kind mismatch, reads the org's execution preset and offers it as the runner choice; the defaults step asks whether to carry TFC versions or use the preset (`SGTerraformVersionSource`) and, for carry, which fallback (`SGDefaultTerraformVersion` or the preset = null); the review shows human-readable rows plus the `tfc-<project>` groups; `_w_select_lines` feeds menus line-by-line so names with spaces/globs survive), `preflight.sh` (`preflight_run <apply|import|all>`, once per process; ✓ lines are plain language and name the connector's role/kind, ✗/! lines name the tfvars field to fix; connector ids, runner groups and DPC kinds are collected from `SGDefault*`, `projectOverrides` and `workspaceOverrides`; besides existence it checks consistency — VCS kind vs. the connector's type, globally and per `projectOverrides` entry (fail), repo URL prefix vs. where the TFC repositories live or the kind's well-known host (warn), `projectOverrides` keys vs. the TFC project names (warn), `--project` values vs. the project names/slugs (fail; a `tfProjects` typo warns), and the workspace-selection preview applies the CLI scope (names, excludes, tags, projects) so its count is what the apply will export; the import context states the group policy and warns when a project's group changed since its last import; it also reports the org's execution preset next to the version/runner policy and warns when `carry` meets a preset that mounts a runner-provided binary), `state.sh` (`.sg/state.json`: `phases.<name>.input_sha`, `import.<seg>.{group,imported,updated,failed,tf_fallback,state_uploaded,state_failed}` (`group` anchors the never-move check; `updated` = PATCHed on a 409), `triggers.<seg>.{set,unchanged,failed,missing,sha}`, `secrets.<name>`; `run_phase` in migrate.sh skips unchanged phases), `report.sh` (`show_migration_summary` after apply — incl. a state-files row —, `show_import_plan` per-workflow table with columns sized to their content — it keeps the rows in `PLAN_ROWS_TSV` (name, group, action, version, runner, triggers, vars, secrets, segment, workspace, project) —, `write_run_result <outcome>` → `export/run-result.json` + `run-summary.md` from `PLAN_ROWS_TSV`, `PLAN_GROUP_ROWS` (plain copy of the group table, set by `plan_groups`), `PLAN_PROBLEMS`, the merged state and `CHECKLIST_OPEN`; per workflow `result` is planned/skipped/failed/updated/created, with the probe workflow — planned `create`, PATCHed on its second pass — reported as created), `errors.sh` (`explain_api_error` regex→hint table, `SG_API_HINTS`), `checklist.sh` (`create_secret_stubs` → SG secret `tfc-<wf>-<VAR>` = `CHANGE_ME`, referenced as `${secret::<name>}` via PATCH + payload patch; `write_checklist` → `export/post-import-checklist.md`, printed to the terminal as one ✓/! status line per section plus the imported groups with UI links — the file has the details; section 5 covers both the TFC state export and the SG state upload (`state_failed` from the import results); sets `CHECKLIST_OPEN`). Parallel jobs (`run_parallel`) run in subshells, so `do_import`/`do_set_triggers` write `.import-result.<seg>.json` / `.triggers-result.<seg>.json` into the export dir and the caller merges them into state. - `scripts/tools.sh` — sourced by the other scripts (sets `SG_REPO_ROOT` to its parent dir). Provides `sg_resolve` (PATH-or-cache), `sg_ensure_{jq,hcl2json,yajsv,sgcli}` (pinned downloads into `.sg/cached/`; `sg-cli` is the Go binary), `sg_retry`, the color vars + log helpers. The Docker image bundles these tools at build time (`yajsv` built from source so arm64 works), so in-container runs never download. - **Variable sets** — `scripts/enrich_variable_sets.sh <tfc-org> <payload...>` (run by `cmd_enrich`, which reads `tfOrg`/`tfHostname` from `terraform.tfvars` via `hcl2json` — variable sets live in the **TFC** org, not `SG_ORG`). Lists all sets + vars via the TFC API (Bearer token from the `terraform login` creds file or `TFE_TOKEN`), computes per-workspace effective vars (scope: global/project/workspace; precedence: priority sets > workspace vars > non-priority sets), and merges them into each workflow (matched by workspace name parsed from `TfStateFilePath`). Sensitive set vars and key conflicts are reported. Skipped by `--no-variable-sets`. diff --git a/README.md b/README.md index ad0850f..d4f451c 100644 --- a/README.md +++ b/README.md @@ -190,7 +190,7 @@ To update workflows with different details, re-run the sg-cli command with the m - **Execution presets.** StackGuardian org admins can define an execution preset (Settings → Runner groups → Execution presets): default runner constraints plus a Terraform and an OpenTofu configuration that the API applies to any new workflow whose payload does not carry those fields. To lean on it, set `SGTerraformVersionSource = "preset"` (no version is sent at all) and/or `SGDefaultRunnerConstraints = null` (no runner constraints are sent); `SGDefaultTerraformVersion = null` keeps carrying TFC pins but hands the unpinned and rejected ones to the preset. `init` offers these choices with the org's current preset shown inline, preflight prints what the preset would supply, and the import plan marks such cells as `preset (…)`. The preset's custom runtime image or runner-provided binary is inherited in every mode, since the migrator never sets those keys. - **Workspaces without variables are imported directly.** sg-cli (up to v2.2.1) drops an empty `iacInputData.data` from the request, and the API then rejects the workflow with `VCSConfig.iacInputData.data: This field is required.` The importer therefore creates workflows that have no Terraform variables straight through the API (same create, same state upload) and uses sg-cli for the rest. This is a workaround until sg-cli picks up sg-sdk-go v1.5.7, which fixes the dropped key. - **Terraform state is uploaded by the migrator, and checked.** sg-cli's own upload sends a PUT that Azure-backed StackGuardian environments reject (missing `x-ms-blob-type`) and it reports success only on a literal `HTTP/1.1 200 OK`, so the importer re-uploads every state file sg-cli reports as failed and records per workflow whether the store accepted it. A workflow without its state counts as a failed import: it is listed in the checklist (`state: N workflow(s) are in SG without their state`) and the file is retried on the next `import`. -- **Re-runs update existing workflows.** sg-cli answers `409 Workflow ID not unique` for a workflow that already exists instead of updating it (its update path waits for a message the API no longer sends), so the importer updates such workflows itself via PATCH and re-uploads their state. A change in `terraform.tfvars` (connector, runner, approvers, version) reaches existing workflows through `apply` + `import`: the regenerated payload differs, the plan shows `update`, and the workflow is PATCHed; an unchanged file shows `skip`. VCS triggers are re-registered only when their configuration changed (the API upserts them). Secrets that already exist are never overwritten. Re-running `import` (or `import --fresh`) is therefore safe and idempotent. +- **Re-runs update existing workflows.** Workflows that already exist in the target group (the plan's `update` rows) are PATCHed directly and their state re-uploaded; only new ones go through the create path. Should a create still meet `409 Workflow ID not unique` (sg-cli's own update path waits for a message the API no longer sends), the importer updates that workflow itself. A change in `terraform.tfvars` (connector, runner, approvers, version) reaches existing workflows through `apply` + `import`: the regenerated payload differs, the plan shows `update`, and the workflow is PATCHed; an unchanged file shows `skip`. VCS triggers are re-registered only when their configuration changed (the API upserts them). Secrets that already exist are never overwritten. Re-running `import` (or `import --fresh`) is therefore safe and idempotent. - **Fail fast.** When more than one workflow is to be imported, the importer first imports a single one (the first selected workflow of the first payload file) and requires both the create and its state upload to succeed before the rest is imported in parallel. An environment problem then costs one workflow, not all of them; the probe workflow is simply updated again with its file. - **State export is idempotent.** Re-running `terraform apply` only pulls state for workspaces not yet exported. Set `forceStateRefresh = true` to re-pull everything. Workspaces not using `remote` execution may export incomplete state — see the summary. - **Workflow naming.** `ResourceName` mirrors the TFC workspace name, sanitized to StackGuardian's rules (1-100 chars, `[-a-zA-Z0-9_]`); any rename is listed in the summary and the checklist. diff --git a/scripts/migrate.sh b/scripts/migrate.sh index 01dc302..f3fec37 100755 --- a/scripts/migrate.sh +++ b/scripts/migrate.sh @@ -671,7 +671,8 @@ cmd_apply() { # tfWorkspaceIgnoreNames; the module applies the tag filters on top. scope_tfvar_args tfvar_args=(${SCOPE_TFVAR_ARGS[@]+"${SCOPE_TFVAR_ARGS[@]}"}) - [ "${#tfvar_args[@]}" -gt 0 ] && sg_log "run scope: $(scope_describe)" + # Scope flags are logged here; the configuration overlay was logged by overlay_build. + [ -n "$(scope_describe)" ] && sg_log "run scope: $(scope_describe)" if [ "$VERBOSE" -eq 1 ]; then # shellcheck disable=SC2119 @@ -794,14 +795,41 @@ upload_state() { # do_import reads those from <out>; non-zero only when a call itself could # not be made. import_bulk() { - local grp="$1" file="$2" out="$3" rc=0 n_direct with_vars entry name err line cur="" cli_out pat + local grp="$1" file="$2" out="$3" rc=0 n_direct with_vars entry name err line cur="" cli_out pat known upd_names create_file local -a redo=() exists=() updated=() : >"$out" - n_direct="$("$JQ_BIN" '[.[] | select((.VCSConfig.iacInputData.data // {}) | length == 0)] | length' "$file")" - with_vars="$file" + + # Workflows that already exist in the group (the plan's "update" rows) are + # PATCHed straight away, with their state re-uploaded; only the rest goes + # through the create path below. The 409 handling further down stays as the + # safety net for a workflow created between the plan and this call. + create_file="$file" + known="$(sg_list_workflows "$grp")" + printf '%s' "$known" | "$JQ_BIN" -e 'type == "array"' >/dev/null 2>&1 || known='[]' + upd_names="$("$JQ_BIN" -c --argjson ex "$known" '[.[].ResourceName | select(. as $n | $ex | index($n) != null)]' "$file")" + if [ "$("$JQ_BIN" 'length' <<<"$upd_names")" -gt 0 ]; then + sg_log "$("$JQ_BIN" 'length' <<<"$upd_names") workflow(s) already exist in $grp — updating them" + while IFS= read -r name; do + [ -n "$name" ] || continue + entry="$("$JQ_BIN" -c --arg n "$name" 'first(.[] | select(.ResourceName == $n))' "$file")" + if err="$(SG_NO_RETRY_RC=22 sg_retry "$RETRIES" "$RETRY_BASE" -- sg_update_workflow "$grp" "$entry" 2>/dev/null)"; then + sg_log " $name: updated" + printf '[updated] %s\n' "$name" >>"$out" + upload_state "$grp" "$name" "$file" "$out" || true + else + # Same shape as sg-cli's failure line (parsed by do_import). + printf 'Failed to update %s: %s\n' "$name" "$(tail -n1 <<<"$err")" | tee -a "$out" + fi + done < <("$JQ_BIN" -r '.[]' <<<"$upd_names") + create_file="$(mktemp)" + "$JQ_BIN" --argjson u "$upd_names" 'map(select(.ResourceName as $n | $u | index($n) == null))' "$file" >"$create_file" + fi + + n_direct="$("$JQ_BIN" '[.[] | select((.VCSConfig.iacInputData.data // {}) | length == 0)] | length' "$create_file")" + with_vars="$create_file" if [ "$n_direct" -gt 0 ]; then with_vars="$(mktemp)" - "$JQ_BIN" 'map(select((.VCSConfig.iacInputData.data // {}) | length > 0))' "$file" >"$with_vars" + "$JQ_BIN" 'map(select((.VCSConfig.iacInputData.data // {}) | length > 0))' "$create_file" >"$with_vars" fi if [ "$("$JQ_BIN" length "$with_vars")" -gt 0 ]; then cli_out="$(mktemp)" @@ -853,8 +881,11 @@ import_bulk() { for name in "${redo[@]}"; do upload_state "$grp" "$name" "$file" "$out" || true; done fi fi - [ "$with_vars" != "$file" ] && rm -f "$with_vars" - [ "$n_direct" -gt 0 ] || return "$rc" + [ "$with_vars" != "$create_file" ] && rm -f "$with_vars" + if [ "$n_direct" -eq 0 ]; then + [ "$create_file" != "$file" ] && rm -f "$create_file" + return "$rc" + fi sg_log "$n_direct workflow(s) have no Terraform variables — creating them via the API directly (sg-cli drops an empty iacInputData.data)" while IFS= read -r entry; do @@ -871,7 +902,8 @@ import_bulk() { # Same shape as sg-cli's failure line (parsed by do_import). printf 'Failed to create %s: %s\n' "$name" "$(tail -n1 <<<"$err")" | tee -a "$out" fi - done < <("$JQ_BIN" -c '.[] | select((.VCSConfig.iacInputData.data // {}) | length == 0)' "$file") + done < <("$JQ_BIN" -c '.[] | select((.VCSConfig.iacInputData.data // {}) | length == 0)' "$create_file") + [ "$create_file" != "$file" ] && rm -f "$create_file" return "$rc" } @@ -911,11 +943,11 @@ do_import() { if [[ "$line" =~ $TF_CEILING_RE ]]; then fb+=("${BASH_REMATCH[1]}") ceiling="${BASH_REMATCH[2]}" - elif [[ "$line" =~ Failed\ to\ create\ ([^:]+):\ [0-9]+:\ (.*)$ ]]; then - failed+=("${BASH_REMATCH[1]}") - explain_api_error "${BASH_REMATCH[2]}" - elif [[ "$line" =~ Failed\ to\ create\ ([^:]+): ]]; then - failed+=("${BASH_REMATCH[1]}") + elif [[ "$line" =~ Failed\ to\ (create|update)\ ([^:]+):\ [0-9]+:\ (.*)$ ]]; then + failed+=("${BASH_REMATCH[2]}") + explain_api_error "${BASH_REMATCH[3]}" + elif [[ "$line" =~ Failed\ to\ (create|update)\ ([^:]+): ]]; then + failed+=("${BASH_REMATCH[2]}") elif [[ "$line" =~ ^\[state\]\ uploaded\ (.+)$ ]]; then st_ok+=("${BASH_REMATCH[1]}") elif [[ "$line" =~ ^\[state\]\ failed\ ([^:]+): ]]; then @@ -948,7 +980,7 @@ do_import() { out="$(mktemp)" import_bulk "$grp" "$tmp" "$out" || rc=1 for name in "${fb[@]}"; do - if grep -q "Failed to create $name:" "$out"; then + if grep -qE "Failed to (create|update) $name:" "$out"; then failed+=("$name") else line="$("$JQ_BIN" -r --arg n "$name" '.[] | select(.ResourceName == $n) | .TerraformConfig.terraformVersion' "$f")" @@ -1242,6 +1274,10 @@ _sg_migrate() { cmds=( $(printf '%s\n' "$SG_COMMAND_DESCS" | sed "s/.*/ '&'/") ) + # Every option has a distinct description on purpose: _arguments folds + # options that share one text into a single "--verbose -v -- ..." row, and + # with a multi-entry matcher-list (oh-my-zsh's default) such rows make zsh + # list the whole table one cell per line, repeated per matcher. _arguments -s \\ '--org[StackGuardian org for import]:org' \\ '--region[StackGuardian region]:region:(eu us)' \\ @@ -1268,10 +1304,14 @@ $(printf '%s\n' "$SG_COMMAND_DESCS" | sed "s/.*/ '&'/") '--vcs-connector[VCS connector for this run (kind looked up in SG)]:id' \\ '--runner-group[Private runner group for this run (shared = SG runners)]:name' \\ '--workflow-group[Workflow group for the --project workflows]:name' \\ - '(-v --verbose)'{-v,--verbose}'[Show full terraform/tool output]' \\ - '(-y --yes)'{-y,--yes}'[Skip the import confirmation prompt]' \\ - '(-h --help)'{-h,--help}'[Show help]' \\ - '(--native --local)'{--native,--local}'[Run natively instead of in Docker]' \\ + '(-v --verbose)-v[Same as --verbose]' \\ + '(-v --verbose)--verbose[Show full terraform/tool output]' \\ + '(-y --yes)-y[Same as --yes]' \\ + '(-y --yes)--yes[Skip the import confirmation prompt]' \\ + '(-h --help)-h[Same as --help]' \\ + '(-h --help)--help[Show help]' \\ + '(--native --local)--native[Run natively instead of in Docker]' \\ + '(--native --local)--local[Same as --native]' \\ '--build[Rebuild the Docker image first]' \\ '1:command:->cmd' \\ '2:shell:->shell' From 8f0ed1584ffd972d23a0d4392001dccb2d4eea73 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adis=20Halilovi=C4=87?= <adis.halilovic@stackguardian.io> Date: Mon, 14 Sep 2026 18:21:38 +0200 Subject: [PATCH 61/71] fix: preflight reports related checks on one line instead of one per check --- CLAUDE.md | 2 +- scripts/lib/preflight.sh | 86 +++++++++++++++++++++++++++++----------- 2 files changed, 63 insertions(+), 25 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index acbfa9c..0f3860e 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -25,7 +25,7 @@ The five phases are wrapped by an orchestrator so users don't run them by hand: - `sg-migrate.sh` (host entrypoint, repo root) — runs `scripts/migrate.sh` **inside the Docker image** (`Dockerfile`), bind-mounting the repo at `/app` and forwarding `SG_API_TOKEN`/`SG_ORG`/`SG_BASE_URL`/`TFE_TOKEN`/`TF_TOKEN_*`. `--tfvars FILE` (or `SG_TFVARS`) is a host path: the wrapper strips the flag, resolves it and passes it on as `SG_TFVARS` — rewritten to `/app/<relative>` when the file is inside the checkout, otherwise mounted read-only at `/tmp/sg-run.tfvars`. For TFC auth it prefers a long-lived `TFE_TOKEN` (env); otherwise it mounts `~/.terraform.d/credentials.tfrc.json` read-only. Runs natively instead when `--native`/`--local` is passed, `SG_NATIVE=1` is set, the command is `clean`/`completion`/help (or no command is given), or Docker is absent. Exports `SG_PROG` so `migrate.sh` shows `./sg-migrate.sh` in its usage/hints. `update` is host-only too (`cmd_update` in the wrapper, never passed to `migrate.sh`): refuses a non-git checkout, a detached HEAD or dirty tracked files, then `git pull --ff-only` and rebuilds the image only when the `Dockerfile` changed between the old and new HEAD — this is how customers pick up fixes from a plain clone (untracked config/output survive). `migrate.sh` lists `update` in its usage/`SG_COMMANDS` and rejects it with a host-only hint. `.gitattributes` forces LF so a Git-for-Windows clone does not CRLF the scripts. - `scripts/migrate.sh` — the actual orchestrator (sources `scripts/lib/*.sh`, see below). Subcommands `init|preflight|apply|enrich|convert|validate|import|triggers|checklist|all|clean|completion` (no command prints the help menu; `enrich` runs in `all` unless `--no-variable-sets`). Flags beyond the basics: `--dry-run` (import, or all: the local export runs, then the plan is shown and nothing is created in SG), `--fresh` (ignore run state), the scope flags from `lib/scope.sh` — `--project NAME|SLUG`, `--workspace GLOB`, `--exclude-workspace GLOB`, `--tag NAME`, `--exclude-tag NAME`, all repeatable; tfvars holds the widest scope, include flags replace the tfvars list, exclude flags add to it; apply passes them as `-var` (`tfProjects`, `workspacenames`, `tfWorkspaceIgnoreNames`, `tfWorkspaceTags`, `tfWorkspaceIgnoreTags`), the later phases apply the same names/excludes to the payload entries (`ws_selected` in bash, `WS_SCOPE_JQ` + `WS_JQ_ARGS` in jq) and `--project` to the payload files by slug (`project_selected`); `--workspace '*'` alone is not a narrowing, so the unchanged-file skip still applies; a scope that matches nothing is an error —, `--tfvars FILE` (`TFVARS`, absolute; also `-var-file` for terraform and exported to the enrich script; `clean --all` only removes the module's own file), `--skip-preflight`, `--no-secret-stubs`, `--upgrade` (with `init`: `tfvars_upgrade`, append the settings an older tfvars lacks; preflight warns about them until then), and the run configuration flags `--set KEY=VALUE`, `--cloud-connector ID`, `--vcs-connector ID`, `--runner-group NAME|shared`, `--workflow-group NAME` (one-run overlay on the tfvars, see `scope.sh`; connector kinds looked up in SG; with `--project` they are that project's `projectOverrides`, else the `SGDefault*` values). `import`/`all` end by writing `export/run-result.json` + `run-summary.md` (`write_run_result`, outcomes planned/blocked/success/failed) for CI. Resolves each tool via `sg_resolve` (PATH first — the image installs them — else `sg_ensure_*` cache). `apply`/`all` first run `require_tfc_auth`, which resolves the token the tfe provider will use (`TFE_TOKEN`, `TF_TOKEN_<host>`, or the `terraform login` file for `tfHostname`) and verifies it with `GET /api/v2/account/details`, failing fast on a missing or rejected (expired) credential. Runs `convert` and `import` in parallel (`--concurrency`, default 4), raises `terraform apply -parallelism`, and retries `sg-cli` imports with backoff (`sg_retry`, `SG_RETRIES`). Import resolves each project's workflow group with `group_for` (legacy `.sg/workflow-groups.json` entry → the payload's `CLIConfiguration.WorkflowGroup.name` via `payload_group` → `tfc-<seg>`) and runs `plan_groups`: STATUS `reuse` (exists) / `create` (missing; created after the confirmation) / `missing!` (`--no-create-groups`) / `moved!`; it collects `PLAN_PROBLEMS` — a project whose workflows still live in the group recorded in `state.json` (or the default group) when the target changed (SG cannot move workflows), and two projects sharing a group with overlapping names — prints them as ✗ lines, still shows the per-workflow plan (also under `--dry-run`) and then dies. The skip set (`state_import_done`) is computed before the plan so the ACTION column can say `skip`/`update`/`create`. Groups are never PATCHed. Imports go through `import_bulk`: it first lists the group (`sg_list_workflows`) and PATCHes every workflow that already exists (`sg_update_workflow`, `[updated]` marker, state re-uploaded, failures as `Failed to update <wf>: ...`), so the plan's `update` rows never hit a create; the remaining, new workflows with Terraform variables go via `sg-cli workflow create --bulk`, workflows **without** via a direct `POST .../wfs/` (`sg_create_workflow` + `sg_upload_tfstate` in `sg_api.sh`) because sg-cli drops an empty `iacInputData.data` and the API rejects the workflow — a workaround (`TODO(sg-cli)` markers) until sg-cli ships with sg-sdk-go >= v1.5.7, which sends the empty object; the direct path prints sg-cli-style `Failed to create <name>: <code>: <body>` lines so the parsing below covers both. State files are the migrator's job on both paths: `import_bulk` tracks sg-cli's per-workflow upload lines (`State file uploaded successfully` / `Failed to upload state file for <wf>`), re-uploads the failures via `upload_state` → `sg_upload_tfstate` (adds the `x-ms-blob-type: BlockBlob` header Azure Blob requires; sg-cli omits it and also treats anything but a literal `HTTP/1.1 200 OK` as failure — second `TODO(sg-cli)`), and writes `[state] uploaded|failed|none <wf>` markers into the parsed output; `do_import` turns them into `state_uploaded`/`state_failed` in the result, a workflow without its state fails the file (and `state_import_done` re-tries it next run). Existing workflows come back from sg-cli as `Failed to create <wf>: 409: Workflow ID not unique` (its update branch matches the outdated text `Workflow name not unique` — third `TODO(sg-cli)`); `import_bulk` collects those, PATCHes them via `sg_update_workflow`, re-uploads their state and drops the failure line, and `sg_create_workflow` does the same fallback on the direct path (prints `updated`). Before the parallel import, `probe_import` imports one workflow alone (first selected of the first file) and dies unless the create and the state upload both succeeded — the fail-fast for environment problems. sg-cli exits 0 even when individual workflows fail, so `do_import` parses its output: a workflow rejected as above SG's managed Terraform ceiling (1.5.7, last MPL/FOSS release) is re-imported with `SGDefaultTerraformVersion` (read from `terraform.tfvars`; an explicit `null` there means the `terraformVersion` key is deleted instead, so the org's execution preset decides — `tfvars_is_null` tells an explicit null from a missing key), the payload patched in place, and the case logged to `export/terraform-version-fallbacks.log` plus a printed notice; any other per-workflow failure fails the run. `cmd_import` reads the org's execution preset (`sg_execution_preset`, `GET /orgs/{org}/` → `Settings.workflowDefaults`) and `preset_labels` (report.sh) turns it into the `preset (1.5.7)` / `preset (private:rg)` cells of the plan and the `SG_TF_FALLBACK_LABEL` used in messages. The trigger pass (`do_set_triggers` → `sg_set_vcs_triggers`) skips workflows that do not exist in SG, keeps the sha of each posted `{VCSConfig, VCSTriggers}` body in `triggers.<seg>.sha` and does not re-POST an unchanged, already-set workflow (`--fresh` does; the endpoint upserts anyway, a 4xx "already exists" from older builds counts as success), and `sg_api_post` returns 22 on 4xx so `sg_retry` (via `SG_NO_RETRY_RC`) does not retry definitive errors. `clean` removes local artifacts (`export/`, TF state, tool cache); `clean --all` also removes config. `completion bash|zsh` prints a completion script (`cmd_completion`; commands + descriptions come from `SG_COMMAND_DESCS` (`SG_COMMANDS` is derived from it), options from `SG_OPTIONS`; keep them in sync with the parser and the host flags; the zsh script also works autoloaded from `$fpath` as `_sg-migrate.sh`); like `clean` it always runs natively. Output is concise by default — `apply` captures terraform's init/plan output and shows only progress + the final summary (or the full log on failure, `TF_IN_AUTOMATION=1`/`-no-color`); `-v`/`--verbose` (`SG_VERBOSE`) streams everything and un-gates the convert detail lines. Colored logging via `sg_step`/`sg_log`/`sg_success`/`sg_warn`/`sg_err`/`sg_row`; paths shown relative via `sg_rel` (all auto-off when not a TTY / `NO_COLOR`). Long steps show a live progress line (`sg_run_quiet` for terraform init/apply, `run_parallel <fn> <max> <label> <items>` for convert/import/triggers — animated only when `SG_ANIMATE=1`, i.e. colors are on; plain log lines otherwise) and report their duration (`sg_fmt_secs`). Phases start with `phase_begin` — inside `all` the header is numbered (`Phase 2/5: ...`, `PHASE_TOTAL` set in `main`) and `phase_took` gives the elapsed time; `run_parallel` prints a job's output as-is when it is a single line and adds the `── <file> ──` header only for longer or failed output (always with `-v`). The import confirmation goes through `sg_confirm` (so `SG_ANSWERS_FILE` works; a declined prompt is a neutral "cancelled" line, not an error) and `import`/`all` end with `finish_line` (outcome, total time, open checklist items from `CHECKLIST_OPEN`). A Ctrl-C is trapped to clear the progress line and print a resume hint. -- `scripts/lib/` — sourced libraries (a `lib/` gitignore rule is negated for this dir): `prompt.sh` (`sg_ask`/`sg_confirm`/`sg_select`; tty or `SG_ANSWERS_FILE` for tests; `SG_NONINTERACTIVE=1`/`-y`/no TTY → defaults), `tfvars.sh` (`tfvars_get` cached hcl2json reads — `tfvars_json` merges the run overlay (`TFVARS_OVERLAY_JSON`) over the file, and `tfvars_get`/`tfvars_get_json` keep a `false` value instead of treating it like a missing key —, `tfvars_has`/`tfvars_is_null` to tell an explicit `null` from a missing key, `tfvars_write` renders the wizard's `W_*` vars — `W_TF_SOURCE`, and `W_TF_VERSION`/`W_RUNNER_JSON` may be the literal `null`; plain strings go through `_tfvars_str`, so quotes/backslashes in names stay valid HCL; `projectOverrides`/`workspaceOverrides` are rendered from `W_PROJECT_OVERRIDES_JSON`/`W_WS_OVERRIDES_JSON` by `_tfvars_map` — `"key" = { attr = <json> }`, which hcl2json round-trips — so hand-written blocks survive a re-run of `init` (`wizard_run` reads them before writing; inner comments are lost); `tfWorkspaceIgnoreNames` is kept the same way (`W_IGNORE_NAMES_JSON`, never asked, shown in the review's scope row), and every other setting the wizard does not ask about (`tfProjects`, `cloudAuthVarPatterns`, hand-added keys) is appended verbatim from `W_PREV_JSON` under "Kept from the previous file"; `tfvars_missing_keys` (keys of `terraform.tfvars.example` the file lacks) and `tfvars_upgrade` (appends those paragraphs of the example — comment + default — under a dated header, `.bak` kept, touches nothing else; behind `init --upgrade`, prompt-free)), `scope.sh` (the run scope: `PROJECT_FILTER`/`WS_FILTER`/`WS_EXCLUDE`/`TAG_FILTER`/`TAG_EXCLUDE` arrays, `names_json`, `slug_of` (the transformer's project-slug rule in bash), `project_selected <seg>`, `ws_narrowed`/`ws_filter_json`/`ws_exclude_json` (tfvars `tfWorkspaceIgnoreNames` ∪ flags, memoized)/`ws_selected <name>` (bash `case` globs), `WS_SCOPE_JQ` + `ws_jq_args` → `WS_JQ_ARGS` for the jq sites, `scope_tags_json`/`scope_ignore_tags_json`, `scope_tfvar_args` → `SCOPE_TFVAR_ARGS` for apply, `scope_describe`, `scope_sha_input` for the apply phase hash; and the **run configuration overlay**: `--set KEY=VALUE` (`SET_VARS`, values via `_overlay_value`: HCL/JSON literals through hcl2json, bare text stays a string), `--cloud-connector`/`--vcs-connector`/`--runner-group`/`--workflow-group` → `overlay_build` looks the connector kinds up with `sg_list_integrations`/`sg_integration_type`/`sg_vcs_kind_of`, resolves `--project` values to raw TFC names (`_overlay_project_name`, fatal for an unknown project, as-given with a warning when TFC is unreachable) and fills `TFVARS_OVERLAY_JSON` — `projectOverrides[<name>]` fields with `--project`, `SGDefault*` keys without; `tfvars_json` deep-merges it over the file so every reader sees it, `scope_tfvar_args` passes each overlay key as `-var`, `scope_sha_input` includes it, `write_run_result` records it under `configuration`; `main` refuses the flags for `init`/`clean`/`completion` and a standalone `import` warns that they only reach the workflows through the export (`RAN_APPLY`)), `tfc_api.sh` (`tfc_http`/`tfc_get_all` paginated, `tfc_list_{orgs,projects,workspaces}` — workspaces include `vcs_provider`/`vcs_url`/`vcs_identifier` —, `tfc_select_workspaces` (the module's name-glob / include-tags(all) / exclude-tags / exclude-name-globs filter, shared by wizard and preflight), `tfc_vcs_summary`/`tfc_vcs_kind_for`/`tfc_vcs_label_for`, `tfc_token` with provider precedence, `require_tfc_auth`), `sg_api.sh` (`_sg_api` 0/22/1 contract + `SG_HTTP_CODE`, `_sg_listall` paginated listall reader (the API pages at 50) behind `sg_list_workflows`/`sg_list_wfgrps`, `sg_set_vcs_triggers`, `sg_list_integrations`, `sg_integration_type`, `sg_vcs_kind_of` (connector type → `sourceConfigDestKind`), `sg_execution_preset` + `sg_preset_{desc,runner_desc,version_desc,runner_provided}` (the org's execution preset and how to describe it), `sg_*_exists`, `sg_list_workflows`, `sg_create_secret`, `sg_patch_workflow`, `wfgroup_*`), `wizard.sh` (`wizard_run`: 5 steps TFC → SG → projects → defaults → review; `wizard_projects` (with >1 selected project, from `W_SEL_PROJECTS_JSON`) asks whether the global connectors and `tfc-<project>` groups apply to all, else per project the cloud connector, VCS connector (kind + prefix follow) and group (default / existing via `sg_list_wfgrps` / typed), writing `projectOverrides`; "same as the default" deletes the key, unmanaged fields are kept, previous picks come first (`_w_first`); the defaults step also asks `stripCloudAuthVars`; the TFC step first asks which TFC projects to migrate (with >1 project: `all` or `pick` by name/slug, resolved to raw names, unknown names re-asked; a previous non-empty `tfProjects` makes `pick` the first item; free text without discovery) → `W_PROJECTS_JSON`, rendered as `tfProjects` (`[]` = all) and applied to the workspace selection before the scope question; it keeps the workspace list and derives, via `tfc_vcs_summary`, which VCS provider the selected workspaces use and the repositories' URL prefix — the SG step lists matching connectors first and takes the repo URL prefix from TFC (never from a previous tfvars written for another provider kind), warns on a kind mismatch, reads the org's execution preset and offers it as the runner choice; the defaults step asks whether to carry TFC versions or use the preset (`SGTerraformVersionSource`) and, for carry, which fallback (`SGDefaultTerraformVersion` or the preset = null); the review shows human-readable rows plus the `tfc-<project>` groups; `_w_select_lines` feeds menus line-by-line so names with spaces/globs survive), `preflight.sh` (`preflight_run <apply|import|all>`, once per process; ✓ lines are plain language and name the connector's role/kind, ✗/! lines name the tfvars field to fix; connector ids, runner groups and DPC kinds are collected from `SGDefault*`, `projectOverrides` and `workspaceOverrides`; besides existence it checks consistency — VCS kind vs. the connector's type, globally and per `projectOverrides` entry (fail), repo URL prefix vs. where the TFC repositories live or the kind's well-known host (warn), `projectOverrides` keys vs. the TFC project names (warn), `--project` values vs. the project names/slugs (fail; a `tfProjects` typo warns), and the workspace-selection preview applies the CLI scope (names, excludes, tags, projects) so its count is what the apply will export; the import context states the group policy and warns when a project's group changed since its last import; it also reports the org's execution preset next to the version/runner policy and warns when `carry` meets a preset that mounts a runner-provided binary), `state.sh` (`.sg/state.json`: `phases.<name>.input_sha`, `import.<seg>.{group,imported,updated,failed,tf_fallback,state_uploaded,state_failed}` (`group` anchors the never-move check; `updated` = PATCHed on a 409), `triggers.<seg>.{set,unchanged,failed,missing,sha}`, `secrets.<name>`; `run_phase` in migrate.sh skips unchanged phases), `report.sh` (`show_migration_summary` after apply — incl. a state-files row —, `show_import_plan` per-workflow table with columns sized to their content — it keeps the rows in `PLAN_ROWS_TSV` (name, group, action, version, runner, triggers, vars, secrets, segment, workspace, project) —, `write_run_result <outcome>` → `export/run-result.json` + `run-summary.md` from `PLAN_ROWS_TSV`, `PLAN_GROUP_ROWS` (plain copy of the group table, set by `plan_groups`), `PLAN_PROBLEMS`, the merged state and `CHECKLIST_OPEN`; per workflow `result` is planned/skipped/failed/updated/created, with the probe workflow — planned `create`, PATCHed on its second pass — reported as created), `errors.sh` (`explain_api_error` regex→hint table, `SG_API_HINTS`), `checklist.sh` (`create_secret_stubs` → SG secret `tfc-<wf>-<VAR>` = `CHANGE_ME`, referenced as `${secret::<name>}` via PATCH + payload patch; `write_checklist` → `export/post-import-checklist.md`, printed to the terminal as one ✓/! status line per section plus the imported groups with UI links — the file has the details; section 5 covers both the TFC state export and the SG state upload (`state_failed` from the import results); sets `CHECKLIST_OPEN`). Parallel jobs (`run_parallel`) run in subshells, so `do_import`/`do_set_triggers` write `.import-result.<seg>.json` / `.triggers-result.<seg>.json` into the export dir and the caller merges them into state. +- `scripts/lib/` — sourced libraries (a `lib/` gitignore rule is negated for this dir): `prompt.sh` (`sg_ask`/`sg_confirm`/`sg_select`; tty or `SG_ANSWERS_FILE` for tests; `SG_NONINTERACTIVE=1`/`-y`/no TTY → defaults), `tfvars.sh` (`tfvars_get` cached hcl2json reads — `tfvars_json` merges the run overlay (`TFVARS_OVERLAY_JSON`) over the file, and `tfvars_get`/`tfvars_get_json` keep a `false` value instead of treating it like a missing key —, `tfvars_has`/`tfvars_is_null` to tell an explicit `null` from a missing key, `tfvars_write` renders the wizard's `W_*` vars — `W_TF_SOURCE`, and `W_TF_VERSION`/`W_RUNNER_JSON` may be the literal `null`; plain strings go through `_tfvars_str`, so quotes/backslashes in names stay valid HCL; `projectOverrides`/`workspaceOverrides` are rendered from `W_PROJECT_OVERRIDES_JSON`/`W_WS_OVERRIDES_JSON` by `_tfvars_map` — `"key" = { attr = <json> }`, which hcl2json round-trips — so hand-written blocks survive a re-run of `init` (`wizard_run` reads them before writing; inner comments are lost); `tfWorkspaceIgnoreNames` is kept the same way (`W_IGNORE_NAMES_JSON`, never asked, shown in the review's scope row), and every other setting the wizard does not ask about (`tfProjects`, `cloudAuthVarPatterns`, hand-added keys) is appended verbatim from `W_PREV_JSON` under "Kept from the previous file"; `tfvars_missing_keys` (keys of `terraform.tfvars.example` the file lacks) and `tfvars_upgrade` (appends those paragraphs of the example — comment + default — under a dated header, `.bak` kept, touches nothing else; behind `init --upgrade`, prompt-free)), `scope.sh` (the run scope: `PROJECT_FILTER`/`WS_FILTER`/`WS_EXCLUDE`/`TAG_FILTER`/`TAG_EXCLUDE` arrays, `names_json`, `slug_of` (the transformer's project-slug rule in bash), `project_selected <seg>`, `ws_narrowed`/`ws_filter_json`/`ws_exclude_json` (tfvars `tfWorkspaceIgnoreNames` ∪ flags, memoized)/`ws_selected <name>` (bash `case` globs), `WS_SCOPE_JQ` + `ws_jq_args` → `WS_JQ_ARGS` for the jq sites, `scope_tags_json`/`scope_ignore_tags_json`, `scope_tfvar_args` → `SCOPE_TFVAR_ARGS` for apply, `scope_describe`, `scope_sha_input` for the apply phase hash; and the **run configuration overlay**: `--set KEY=VALUE` (`SET_VARS`, values via `_overlay_value`: HCL/JSON literals through hcl2json, bare text stays a string), `--cloud-connector`/`--vcs-connector`/`--runner-group`/`--workflow-group` → `overlay_build` looks the connector kinds up with `sg_list_integrations`/`sg_integration_type`/`sg_vcs_kind_of`, resolves `--project` values to raw TFC names (`_overlay_project_name`, fatal for an unknown project, as-given with a warning when TFC is unreachable) and fills `TFVARS_OVERLAY_JSON` — `projectOverrides[<name>]` fields with `--project`, `SGDefault*` keys without; `tfvars_json` deep-merges it over the file so every reader sees it, `scope_tfvar_args` passes each overlay key as `-var`, `scope_sha_input` includes it, `write_run_result` records it under `configuration`; `main` refuses the flags for `init`/`clean`/`completion` and a standalone `import` warns that they only reach the workflows through the export (`RAN_APPLY`)), `tfc_api.sh` (`tfc_http`/`tfc_get_all` paginated, `tfc_list_{orgs,projects,workspaces}` — workspaces include `vcs_provider`/`vcs_url`/`vcs_identifier` —, `tfc_select_workspaces` (the module's name-glob / include-tags(all) / exclude-tags / exclude-name-globs filter, shared by wizard and preflight), `tfc_vcs_summary`/`tfc_vcs_kind_for`/`tfc_vcs_label_for`, `tfc_token` with provider precedence, `require_tfc_auth`), `sg_api.sh` (`_sg_api` 0/22/1 contract + `SG_HTTP_CODE`, `_sg_listall` paginated listall reader (the API pages at 50) behind `sg_list_workflows`/`sg_list_wfgrps`, `sg_set_vcs_triggers`, `sg_list_integrations`, `sg_integration_type`, `sg_vcs_kind_of` (connector type → `sourceConfigDestKind`), `sg_execution_preset` + `sg_preset_{desc,runner_desc,version_desc,runner_provided}` (the org's execution preset and how to describe it), `sg_*_exists`, `sg_list_workflows`, `sg_create_secret`, `sg_patch_workflow`, `wfgroup_*`), `wizard.sh` (`wizard_run`: 5 steps TFC → SG → projects → defaults → review; `wizard_projects` (with >1 selected project, from `W_SEL_PROJECTS_JSON`) asks whether the global connectors and `tfc-<project>` groups apply to all, else per project the cloud connector, VCS connector (kind + prefix follow) and group (default / existing via `sg_list_wfgrps` / typed), writing `projectOverrides`; "same as the default" deletes the key, unmanaged fields are kept, previous picks come first (`_w_first`); the defaults step also asks `stripCloudAuthVars`; the TFC step first asks which TFC projects to migrate (with >1 project: `all` or `pick` by name/slug, resolved to raw names, unknown names re-asked; a previous non-empty `tfProjects` makes `pick` the first item; free text without discovery) → `W_PROJECTS_JSON`, rendered as `tfProjects` (`[]` = all) and applied to the workspace selection before the scope question; it keeps the workspace list and derives, via `tfc_vcs_summary`, which VCS provider the selected workspaces use and the repositories' URL prefix — the SG step lists matching connectors first and takes the repo URL prefix from TFC (never from a previous tfvars written for another provider kind), warns on a kind mismatch, reads the org's execution preset and offers it as the runner choice; the defaults step asks whether to carry TFC versions or use the preset (`SGTerraformVersionSource`) and, for carry, which fallback (`SGDefaultTerraformVersion` or the preset = null); the review shows human-readable rows plus the `tfc-<project>` groups; `_w_select_lines` feeds menus line-by-line so names with spaces/globs survive), `preflight.sh` (`preflight_run <apply|import|all>`, once per process; ✓ lines are plain language and name the connector's role/kind — related passing checks share one line (connector + kind + repo prefix, the project inside the selection line, version + runners when both come from the preset, one summary for the override keys) and routine ones are only counted (`pf_pass`), while ✗/! lines always print and name the tfvars field to fix; connector ids, runner groups and DPC kinds are collected from `SGDefault*`, `projectOverrides` and `workspaceOverrides`; besides existence it checks consistency — VCS kind vs. the connector's type, globally and per `projectOverrides` entry (fail), repo URL prefix vs. where the TFC repositories live or the kind's well-known host (warn), `projectOverrides` keys vs. the TFC project names (warn), `--project` values vs. the project names/slugs (fail; a `tfProjects` typo warns), and the workspace-selection preview applies the CLI scope (names, excludes, tags, projects) so its count is what the apply will export; the import context states the group policy and warns when a project's group changed since its last import; it also reports the org's execution preset next to the version/runner policy and warns when `carry` meets a preset that mounts a runner-provided binary), `state.sh` (`.sg/state.json`: `phases.<name>.input_sha`, `import.<seg>.{group,imported,updated,failed,tf_fallback,state_uploaded,state_failed}` (`group` anchors the never-move check; `updated` = PATCHed on a 409), `triggers.<seg>.{set,unchanged,failed,missing,sha}`, `secrets.<name>`; `run_phase` in migrate.sh skips unchanged phases), `report.sh` (`show_migration_summary` after apply — incl. a state-files row —, `show_import_plan` per-workflow table with columns sized to their content — it keeps the rows in `PLAN_ROWS_TSV` (name, group, action, version, runner, triggers, vars, secrets, segment, workspace, project) —, `write_run_result <outcome>` → `export/run-result.json` + `run-summary.md` from `PLAN_ROWS_TSV`, `PLAN_GROUP_ROWS` (plain copy of the group table, set by `plan_groups`), `PLAN_PROBLEMS`, the merged state and `CHECKLIST_OPEN`; per workflow `result` is planned/skipped/failed/updated/created, with the probe workflow — planned `create`, PATCHed on its second pass — reported as created), `errors.sh` (`explain_api_error` regex→hint table, `SG_API_HINTS`), `checklist.sh` (`create_secret_stubs` → SG secret `tfc-<wf>-<VAR>` = `CHANGE_ME`, referenced as `${secret::<name>}` via PATCH + payload patch; `write_checklist` → `export/post-import-checklist.md`, printed to the terminal as one ✓/! status line per section plus the imported groups with UI links — the file has the details; section 5 covers both the TFC state export and the SG state upload (`state_failed` from the import results); sets `CHECKLIST_OPEN`). Parallel jobs (`run_parallel`) run in subshells, so `do_import`/`do_set_triggers` write `.import-result.<seg>.json` / `.triggers-result.<seg>.json` into the export dir and the caller merges them into state. - `scripts/tools.sh` — sourced by the other scripts (sets `SG_REPO_ROOT` to its parent dir). Provides `sg_resolve` (PATH-or-cache), `sg_ensure_{jq,hcl2json,yajsv,sgcli}` (pinned downloads into `.sg/cached/`; `sg-cli` is the Go binary), `sg_retry`, the color vars + log helpers. The Docker image bundles these tools at build time (`yajsv` built from source so arm64 works), so in-container runs never download. - **Variable sets** — `scripts/enrich_variable_sets.sh <tfc-org> <payload...>` (run by `cmd_enrich`, which reads `tfOrg`/`tfHostname` from `terraform.tfvars` via `hcl2json` — variable sets live in the **TFC** org, not `SG_ORG`). Lists all sets + vars via the TFC API (Bearer token from the `terraform login` creds file or `TFE_TOKEN`), computes per-workspace effective vars (scope: global/project/workspace; precedence: priority sets > workspace vars > non-priority sets), and merges them into each workflow (matched by workspace name parsed from `TfStateFilePath`). Sensitive set vars and key conflicts are reported. Skipped by `--no-variable-sets`. - **Regions** — `--region eu|us` / `SG_REGION` (`region_urls`/`region_apply` in migrate.sh) pick `SG_BASE_URL` and `SG_UI_URL` together: eu = `api.app`/`app.stackguardian.io` (default), us = `api.us`/`us.stackguardian.io`, and the undocumented internal `dash` = `testapi.qa`/`dash.qa.stackguardian.io` (never mention it in user-facing docs). An explicit `SG_BASE_URL`/`SG_UI_URL` wins over the region. `init` stores `config.sg_region` in `.sg/state.json` (older files only have `sg_base_url`, still honoured); the plan header and `run-result.json` (`region`, `apiUrl`, `uiUrl`) show what a run used. The wrapper forwards `SG_REGION`. diff --git a/scripts/lib/preflight.sh b/scripts/lib/preflight.sh index 7af4e00..903bf57 100644 --- a/scripts/lib/preflight.sh +++ b/scripts/lib/preflight.sh @@ -5,13 +5,18 @@ # work and that everything terraform.tfvars refers to actually exists in TFC # and StackGuardian — and that the pieces agree with each other (VCS connector # kind vs. sourceConfigDestKind, repo URL prefix vs. where the TFC repositories -# live). Prints one line per check (✓ ok, ! warning, ✗ failure) and fails the -# run when any check fails. Skipped with --skip-preflight. +# live). Prints ✓ / ! / ✗ lines and fails the run when any check fails. +# Skipped with --skip-preflight. Not every passing check gets its own line: +# related ones are reported together (one line per connector, the project +# inside the selection line, version and runners) and a few routine ones are +# only counted (pf_pass) — a failing check always prints, naming the tfvars +# field to fix. PF_OK=0 PF_FAIL=0 PF_WARN=0 pf_ok() { printf ' %s✓%s %s\n' "$C_GREEN" "$C_RESET" "$*" >&2; PF_OK=$((PF_OK + 1)); } +pf_pass() { PF_OK=$((PF_OK + 1)); } pf_warn() { printf ' %s!%s %s\n' "$C_YELLOW" "$C_RESET" "$*" >&2; PF_WARN=$((PF_WARN + 1)); } pf_fail() { printf ' %s✗%s %s\n' "$C_RED$C_BOLD" "$C_RESET" "$*" >&2; PF_FAIL=$((PF_FAIL + 1)); } @@ -35,9 +40,9 @@ preflight_tfc() { esac return 0 fi - pf_ok "TFC credentials valid ($host, via $(tfc_token_source))" + pf_pass # credentials; reported together with the organisation below if tfc_http "organizations/$org" >/dev/null; then - pf_ok "TFC organisation '$org' accessible" + pf_ok "TFC organisation '$org' accessible ($host, via $(tfc_token_source))" else case "$TFC_HTTP_CODE" in 404 | 403) pf_fail "TFC organisation '$org' not found or not accessible with this token (HTTP $TFC_HTTP_CODE) — check tfOrg" ;; @@ -49,7 +54,7 @@ preflight_tfc() { # tags, exclude names) with the CLI scope applied (lib/scope.sh, when loaded), # so the count is the one the apply that follows will export. if body="$(tfc_list_workspaces "$org" 2>/dev/null)"; then - local names ignore_names tags ignore_tags projects from_cli=0 p slug hit + local names ignore_names tags ignore_tags projects from_cli=0 p slug hit in_projects="" PF_TFC_PROJECTS="$(tfc_list_projects "$org" 2>/dev/null || echo '[]')" names="$(tfvars_get_json .workspacenames)" ignore_names="$(tfvars_get_json .tfWorkspaceIgnoreNames)" @@ -73,7 +78,9 @@ preflight_tfc() { slug="$(printf '%s' "$p" | tr '[:upper:]' '[:lower:]' | sed -E 's/[^a-z0-9-]+/-/g')" hit="$(printf '%s' "$PF_TFC_PROJECTS" | "$jqb" -r --arg s "$slug" '[.[] | select((.name | ascii_downcase | gsub("[^a-z0-9-]+"; "-")) == $s) | .name] | first // empty')" if [ -n "$hit" ]; then - pf_ok "project '$p' is TFC project '$hit'" + # Named as in TFC: part of the selection line below; a slug is spelled out. + if [ "$p" = "$hit" ]; then pf_pass; else pf_ok "project '$p' is TFC project '$hit'"; fi + in_projects="$in_projects${in_projects:+, }'$hit'" elif [ "$from_cli" -eq 1 ]; then pf_fail "--project '$p' matches no TFC project in '$org' (projects: $(printf '%s' "$PF_TFC_PROJECTS" | "$jqb" -r '[.[].name] | join(", ")'))" else @@ -87,7 +94,7 @@ preflight_tfc() { fi n="$(printf '%s' "$sel" | "$jqb" 'length')" if [ "$n" -gt 0 ]; then - pf_ok "$n workspace(s) match the selection (of $(printf '%s' "$body" | "$jqb" 'length') in the org)" + pf_ok "$n workspace(s) match the selection${in_projects:+ in project(s) $in_projects} (of $(printf '%s' "$body" | "$jqb" 'length') in the org)" else local cli="" declare -F scope_describe >/dev/null && cli="$(scope_describe)" @@ -140,7 +147,9 @@ preflight_sg() { /integrations/*) if printf '%s' "$names" | "$jqb" -e --arg n "${id#/integrations/}" 'index($n) != null' >/dev/null; then kind="$(sg_integration_type "$ints" "$id")" - pf_ok "$label connector ${id#/integrations/} exists${kind:+ ($kind)}" + # The default VCS connector is reported by preflight_config together + # with its kind and the repo URL prefix (one line per connector). + if [ "$label" = "VCS" ]; then pf_pass; else pf_ok "$label connector ${id#/integrations/}${kind:+ ($kind)}"; fi else pf_fail "$label connector $id not found in org '$ORG' (available: $(printf '%s' "$names" | "$jqb" -r 'join(", ")'))" fi @@ -171,11 +180,10 @@ preflight_sg() { ((.workspaceOverrides // {}) | to_entries[]? | .value.RunnerConstraints // {} | select(.type == "private") | .names[]?) ] | unique | .[]') if tfvars_is_null SGDefaultRunnerConstraints; then - if [ "${PF_PRESET_READ:-0}" -eq 1 ]; then - pf_ok "runners: from the org's execution preset — $(sg_preset_runner_desc "$PF_PRESET")" - else - pf_ok "runners: from the org's execution preset (platform default: shared runners)" - fi + # Reported by preflight_config next to the Terraform version policy (one + # line when both come from the preset). + PF_RUNNERS_PRESET=1 + pf_pass else n="$(tfvars_json | "$jqb" -r '(.SGDefaultRunnerConstraints // {}).type // "shared"')" if [ "$n" = "shared" ]; then pf_ok "runners: StackGuardian shared runners"; fi @@ -211,12 +219,15 @@ preflight_config() { conn="$(tfvars_get .SGDefaultVCSAuthIntegrationID)" conn_kind="" [ -n "${PF_SG_INTS:-}" ] && [ -n "$conn" ] && conn_kind="$(sg_vcs_kind_of "$(sg_integration_type "$PF_SG_INTS" "$conn")")" + # One line for the VCS side: connector (verified in preflight_sg), kind and + # — when it checks out below — the repo URL prefix. + local vcs_line="" prefix_ok="" if [ -n "$conn_kind" ] && [ "$conn_kind" != "$v" ]; then pf_fail "VCS kind $v does not match connector ${conn#/integrations/}, which is a $conn_kind connector — set SGDefaultSourceConfigDestKind = \"$conn_kind\" (or pick another connector)" elif [ -n "$conn_kind" ]; then - pf_ok "VCS kind $v matches connector ${conn#/integrations/}" + vcs_line="VCS connector ${conn#/integrations/} ($v)" else - pf_ok "VCS kind $v" + vcs_line="VCS kind $v" fi # Repo URL prefix: compare with where the TFC workspaces' repositories live @@ -238,7 +249,7 @@ preflight_config() { pf_fail "SGDefaultIACVCSRepoPrefix is not set — the repositories' base URL, e.g. https://github.com" elif [ -n "$tfc_prefix" ]; then case "$host" in - *"$(_pf_host "$tfc_prefix")") pf_ok "repo URL prefix $prefix matches the TFC repositories" ;; + *"$(_pf_host "$tfc_prefix")") prefix_ok="repositories under $prefix, as in TFC" ;; *) pf_warn "repo URL prefix $prefix does not match where the TFC repositories live ($tfc_prefix) — every workflow would clone from the wrong place unless the repositories moved; check SGDefaultIACVCSRepoPrefix" ;; esac else @@ -246,9 +257,15 @@ preflight_config() { if [ -n "$kind" ] && [ "$kind" != "$v" ] && [ "$v" != "GIT_OTHER" ]; then pf_warn "repo URL prefix $prefix looks like $kind but the VCS kind is $v — check SGDefaultIACVCSRepoPrefix / SGDefaultSourceConfigDestKind" else - pf_ok "repo URL prefix $prefix" + prefix_ok="repositories under $prefix" fi fi + if [ -n "$vcs_line" ]; then + [ -n "$prefix_ok" ] && pf_pass # the prefix check, folded into the connector line + pf_ok "$vcs_line${prefix_ok:+, $prefix_ok}" + elif [ -n "$prefix_ok" ]; then + pf_ok "repo URL prefix $prefix" + fi if [ -n "$tfc_kind" ] && [ "$tfc_kind" != "$v" ] && [ "$v" != "GIT_OTHER" ]; then pf_warn "the TFC workspaces are connected to $(tfc_vcs_label_for "$prov") but the VCS kind is $v" fi @@ -270,14 +287,14 @@ preflight_config() { while IFS= read -r v; do [ -n "$v" ] || continue case "$v" in - AWS_STATIC | AWS_RBAC | AWS_OIDC | AZURE_STATIC | AZURE_OIDC | AZURE_MANAGED_ID_OIDC | GCP_STATIC | GCP_OIDC) pf_ok "cloud connector kind $v" ;; + AWS_STATIC | AWS_RBAC | AWS_OIDC | AZURE_STATIC | AZURE_OIDC | AZURE_MANAGED_ID_OIDC | GCP_STATIC | GCP_OIDC) pf_pass ;; # shown with the connector (preflight_sg) *) pf_fail "cloud connector kind '$v' (DeploymentPlatformConfig) is not one of AWS_STATIC, AWS_RBAC, AWS_OIDC, AZURE_STATIC, AZURE_OIDC, AZURE_MANAGED_ID_OIDC, GCP_STATIC, GCP_OIDC — a VCS connector was picked as the cloud connector?" ;; esac done < <(tfvars_json | "$jqb" -r '[ (.SGDefaultDeploymentPlatformConfig // [])[]?.kind, ((.projectOverrides // {}) | to_entries[]? | .value.DeploymentPlatformConfig // [] | .[]?.kind), ((.workspaceOverrides // {}) | to_entries[]? | .value.DeploymentPlatformConfig // [] | .[]?.kind) ] | map(select(. != null)) | unique | .[]') # Cloud credential env vars: stripped per connector kind, or kept. if [ "$(tfvars_get .stripCloudAuthVars true)" != "false" ]; then - pf_ok "cloud credential variables (ARM_*, AWS_ACCESS_KEY_ID, GOOGLE_CREDENTIALS, ... per connector kind) are stripped — the connector provides them (stripCloudAuthVars)" + pf_ok "cloud credential env vars stripped, the connector provides them (stripCloudAuthVars)" else pf_ok "cloud credential variables are kept (stripCloudAuthVars = false)" fi @@ -295,13 +312,26 @@ preflight_config() { esac [ "${PF_PRESET_READ:-0}" -eq 1 ] && preset_note="$(sg_preset_desc "$PF_PRESET")" if tfvars_is_null SGDefaultTerraformVersion; then v=""; else v="$(tfvars_get .SGDefaultTerraformVersion TERRAFORM-1.5.7)"; fi + # Runners left to the preset (preflight_sg) are reported here: on the version + # line when that is preset too, else on their own. + local what="Terraform version" runners_note="" + if [ "${PF_RUNNERS_PRESET:-0}" -eq 1 ]; then + if [ "$src" = "preset" ]; then + what="Terraform version and runners" + elif [ "${PF_PRESET_READ:-0}" -eq 1 ]; then + runners_note="runners: from the org's execution preset — $(sg_preset_runner_desc "$PF_PRESET")" + else + runners_note="runners: from the org's execution preset (platform default: shared runners)" + fi + fi if [ "$src" = "preset" ]; then if [ -n "$preset_note" ]; then - pf_ok "Terraform version: from the org's execution preset — $preset_note" + pf_ok "$what: from the org's execution preset — $preset_note" else - pf_warn "Terraform version: from the org's execution preset, which could not be read here (platform default if none is configured: managed Terraform 1.5.7 on shared runners)" + pf_warn "$what: from the org's execution preset, which could not be read here (platform default: managed Terraform 1.5.7 on shared runners)" fi else + [ -n "$runners_note" ] && pf_ok "$runners_note" if [ -z "$v" ]; then pf_ok "Terraform version: pinned TFC versions are carried over; unpinned or rejected pins go to the execution preset${preset_note:+ — $preset_note}" elif [[ "$v" =~ ^TERRAFORM-([0-9]+)\.([0-9]+)\.([0-9]+)$ ]]; then @@ -321,15 +351,18 @@ preflight_config() { export_path="$(tfvars_get .exportPath export)" case "$export_path" in /*) want="$export_path" ;; *) want="$SG_REPO_ROOT/$export_path" ;; esac if [ "$(cd "$(dirname "$want")" 2>/dev/null && pwd)/$(basename "$want")" = "$(cd "$(dirname "$EXPORT_DIR")" 2>/dev/null && pwd)/$(basename "$EXPORT_DIR")" ]; then - pf_ok "export directory $(sg_rel "$EXPORT_DIR")/" + pf_pass # export directory agrees with exportPath else pf_warn "exportPath in tfvars ($export_path) differs from the orchestrator's export dir ($(sg_rel "$EXPORT_DIR")) — payloads would be written where later phases don't look" fi + # Override keys vs. TFC: one summary line for the matches, a warning per typo. + local n_ws_ov=0 n_pr_ov=0 ov_note="" if [ -n "${PF_TFC_WORKSPACES:-}" ]; then while IFS= read -r v; do [ -n "$v" ] || continue if printf '%s' "$PF_TFC_WORKSPACES" | "$jqb" -e --arg n "$v" '[.[].name] | index($n) != null' >/dev/null; then - pf_ok "workspaceOverrides['$v'] matches a workspace" + pf_pass + n_ws_ov=$((n_ws_ov + 1)) else pf_warn "workspaceOverrides['$v'] does not match any workspace in the org (typo?)" fi @@ -339,12 +372,16 @@ preflight_config() { while IFS= read -r v; do [ -n "$v" ] || continue if printf '%s' "$PF_TFC_PROJECTS" | "$jqb" -e --arg n "$v" '[.[].name] | index($n) != null' >/dev/null; then - pf_ok "projectOverrides['$v'] matches a TFC project" + pf_pass + n_pr_ov=$((n_pr_ov + 1)) else pf_warn "projectOverrides['$v'] matches no TFC project in the org (typo? projects: $(printf '%s' "$PF_TFC_PROJECTS" | "$jqb" -r '[.[].name] | join(", ")')) — its settings would apply to nothing" fi done < <(tfvars_json | "$jqb" -r '(.projectOverrides // {}) | keys[]') fi + [ "$n_pr_ov" -gt 0 ] && ov_note="$n_pr_ov projectOverrides" + [ "$n_ws_ov" -gt 0 ] && ov_note="$ov_note${ov_note:+ and }$n_ws_ov workspaceOverrides" + [ -n "$ov_note" ] && printf ' %s✓%s %s\n' "$C_GREEN" "$C_RESET" "$ov_note entries name existing TFC projects/workspaces" >&2 return 0 } @@ -390,6 +427,7 @@ preflight_run() { PF_SG_INTS="" PF_PRESET="{}" PF_PRESET_READ=0 + PF_RUNNERS_PRESET=0 local parse_err if ! parse_err="$(tfvars_valid)"; then pf_fail "$(sg_rel "$TFVARS") is not valid HCL: ${parse_err:-parse error}" From c93ab6810e982b7edc5cf217dea0ad77c43132e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adis=20Halilovi=C4=87?= <adis.halilovic@stackguardian.io> Date: Mon, 14 Sep 2026 18:30:14 +0200 Subject: [PATCH 62/71] fix: empty-object defaults expanded to a literal {\} under macOS bash 3.2, so a native enrich merged nothing --- scripts/lib/report.sh | 4 ++-- scripts/lib/scope.sh | 2 +- scripts/lib/tfvars.sh | 7 +++++-- scripts/lib/wizard.sh | 10 +++++----- 4 files changed, 13 insertions(+), 10 deletions(-) diff --git a/scripts/lib/report.sh b/scripts/lib/report.sh index 188f8d8..ac50ccf 100644 --- a/scripts/lib/report.sh +++ b/scripts/lib/report.sh @@ -192,12 +192,12 @@ write_run_result() { state_read | "$JQ_BIN" --arg cmd "$PROG ${SG_RUN_ARGS:-}" --arg at "$(state_now)" --argjson took "$((SECONDS - RUN_T0))" \ --arg org "$ORG" --arg url "$SG_BASE_URL" --arg region "${SG_REGION:-}" --arg ui "${SG_UI_URL:-}" --argjson dry "$([ "${DRY_RUN:-0}" -eq 1 ] && echo true || echo false)" \ --arg outcome "$outcome" --argjson scope "$scope" --argjson groups "${PLAN_GROUP_ROWS:-[]}" \ - --argjson overlay "${TFVARS_OVERLAY_JSON:-{\}}" --arg overlay_desc "$(declare -F overlay_describe >/dev/null && overlay_describe || true)" \ + --argjson overlay "${TFVARS_OVERLAY_JSON:-null}" --arg overlay_desc "$(declare -F overlay_describe >/dev/null && overlay_describe || true)" \ --argjson problems "$problems" --argjson rows "$rows" --argjson open "${CHECKLIST_OPEN:-0}" ' . as $st | { command: $cmd, at: $at, tookSeconds: $took, org: $org, region: $region, apiUrl: $url, uiUrl: $ui, dryRun: $dry, outcome: $outcome, - scope: $scope, configuration: {description: $overlay_desc, overlay: $overlay}, groups: $groups, problems: $problems, + scope: $scope, configuration: {description: $overlay_desc, overlay: ($overlay // {})}, groups: $groups, problems: $problems, workflows: [ $rows[] | . as $r | ($st.import[$r.segment] // {}) as $imp | ($st.triggers[$r.segment] // {}) as $tr | . + { diff --git a/scripts/lib/scope.sh b/scripts/lib/scope.sh index 7d7ecc8..d6f3d8f 100644 --- a/scripts/lib/scope.sh +++ b/scripts/lib/scope.sh @@ -138,7 +138,7 @@ scope_tfvar_args() { # scope flag on the same variable wins because terraform takes the last -var. while IFS= read -r k; do [ -n "$k" ] && SCOPE_TFVAR_ARGS+=(-var "$k=$(tfvars_get_json ".$k")") - done < <(printf '%s' "${TFVARS_OVERLAY_JSON:-{\}}" | "$JQ_BIN" -r 'keys[]') + done < <(printf '%s' "${TFVARS_OVERLAY_JSON:-"{}"}" | "$JQ_BIN" -r 'keys[]') [ "${#PROJECT_FILTER[@]}" -gt 0 ] && SCOPE_TFVAR_ARGS+=(-var "tfProjects=$(names_json "${PROJECT_FILTER[@]}")") [ "${#WS_FILTER[@]}" -gt 0 ] && SCOPE_TFVAR_ARGS+=(-var "workspacenames=$(names_json "${WS_FILTER[@]}")") [ "${#WS_EXCLUDE[@]}" -gt 0 ] && SCOPE_TFVAR_ARGS+=(-var "tfWorkspaceIgnoreNames=$(ws_exclude_json)") diff --git a/scripts/lib/tfvars.sh b/scripts/lib/tfvars.sh index 20d757e..df165c2 100644 --- a/scripts/lib/tfvars.sh +++ b/scripts/lib/tfvars.sh @@ -12,7 +12,10 @@ _TFVARS_JSON_FOR="" # values the run actually uses. Objects merge deeply, so an overlay entry for # one project keeps the other projectOverrides of the file. tfvars_json() { - local overlay="${TFVARS_OVERLAY_JSON:-{\}}" + # A brace default "${var:-{\}}" is a literal {\} under bash 3.2 (macOS /bin/bash); + # "${var:-"{}"}" is the form that works everywhere. + local overlay="${TFVARS_OVERLAY_JSON:-}" + [ -n "$overlay" ] || overlay='{}' if [ -z "$_TFVARS_JSON" ] || [ "$_TFVARS_JSON_FOR" != "$TFVARS|$overlay" ]; then if [ -f "$TFVARS" ]; then _TFVARS_JSON="$("$(sg_resolve hcl2json sg_ensure_hcl2json)" "$TFVARS" 2>/dev/null || echo '{}')" @@ -89,7 +92,7 @@ _tfvars_map() { | " \($k | tojson) = {" + (if ($notes[$k] // "") != "" then " # \($notes[$k])" else "" end) + "\n" + ([.value | to_entries[] | " \(.key | pad($w)) = \(.value | hcl)"] | join("\n")) + "\n }"] | join("\n")) + "\n}" end' 2>/dev/null)" || out="" - printf '%s' "${out:-\{\}}" + printf '%s' "${out:-"{}"}" } # _tfvars_map_commented <json-object> [notes-json] — the map's entries (without diff --git a/scripts/lib/wizard.sh b/scripts/lib/wizard.sh index 1a63334..a3f3e60 100644 --- a/scripts/lib/wizard.sh +++ b/scripts/lib/wizard.sh @@ -604,9 +604,9 @@ wizard_review() { elif [ -n "${W_GROUPS:-}" ]; then sg_row "Workflow groups" "$W_GROUPS" fi - k="$(printf '%s' "${W_PROJECT_OVERRIDES_JSON:-{\}}" | "$(sg_resolve jq sg_ensure_jq)" 'length' 2>/dev/null || echo 0)" + k="$(printf '%s' "${W_PROJECT_OVERRIDES_JSON:-"{}"}" | "$(sg_resolve jq sg_ensure_jq)" 'length' 2>/dev/null || echo 0)" [ -z "${W_PROJECT_ROWS:-}" ] && [ "$k" -gt 0 ] && sg_row "Project settings" "$k projectOverrides entr(y/ies) kept from the current file" - k="$(printf '%s' "${W_WS_OVERRIDES_JSON:-{\}}" | "$(sg_resolve jq sg_ensure_jq)" 'length' 2>/dev/null || echo 0)" + k="$(printf '%s' "${W_WS_OVERRIDES_JSON:-"{}"}" | "$(sg_resolve jq sg_ensure_jq)" 'length' 2>/dev/null || echo 0)" [ "$k" -gt 0 ] && sg_row "Workspace overrides" "$k workspaceOverrides entr(y/ies) kept from the current file" sg_row "StackGuardian org" "$ORG" sg_row "VCS connector" "${W_VCS_INTEGRATION#/integrations/} ($W_DEST_KIND) — repositories under $W_REPO_PREFIX" @@ -632,7 +632,7 @@ wizard_review() { sg_row "Terraform version" "$tf" [ "${W_DPC_PLACEHOLDER:-0}" -eq 1 ] && sg_warn "cloud connector left as a placeholder — edit SGDefaultDeploymentPlatformConfig in $(sg_rel "$TFVARS") before 'apply'" [ "${W_TFC_VCS_OTHER:-0}" -gt 0 ] && sg_warn "$W_TFC_VCS_OTHER workspace(s) use a different VCS provider than the default above — give them their own connector/prefix via projectOverrides or workspaceOverrides in $(sg_rel "$TFVARS")" - k="$(printf '%s' "${W_WS_TEMPLATE_JSON:-{\}}" | "$(sg_resolve jq sg_ensure_jq)" 'length' 2>/dev/null || echo 0)" + k="$(printf '%s' "${W_WS_TEMPLATE_JSON:-"{}"}" | "$(sg_resolve jq sg_ensure_jq)" 'length' 2>/dev/null || echo 0)" [ "$k" -gt 0 ] && sg_dim "the file also gets a commented, ready-to-uncomment entry per project and per workspace ($k) for later fine-tuning" sg_dim "approvers and the repo URL prefix can be edited in $(sg_rel "$TFVARS")" sg_confirm "Write $(sg_rel "$TFVARS")?" Y @@ -650,7 +650,7 @@ wizard_templates() { [ "$runner" = "null" ] && runner='{"type":"shared"}' tfv="${W_TF_VERSION:-null}" [ "$tfv" = "null" ] && tfv="TERRAFORM-1.5.7" - W_PROJECT_TEMPLATE_JSON="$(printf '%s' "${W_SEL_PROJECTS_JSON:-[]}" | "$jqb" -c --argjson have "${W_PROJECT_OVERRIDES_JSON:-{\}}" \ + W_PROJECT_TEMPLATE_JSON="$(printf '%s' "${W_SEL_PROJECTS_JSON:-[]}" | "$jqb" -c --argjson have "${W_PROJECT_OVERRIDES_JSON:-"{}"}" \ --argjson dpc "${W_DPC_JSON:-[]}" --arg vcs "${W_VCS_INTEGRATION:-}" --argjson runner "$runner" --argjson appr "${W_APPROVERS_JSON:-[]}" ' map(select(.name as $n | ($have | has($n)) | not)) | map({key: .name, value: {workflowGroup: ("tfc-" + .segment), DeploymentPlatformConfig: $dpc, vcsAuthIntegrationID: $vcs, RunnerConstraints: $runner, Approvers: $appr}}) @@ -658,7 +658,7 @@ wizard_templates() { W_PROJECT_TEMPLATE_NOTES="$(printf '%s' "${W_SEL_PROJECTS_JSON:-[]}" | "$jqb" -c 'map({key: .name, value: "\(.count) workspace(s)"}) | from_entries' 2>/dev/null || echo '{}')" # A workspace's template shows what it gets today: its project's override # where one exists, else the global pick. - W_WS_TEMPLATE_JSON="$(printf '%s' "${W_SEL_WS_JSON:-[]}" | "$jqb" -c --argjson have "${W_WS_OVERRIDES_JSON:-{\}}" --argjson projects "${W_PROJECT_OVERRIDES_JSON:-{\}}" --argjson pr "${W_SEL_PROJECTS_JSON:-[]}" \ + W_WS_TEMPLATE_JSON="$(printf '%s' "${W_SEL_WS_JSON:-[]}" | "$jqb" -c --argjson have "${W_WS_OVERRIDES_JSON:-"{}"}" --argjson projects "${W_PROJECT_OVERRIDES_JSON:-"{}"}" --argjson pr "${W_SEL_PROJECTS_JSON:-[]}" \ --argjson dpc "${W_DPC_JSON:-[]}" --arg vcs "${W_VCS_INTEGRATION:-}" --argjson runner "$runner" --argjson appr "${W_APPROVERS_JSON:-[]}" --arg tfv "$tfv" ' ($pr | map({key: .id, value: .name}) | from_entries) as $names | sort_by(.name) From a5600e89a73fd0ee8d5d603ec312637f4d635f67 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adis=20Halilovi=C4=87?= <adis.halilovic@stackguardian.io> Date: Mon, 14 Sep 2026 18:30:14 +0200 Subject: [PATCH 63/71] fix: variable-set phase reports the sets and the files that gained variables on one line each; details with -v --- scripts/enrich_variable_sets.sh | 30 +++++++++++++++++++++++------- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/scripts/enrich_variable_sets.sh b/scripts/enrich_variable_sets.sh index 5ef520c..56217f3 100755 --- a/scripts/enrich_variable_sets.sh +++ b/scripts/enrich_variable_sets.sh @@ -10,6 +10,8 @@ # vars can't be read from the API and are skipped + reported. # # Usage: enrich_variable_sets.sh <org> <payload.json> [more.json ...] +# Output: one line naming the sets and one per payload that gained variables; +# SG_VERBOSE=1 adds the fetch step and each set's scope and size. set -uo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" @@ -20,6 +22,7 @@ source "$SCRIPT_DIR/lib/tfvars.sh" # shellcheck source=lib/tfc_api.sh source "$SCRIPT_DIR/lib/tfc_api.sh" TFVARS="${TFVARS:-$SG_REPO_ROOT/transformer/terraform-cloud/terraform.tfvars}" +VERBOSE="${SG_VERBOSE:-0}" ORG="${1:-}" shift || true @@ -48,7 +51,7 @@ trap 'rm -rf "$WORK"' EXIT # fetch_all <path> — paginated GET, merged .data array (lib/tfc_api.sh). fetch_all() { tfc_get_all "$1"; } -sg_log "fetching workspaces and variable sets from $HOST (org: $ORG)..." +[ "$VERBOSE" -eq 1 ] && sg_log "fetching workspaces and variable sets from $HOST (org: $ORG)..." # name -> {id, project} fetch_all "organizations/$ORG/workspaces" | @@ -78,9 +81,11 @@ if [ "$set_count" -eq 0 ]; then sg_log "no variable sets found; nothing to enrich" exit 0 fi -sg_log "resolving $set_count variable set(s) across workspaces:" -# One line per set: name, scope, size — so it is clear where merged vars come from. -"$JQ_BIN" -r '.[] | " - \(.name): \(if .global then "global" else ([(if (.projids | length) > 0 then "\(.projids | length) project(s)" else empty end), (if (.wsids | length) > 0 then "\(.wsids | length) workspace(s)" else empty end)] | if length == 0 then "unassigned" else join(", ") end) end), \(.vars | length) var(s)\(if .priority then ", priority" else "" end)"' "$WORK/sets.json" >&2 +# The sets by name; with -v one line per set (scope, size) so it is clear +# where merged vars come from. +set_names="$("$JQ_BIN" -r '[.[].name] | if length > 6 then (.[:6] | join(", ")) + ", ... \(length - 6) more" else join(", ") end' "$WORK/sets.json")" +[ "$VERBOSE" -eq 1 ] && sg_log "resolving $set_count variable set(s) across workspaces:" +[ "$VERBOSE" -eq 1 ] && "$JQ_BIN" -r '.[] | " - \(.name): \(if .global then "global" else ([(if (.projids | length) > 0 then "\(.projids | length) project(s)" else empty end), (if (.wsids | length) > 0 then "\(.wsids | length) workspace(s)" else empty end)] | if length == 0 then "unassigned" else join(", ") end) end), \(.vars | length) var(s)\(if .priority then ", priority" else "" end)"' "$WORK/sets.json" >&2 # Per workspace -> list of winning vars (set-vs-set precedence resolved; tagged # with priority + sensitive + conflict). rank: non-priority global/proj/ws = 1/2/3, @@ -126,7 +131,10 @@ CLOUD_JSON="$(tfvars_get_json .cloudAuthVarPatterns)" "$JQ_BIN" -s '[.[][] | {key: ((.CLIConfiguration.TfStateFilePath // "") | sub(".*/"; "") | sub("\\.tfstate$"; "")), value: ((.DeploymentPlatformConfig[0].kind // "") | split("_")[0])}] | from_entries' "$@" >"$WORK/cloud.json" -# Merge the effective set vars into each payload, then report counts. +# Merge the effective set vars into each payload, then report counts: the +# files that gained variables get a line each, the rest are only counted. +unchanged=0 +changed_lines=() for f in "$@"; do before_tf="$("$JQ_BIN" '[.[].VCSConfig.iacInputData.data | length] | add // 0' "$f")" before_env="$("$JQ_BIN" '[.[].EnvironmentVariables | length] | add // 0' "$f")" @@ -158,11 +166,19 @@ for f in "$@"; do after_tf="$("$JQ_BIN" '[.[].VCSConfig.iacInputData.data | length] | add // 0' "$f")" after_env="$("$JQ_BIN" '[.[].EnvironmentVariables | length] | add // 0' "$f")" if [ "$((after_tf - before_tf + after_env - before_env))" -eq 0 ]; then - sg_log "$(basename "$f"): no new variables (sets only override or add nothing here)" + unchanged=$((unchanged + 1)) else - sg_log "$(basename "$f"): +$((after_tf - before_tf)) terraform, +$((after_env - before_env)) env var(s) from variable sets" + changed_lines+=("$(basename "$f"): +$((after_tf - before_tf)) terraform, +$((after_env - before_env)) env var(s) from variable sets") fi done +if [ "$unchanged" -eq 0 ]; then + sg_log "$set_count variable set(s) resolved ($set_names)" +elif [ "$unchanged" -eq "$#" ]; then + sg_log "$set_count variable set(s) resolved ($set_names); no new variables for the exported workspaces (the sets only override or add nothing here)" +else + sg_log "$set_count variable set(s) resolved ($set_names); $unchanged payload file(s) gained nothing" +fi +for line in ${changed_lines[@]+"${changed_lines[@]}"}; do sg_log "$line"; done # Report cloud credential set vars stripped (the connector provides them), # then sensitive set vars (cannot be migrated; stripped ones excluded) and key From 8a92d56f8297d3fcc995c49df6102b1ad020df7f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adis=20Halilovi=C4=87?= <adis.halilovic@stackguardian.io> Date: Mon, 14 Sep 2026 18:36:37 +0200 Subject: [PATCH 64/71] fix: the per-file trigger line leaves out zero counts --- scripts/migrate.sh | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/scripts/migrate.sh b/scripts/migrate.sh index f3fec37..ef92504 100755 --- a/scripts/migrate.sh +++ b/scripts/migrate.sh @@ -578,7 +578,14 @@ do_set_triggers() { rc=1 fi done - sg_log "$(basename "$f"): triggers set on ${#ok[@]} workflow(s), ${#unchanged[@]} unchanged, skipped $skip without triggers" + # One line per file; parts that are zero are left out. + local parts="" + [ "${#ok[@]}" -gt 0 ] && parts="triggers set on ${#ok[@]} workflow(s)" + [ "${#unchanged[@]}" -gt 0 ] && parts="$parts${parts:+, }${#unchanged[@]} unchanged" + [ "$skip" -gt 0 ] && parts="$parts${parts:+, }$skip without triggers" + [ "${#failed[@]}" -gt 0 ] && parts="$parts${parts:+, }${#failed[@]} failed" + [ "${#missing[@]}" -gt 0 ] && parts="$parts${parts:+, }${#missing[@]} not in SG" + sg_log "$(basename "$f"): ${parts:-no workflows with triggers}" "$JQ_BIN" -nc --arg g "$grp" --argjson sha "$shas" \ --argjson ok "$([ "${#ok[@]}" -gt 0 ] && names_json "${ok[@]}" || echo '[]')" \ --argjson unchanged "$([ "${#unchanged[@]}" -gt 0 ] && names_json "${unchanged[@]}" || echo '[]')" \ From 64174639815dd8cc2b1a2c1e04558a98bdb35f4f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adis=20Halilovi=C4=87?= <adis.halilovic@stackguardian.io> Date: Mon, 14 Sep 2026 18:36:37 +0200 Subject: [PATCH 65/71] fix: import prints one line per workflow (created/updated, state) instead of sg-cli's raw output; -v keeps it --- CLAUDE.md | 2 +- scripts/migrate.sh | 141 +++++++++++++++++++++++++++++++-------------- scripts/tools.sh | 5 ++ 3 files changed, 105 insertions(+), 43 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 0f3860e..18eed4d 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -24,7 +24,7 @@ The migration is a user-driven pipeline, not a single program: The five phases are wrapped by an orchestrator so users don't run them by hand: - `sg-migrate.sh` (host entrypoint, repo root) — runs `scripts/migrate.sh` **inside the Docker image** (`Dockerfile`), bind-mounting the repo at `/app` and forwarding `SG_API_TOKEN`/`SG_ORG`/`SG_BASE_URL`/`TFE_TOKEN`/`TF_TOKEN_*`. `--tfvars FILE` (or `SG_TFVARS`) is a host path: the wrapper strips the flag, resolves it and passes it on as `SG_TFVARS` — rewritten to `/app/<relative>` when the file is inside the checkout, otherwise mounted read-only at `/tmp/sg-run.tfvars`. For TFC auth it prefers a long-lived `TFE_TOKEN` (env); otherwise it mounts `~/.terraform.d/credentials.tfrc.json` read-only. Runs natively instead when `--native`/`--local` is passed, `SG_NATIVE=1` is set, the command is `clean`/`completion`/help (or no command is given), or Docker is absent. Exports `SG_PROG` so `migrate.sh` shows `./sg-migrate.sh` in its usage/hints. `update` is host-only too (`cmd_update` in the wrapper, never passed to `migrate.sh`): refuses a non-git checkout, a detached HEAD or dirty tracked files, then `git pull --ff-only` and rebuilds the image only when the `Dockerfile` changed between the old and new HEAD — this is how customers pick up fixes from a plain clone (untracked config/output survive). `migrate.sh` lists `update` in its usage/`SG_COMMANDS` and rejects it with a host-only hint. `.gitattributes` forces LF so a Git-for-Windows clone does not CRLF the scripts. -- `scripts/migrate.sh` — the actual orchestrator (sources `scripts/lib/*.sh`, see below). Subcommands `init|preflight|apply|enrich|convert|validate|import|triggers|checklist|all|clean|completion` (no command prints the help menu; `enrich` runs in `all` unless `--no-variable-sets`). Flags beyond the basics: `--dry-run` (import, or all: the local export runs, then the plan is shown and nothing is created in SG), `--fresh` (ignore run state), the scope flags from `lib/scope.sh` — `--project NAME|SLUG`, `--workspace GLOB`, `--exclude-workspace GLOB`, `--tag NAME`, `--exclude-tag NAME`, all repeatable; tfvars holds the widest scope, include flags replace the tfvars list, exclude flags add to it; apply passes them as `-var` (`tfProjects`, `workspacenames`, `tfWorkspaceIgnoreNames`, `tfWorkspaceTags`, `tfWorkspaceIgnoreTags`), the later phases apply the same names/excludes to the payload entries (`ws_selected` in bash, `WS_SCOPE_JQ` + `WS_JQ_ARGS` in jq) and `--project` to the payload files by slug (`project_selected`); `--workspace '*'` alone is not a narrowing, so the unchanged-file skip still applies; a scope that matches nothing is an error —, `--tfvars FILE` (`TFVARS`, absolute; also `-var-file` for terraform and exported to the enrich script; `clean --all` only removes the module's own file), `--skip-preflight`, `--no-secret-stubs`, `--upgrade` (with `init`: `tfvars_upgrade`, append the settings an older tfvars lacks; preflight warns about them until then), and the run configuration flags `--set KEY=VALUE`, `--cloud-connector ID`, `--vcs-connector ID`, `--runner-group NAME|shared`, `--workflow-group NAME` (one-run overlay on the tfvars, see `scope.sh`; connector kinds looked up in SG; with `--project` they are that project's `projectOverrides`, else the `SGDefault*` values). `import`/`all` end by writing `export/run-result.json` + `run-summary.md` (`write_run_result`, outcomes planned/blocked/success/failed) for CI. Resolves each tool via `sg_resolve` (PATH first — the image installs them — else `sg_ensure_*` cache). `apply`/`all` first run `require_tfc_auth`, which resolves the token the tfe provider will use (`TFE_TOKEN`, `TF_TOKEN_<host>`, or the `terraform login` file for `tfHostname`) and verifies it with `GET /api/v2/account/details`, failing fast on a missing or rejected (expired) credential. Runs `convert` and `import` in parallel (`--concurrency`, default 4), raises `terraform apply -parallelism`, and retries `sg-cli` imports with backoff (`sg_retry`, `SG_RETRIES`). Import resolves each project's workflow group with `group_for` (legacy `.sg/workflow-groups.json` entry → the payload's `CLIConfiguration.WorkflowGroup.name` via `payload_group` → `tfc-<seg>`) and runs `plan_groups`: STATUS `reuse` (exists) / `create` (missing; created after the confirmation) / `missing!` (`--no-create-groups`) / `moved!`; it collects `PLAN_PROBLEMS` — a project whose workflows still live in the group recorded in `state.json` (or the default group) when the target changed (SG cannot move workflows), and two projects sharing a group with overlapping names — prints them as ✗ lines, still shows the per-workflow plan (also under `--dry-run`) and then dies. The skip set (`state_import_done`) is computed before the plan so the ACTION column can say `skip`/`update`/`create`. Groups are never PATCHed. Imports go through `import_bulk`: it first lists the group (`sg_list_workflows`) and PATCHes every workflow that already exists (`sg_update_workflow`, `[updated]` marker, state re-uploaded, failures as `Failed to update <wf>: ...`), so the plan's `update` rows never hit a create; the remaining, new workflows with Terraform variables go via `sg-cli workflow create --bulk`, workflows **without** via a direct `POST .../wfs/` (`sg_create_workflow` + `sg_upload_tfstate` in `sg_api.sh`) because sg-cli drops an empty `iacInputData.data` and the API rejects the workflow — a workaround (`TODO(sg-cli)` markers) until sg-cli ships with sg-sdk-go >= v1.5.7, which sends the empty object; the direct path prints sg-cli-style `Failed to create <name>: <code>: <body>` lines so the parsing below covers both. State files are the migrator's job on both paths: `import_bulk` tracks sg-cli's per-workflow upload lines (`State file uploaded successfully` / `Failed to upload state file for <wf>`), re-uploads the failures via `upload_state` → `sg_upload_tfstate` (adds the `x-ms-blob-type: BlockBlob` header Azure Blob requires; sg-cli omits it and also treats anything but a literal `HTTP/1.1 200 OK` as failure — second `TODO(sg-cli)`), and writes `[state] uploaded|failed|none <wf>` markers into the parsed output; `do_import` turns them into `state_uploaded`/`state_failed` in the result, a workflow without its state fails the file (and `state_import_done` re-tries it next run). Existing workflows come back from sg-cli as `Failed to create <wf>: 409: Workflow ID not unique` (its update branch matches the outdated text `Workflow name not unique` — third `TODO(sg-cli)`); `import_bulk` collects those, PATCHes them via `sg_update_workflow`, re-uploads their state and drops the failure line, and `sg_create_workflow` does the same fallback on the direct path (prints `updated`). Before the parallel import, `probe_import` imports one workflow alone (first selected of the first file) and dies unless the create and the state upload both succeeded — the fail-fast for environment problems. sg-cli exits 0 even when individual workflows fail, so `do_import` parses its output: a workflow rejected as above SG's managed Terraform ceiling (1.5.7, last MPL/FOSS release) is re-imported with `SGDefaultTerraformVersion` (read from `terraform.tfvars`; an explicit `null` there means the `terraformVersion` key is deleted instead, so the org's execution preset decides — `tfvars_is_null` tells an explicit null from a missing key), the payload patched in place, and the case logged to `export/terraform-version-fallbacks.log` plus a printed notice; any other per-workflow failure fails the run. `cmd_import` reads the org's execution preset (`sg_execution_preset`, `GET /orgs/{org}/` → `Settings.workflowDefaults`) and `preset_labels` (report.sh) turns it into the `preset (1.5.7)` / `preset (private:rg)` cells of the plan and the `SG_TF_FALLBACK_LABEL` used in messages. The trigger pass (`do_set_triggers` → `sg_set_vcs_triggers`) skips workflows that do not exist in SG, keeps the sha of each posted `{VCSConfig, VCSTriggers}` body in `triggers.<seg>.sha` and does not re-POST an unchanged, already-set workflow (`--fresh` does; the endpoint upserts anyway, a 4xx "already exists" from older builds counts as success), and `sg_api_post` returns 22 on 4xx so `sg_retry` (via `SG_NO_RETRY_RC`) does not retry definitive errors. `clean` removes local artifacts (`export/`, TF state, tool cache); `clean --all` also removes config. `completion bash|zsh` prints a completion script (`cmd_completion`; commands + descriptions come from `SG_COMMAND_DESCS` (`SG_COMMANDS` is derived from it), options from `SG_OPTIONS`; keep them in sync with the parser and the host flags; the zsh script also works autoloaded from `$fpath` as `_sg-migrate.sh`); like `clean` it always runs natively. Output is concise by default — `apply` captures terraform's init/plan output and shows only progress + the final summary (or the full log on failure, `TF_IN_AUTOMATION=1`/`-no-color`); `-v`/`--verbose` (`SG_VERBOSE`) streams everything and un-gates the convert detail lines. Colored logging via `sg_step`/`sg_log`/`sg_success`/`sg_warn`/`sg_err`/`sg_row`; paths shown relative via `sg_rel` (all auto-off when not a TTY / `NO_COLOR`). Long steps show a live progress line (`sg_run_quiet` for terraform init/apply, `run_parallel <fn> <max> <label> <items>` for convert/import/triggers — animated only when `SG_ANIMATE=1`, i.e. colors are on; plain log lines otherwise) and report their duration (`sg_fmt_secs`). Phases start with `phase_begin` — inside `all` the header is numbered (`Phase 2/5: ...`, `PHASE_TOTAL` set in `main`) and `phase_took` gives the elapsed time; `run_parallel` prints a job's output as-is when it is a single line and adds the `── <file> ──` header only for longer or failed output (always with `-v`). The import confirmation goes through `sg_confirm` (so `SG_ANSWERS_FILE` works; a declined prompt is a neutral "cancelled" line, not an error) and `import`/`all` end with `finish_line` (outcome, total time, open checklist items from `CHECKLIST_OPEN`). A Ctrl-C is trapped to clear the progress line and print a resume hint. +- `scripts/migrate.sh` — the actual orchestrator (sources `scripts/lib/*.sh`, see below). Subcommands `init|preflight|apply|enrich|convert|validate|import|triggers|checklist|all|clean|completion` (no command prints the help menu; `enrich` runs in `all` unless `--no-variable-sets`). Flags beyond the basics: `--dry-run` (import, or all: the local export runs, then the plan is shown and nothing is created in SG), `--fresh` (ignore run state), the scope flags from `lib/scope.sh` — `--project NAME|SLUG`, `--workspace GLOB`, `--exclude-workspace GLOB`, `--tag NAME`, `--exclude-tag NAME`, all repeatable; tfvars holds the widest scope, include flags replace the tfvars list, exclude flags add to it; apply passes them as `-var` (`tfProjects`, `workspacenames`, `tfWorkspaceIgnoreNames`, `tfWorkspaceTags`, `tfWorkspaceIgnoreTags`), the later phases apply the same names/excludes to the payload entries (`ws_selected` in bash, `WS_SCOPE_JQ` + `WS_JQ_ARGS` in jq) and `--project` to the payload files by slug (`project_selected`); `--workspace '*'` alone is not a narrowing, so the unchanged-file skip still applies; a scope that matches nothing is an error —, `--tfvars FILE` (`TFVARS`, absolute; also `-var-file` for terraform and exported to the enrich script; `clean --all` only removes the module's own file), `--skip-preflight`, `--no-secret-stubs`, `--upgrade` (with `init`: `tfvars_upgrade`, append the settings an older tfvars lacks; preflight warns about them until then), and the run configuration flags `--set KEY=VALUE`, `--cloud-connector ID`, `--vcs-connector ID`, `--runner-group NAME|shared`, `--workflow-group NAME` (one-run overlay on the tfvars, see `scope.sh`; connector kinds looked up in SG; with `--project` they are that project's `projectOverrides`, else the `SGDefault*` values). `import`/`all` end by writing `export/run-result.json` + `run-summary.md` (`write_run_result`, outcomes planned/blocked/success/failed) for CI. Resolves each tool via `sg_resolve` (PATH first — the image installs them — else `sg_ensure_*` cache). `apply`/`all` first run `require_tfc_auth`, which resolves the token the tfe provider will use (`TFE_TOKEN`, `TF_TOKEN_<host>`, or the `terraform login` file for `tfHostname`) and verifies it with `GET /api/v2/account/details`, failing fast on a missing or rejected (expired) credential. Runs `convert` and `import` in parallel (`--concurrency`, default 4), raises `terraform apply -parallelism`, and retries `sg-cli` imports with backoff (`sg_retry`, `SG_RETRIES`). Import resolves each project's workflow group with `group_for` (legacy `.sg/workflow-groups.json` entry → the payload's `CLIConfiguration.WorkflowGroup.name` via `payload_group` → `tfc-<seg>`) and runs `plan_groups`: STATUS `reuse` (exists) / `create` (missing; created after the confirmation) / `missing!` (`--no-create-groups`) / `moved!`; it collects `PLAN_PROBLEMS` — a project whose workflows still live in the group recorded in `state.json` (or the default group) when the target changed (SG cannot move workflows), and two projects sharing a group with overlapping names — prints them as ✗ lines, still shows the per-workflow plan (also under `--dry-run`) and then dies. The skip set (`state_import_done`) is computed before the plan so the ACTION column can say `skip`/`update`/`create`. Groups are never PATCHed. Imports go through `import_bulk`: it first lists the group (`sg_list_workflows`) and PATCHes every workflow that already exists (`sg_update_workflow`, `[updated]` marker, state re-uploaded, failures as `Failed to update <wf>: ...`), so the plan's `update` rows never hit a create; the remaining, new workflows with Terraform variables go via `sg-cli workflow create --bulk`, workflows **without** via a direct `POST .../wfs/` (`sg_create_workflow` + `sg_upload_tfstate` in `sg_api.sh`) because sg-cli drops an empty `iacInputData.data` and the API rejects the workflow — a workaround (`TODO(sg-cli)` markers) until sg-cli ships with sg-sdk-go >= v1.5.7, which sends the empty object; the direct path prints sg-cli-style `Failed to create <name>: <code>: <body>` lines so the parsing below covers both. sg-cli's own output is shown only with `-v`; by default `import_bulk` renders one line per workflow from it — `wf_line` (`✓ <group>/<wf> created|updated[, state uploaded]`, a `!` line when the state upload failed) and `wf_failed` (`✗ <group>/<wf> not created|updated — <code>: <body>`, followed by `explain_api_error`'s hint) — while the raw lines still go to the parse file for `do_import`; `upload_state` prints nothing itself and leaves `US_RESULT`/`US_WHY` for `wf_line`; the `importing <file> -> <group>` and "N workflow(s) already exist / have no Terraform variables" lines are `-v` only. State files are the migrator's job on both paths: `import_bulk` tracks sg-cli's per-workflow upload lines (`State file uploaded successfully` / `Failed to upload state file for <wf>`), re-uploads the failures via `upload_state` → `sg_upload_tfstate` (adds the `x-ms-blob-type: BlockBlob` header Azure Blob requires; sg-cli omits it and also treats anything but a literal `HTTP/1.1 200 OK` as failure — second `TODO(sg-cli)`), and writes `[state] uploaded|failed|none <wf>` markers into the parsed output; `do_import` turns them into `state_uploaded`/`state_failed` in the result, a workflow without its state fails the file (and `state_import_done` re-tries it next run). Existing workflows come back from sg-cli as `Failed to create <wf>: 409: Workflow ID not unique` (its update branch matches the outdated text `Workflow name not unique` — third `TODO(sg-cli)`); `import_bulk` collects those, PATCHes them via `sg_update_workflow`, re-uploads their state and drops the failure line, and `sg_create_workflow` does the same fallback on the direct path (prints `updated`). Before the parallel import, `probe_import` imports one workflow alone (first selected of the first file) and dies unless the create and the state upload both succeeded — the fail-fast for environment problems. sg-cli exits 0 even when individual workflows fail, so `do_import` parses its output: a workflow rejected as above SG's managed Terraform ceiling (1.5.7, last MPL/FOSS release) is re-imported with `SGDefaultTerraformVersion` (read from `terraform.tfvars`; an explicit `null` there means the `terraformVersion` key is deleted instead, so the org's execution preset decides — `tfvars_is_null` tells an explicit null from a missing key), the payload patched in place, and the case logged to `export/terraform-version-fallbacks.log` plus a printed notice; any other per-workflow failure fails the run. `cmd_import` reads the org's execution preset (`sg_execution_preset`, `GET /orgs/{org}/` → `Settings.workflowDefaults`) and `preset_labels` (report.sh) turns it into the `preset (1.5.7)` / `preset (private:rg)` cells of the plan and the `SG_TF_FALLBACK_LABEL` used in messages. The trigger pass (`do_set_triggers` → `sg_set_vcs_triggers`) skips workflows that do not exist in SG, keeps the sha of each posted `{VCSConfig, VCSTriggers}` body in `triggers.<seg>.sha` and does not re-POST an unchanged, already-set workflow (`--fresh` does; the endpoint upserts anyway, a 4xx "already exists" from older builds counts as success), and `sg_api_post` returns 22 on 4xx so `sg_retry` (via `SG_NO_RETRY_RC`) does not retry definitive errors; `do_set_triggers` ends with one line per file that leaves out zero counts (`triggers set on N workflow(s), M unchanged, K without triggers, F failed, X not in SG`). `clean` removes local artifacts (`export/`, TF state, tool cache); `clean --all` also removes config. `completion bash|zsh` prints a completion script (`cmd_completion`; commands + descriptions come from `SG_COMMAND_DESCS` (`SG_COMMANDS` is derived from it), options from `SG_OPTIONS`; keep them in sync with the parser and the host flags; the zsh script also works autoloaded from `$fpath` as `_sg-migrate.sh`); like `clean` it always runs natively. Output is concise by default — `apply` captures terraform's init/plan output and shows only progress + the final summary (or the full log on failure, `TF_IN_AUTOMATION=1`/`-no-color`); `-v`/`--verbose` (`SG_VERBOSE`) streams everything and un-gates the convert detail lines. Colored logging via `sg_step`/`sg_log`/`sg_success`/`sg_warn`/`sg_err`/`sg_row`, plus the indented per-item status lines `sg_ok`/`sg_bad`/`sg_note` (✓/✗/!); paths shown relative via `sg_rel` (all auto-off when not a TTY / `NO_COLOR`). Long steps show a live progress line (`sg_run_quiet` for terraform init/apply, `run_parallel <fn> <max> <label> <items>` for convert/import/triggers — animated only when `SG_ANIMATE=1`, i.e. colors are on; plain log lines otherwise) and report their duration (`sg_fmt_secs`). Phases start with `phase_begin` — inside `all` the header is numbered (`Phase 2/5: ...`, `PHASE_TOTAL` set in `main`) and `phase_took` gives the elapsed time; `run_parallel` prints a job's output as-is when it is a single line and adds the `── <file> ──` header only for longer or failed output (always with `-v`). The import confirmation goes through `sg_confirm` (so `SG_ANSWERS_FILE` works; a declined prompt is a neutral "cancelled" line, not an error) and `import`/`all` end with `finish_line` (outcome, total time, open checklist items from `CHECKLIST_OPEN`). A Ctrl-C is trapped to clear the progress line and print a resume hint. - `scripts/lib/` — sourced libraries (a `lib/` gitignore rule is negated for this dir): `prompt.sh` (`sg_ask`/`sg_confirm`/`sg_select`; tty or `SG_ANSWERS_FILE` for tests; `SG_NONINTERACTIVE=1`/`-y`/no TTY → defaults), `tfvars.sh` (`tfvars_get` cached hcl2json reads — `tfvars_json` merges the run overlay (`TFVARS_OVERLAY_JSON`) over the file, and `tfvars_get`/`tfvars_get_json` keep a `false` value instead of treating it like a missing key —, `tfvars_has`/`tfvars_is_null` to tell an explicit `null` from a missing key, `tfvars_write` renders the wizard's `W_*` vars — `W_TF_SOURCE`, and `W_TF_VERSION`/`W_RUNNER_JSON` may be the literal `null`; plain strings go through `_tfvars_str`, so quotes/backslashes in names stay valid HCL; `projectOverrides`/`workspaceOverrides` are rendered from `W_PROJECT_OVERRIDES_JSON`/`W_WS_OVERRIDES_JSON` by `_tfvars_map` — `"key" = { attr = <json> }`, which hcl2json round-trips — so hand-written blocks survive a re-run of `init` (`wizard_run` reads them before writing; inner comments are lost); `tfWorkspaceIgnoreNames` is kept the same way (`W_IGNORE_NAMES_JSON`, never asked, shown in the review's scope row), and every other setting the wizard does not ask about (`tfProjects`, `cloudAuthVarPatterns`, hand-added keys) is appended verbatim from `W_PREV_JSON` under "Kept from the previous file"; `tfvars_missing_keys` (keys of `terraform.tfvars.example` the file lacks) and `tfvars_upgrade` (appends those paragraphs of the example — comment + default — under a dated header, `.bak` kept, touches nothing else; behind `init --upgrade`, prompt-free)), `scope.sh` (the run scope: `PROJECT_FILTER`/`WS_FILTER`/`WS_EXCLUDE`/`TAG_FILTER`/`TAG_EXCLUDE` arrays, `names_json`, `slug_of` (the transformer's project-slug rule in bash), `project_selected <seg>`, `ws_narrowed`/`ws_filter_json`/`ws_exclude_json` (tfvars `tfWorkspaceIgnoreNames` ∪ flags, memoized)/`ws_selected <name>` (bash `case` globs), `WS_SCOPE_JQ` + `ws_jq_args` → `WS_JQ_ARGS` for the jq sites, `scope_tags_json`/`scope_ignore_tags_json`, `scope_tfvar_args` → `SCOPE_TFVAR_ARGS` for apply, `scope_describe`, `scope_sha_input` for the apply phase hash; and the **run configuration overlay**: `--set KEY=VALUE` (`SET_VARS`, values via `_overlay_value`: HCL/JSON literals through hcl2json, bare text stays a string), `--cloud-connector`/`--vcs-connector`/`--runner-group`/`--workflow-group` → `overlay_build` looks the connector kinds up with `sg_list_integrations`/`sg_integration_type`/`sg_vcs_kind_of`, resolves `--project` values to raw TFC names (`_overlay_project_name`, fatal for an unknown project, as-given with a warning when TFC is unreachable) and fills `TFVARS_OVERLAY_JSON` — `projectOverrides[<name>]` fields with `--project`, `SGDefault*` keys without; `tfvars_json` deep-merges it over the file so every reader sees it, `scope_tfvar_args` passes each overlay key as `-var`, `scope_sha_input` includes it, `write_run_result` records it under `configuration`; `main` refuses the flags for `init`/`clean`/`completion` and a standalone `import` warns that they only reach the workflows through the export (`RAN_APPLY`)), `tfc_api.sh` (`tfc_http`/`tfc_get_all` paginated, `tfc_list_{orgs,projects,workspaces}` — workspaces include `vcs_provider`/`vcs_url`/`vcs_identifier` —, `tfc_select_workspaces` (the module's name-glob / include-tags(all) / exclude-tags / exclude-name-globs filter, shared by wizard and preflight), `tfc_vcs_summary`/`tfc_vcs_kind_for`/`tfc_vcs_label_for`, `tfc_token` with provider precedence, `require_tfc_auth`), `sg_api.sh` (`_sg_api` 0/22/1 contract + `SG_HTTP_CODE`, `_sg_listall` paginated listall reader (the API pages at 50) behind `sg_list_workflows`/`sg_list_wfgrps`, `sg_set_vcs_triggers`, `sg_list_integrations`, `sg_integration_type`, `sg_vcs_kind_of` (connector type → `sourceConfigDestKind`), `sg_execution_preset` + `sg_preset_{desc,runner_desc,version_desc,runner_provided}` (the org's execution preset and how to describe it), `sg_*_exists`, `sg_list_workflows`, `sg_create_secret`, `sg_patch_workflow`, `wfgroup_*`), `wizard.sh` (`wizard_run`: 5 steps TFC → SG → projects → defaults → review; `wizard_projects` (with >1 selected project, from `W_SEL_PROJECTS_JSON`) asks whether the global connectors and `tfc-<project>` groups apply to all, else per project the cloud connector, VCS connector (kind + prefix follow) and group (default / existing via `sg_list_wfgrps` / typed), writing `projectOverrides`; "same as the default" deletes the key, unmanaged fields are kept, previous picks come first (`_w_first`); the defaults step also asks `stripCloudAuthVars`; the TFC step first asks which TFC projects to migrate (with >1 project: `all` or `pick` by name/slug, resolved to raw names, unknown names re-asked; a previous non-empty `tfProjects` makes `pick` the first item; free text without discovery) → `W_PROJECTS_JSON`, rendered as `tfProjects` (`[]` = all) and applied to the workspace selection before the scope question; it keeps the workspace list and derives, via `tfc_vcs_summary`, which VCS provider the selected workspaces use and the repositories' URL prefix — the SG step lists matching connectors first and takes the repo URL prefix from TFC (never from a previous tfvars written for another provider kind), warns on a kind mismatch, reads the org's execution preset and offers it as the runner choice; the defaults step asks whether to carry TFC versions or use the preset (`SGTerraformVersionSource`) and, for carry, which fallback (`SGDefaultTerraformVersion` or the preset = null); the review shows human-readable rows plus the `tfc-<project>` groups; `_w_select_lines` feeds menus line-by-line so names with spaces/globs survive), `preflight.sh` (`preflight_run <apply|import|all>`, once per process; ✓ lines are plain language and name the connector's role/kind — related passing checks share one line (connector + kind + repo prefix, the project inside the selection line, version + runners when both come from the preset, one summary for the override keys) and routine ones are only counted (`pf_pass`), while ✗/! lines always print and name the tfvars field to fix; connector ids, runner groups and DPC kinds are collected from `SGDefault*`, `projectOverrides` and `workspaceOverrides`; besides existence it checks consistency — VCS kind vs. the connector's type, globally and per `projectOverrides` entry (fail), repo URL prefix vs. where the TFC repositories live or the kind's well-known host (warn), `projectOverrides` keys vs. the TFC project names (warn), `--project` values vs. the project names/slugs (fail; a `tfProjects` typo warns), and the workspace-selection preview applies the CLI scope (names, excludes, tags, projects) so its count is what the apply will export; the import context states the group policy and warns when a project's group changed since its last import; it also reports the org's execution preset next to the version/runner policy and warns when `carry` meets a preset that mounts a runner-provided binary), `state.sh` (`.sg/state.json`: `phases.<name>.input_sha`, `import.<seg>.{group,imported,updated,failed,tf_fallback,state_uploaded,state_failed}` (`group` anchors the never-move check; `updated` = PATCHed on a 409), `triggers.<seg>.{set,unchanged,failed,missing,sha}`, `secrets.<name>`; `run_phase` in migrate.sh skips unchanged phases), `report.sh` (`show_migration_summary` after apply — incl. a state-files row —, `show_import_plan` per-workflow table with columns sized to their content — it keeps the rows in `PLAN_ROWS_TSV` (name, group, action, version, runner, triggers, vars, secrets, segment, workspace, project) —, `write_run_result <outcome>` → `export/run-result.json` + `run-summary.md` from `PLAN_ROWS_TSV`, `PLAN_GROUP_ROWS` (plain copy of the group table, set by `plan_groups`), `PLAN_PROBLEMS`, the merged state and `CHECKLIST_OPEN`; per workflow `result` is planned/skipped/failed/updated/created, with the probe workflow — planned `create`, PATCHed on its second pass — reported as created), `errors.sh` (`explain_api_error` regex→hint table, `SG_API_HINTS`), `checklist.sh` (`create_secret_stubs` → SG secret `tfc-<wf>-<VAR>` = `CHANGE_ME`, referenced as `${secret::<name>}` via PATCH + payload patch; `write_checklist` → `export/post-import-checklist.md`, printed to the terminal as one ✓/! status line per section plus the imported groups with UI links — the file has the details; section 5 covers both the TFC state export and the SG state upload (`state_failed` from the import results); sets `CHECKLIST_OPEN`). Parallel jobs (`run_parallel`) run in subshells, so `do_import`/`do_set_triggers` write `.import-result.<seg>.json` / `.triggers-result.<seg>.json` into the export dir and the caller merges them into state. - `scripts/tools.sh` — sourced by the other scripts (sets `SG_REPO_ROOT` to its parent dir). Provides `sg_resolve` (PATH-or-cache), `sg_ensure_{jq,hcl2json,yajsv,sgcli}` (pinned downloads into `.sg/cached/`; `sg-cli` is the Go binary), `sg_retry`, the color vars + log helpers. The Docker image bundles these tools at build time (`yajsv` built from source so arm64 works), so in-container runs never download. - **Variable sets** — `scripts/enrich_variable_sets.sh <tfc-org> <payload...>` (run by `cmd_enrich`, which reads `tfOrg`/`tfHostname` from `terraform.tfvars` via `hcl2json` — variable sets live in the **TFC** org, not `SG_ORG`). Lists all sets + vars via the TFC API (Bearer token from the `terraform login` creds file or `TFE_TOKEN`), computes per-workspace effective vars (scope: global/project/workspace; precedence: priority sets > workspace vars > non-priority sets), and merges them into each workflow (matched by workspace name parsed from `TfStateFilePath`). Sensitive set vars and key conflicts are reported. Skipped by `--no-variable-sets`. diff --git a/scripts/migrate.sh b/scripts/migrate.sh index ef92504..24ac957 100755 --- a/scripts/migrate.sh +++ b/scripts/migrate.sh @@ -744,12 +744,16 @@ cmd_validate() { fi } -# sgcli_bulk <group> <file> <out> — run the bulk create, teeing output to <out>. -# sg-cli exits 0 even when individual workflows fail, so callers must inspect -# the output ("Failed to create <name>: ..." lines). +# sgcli_bulk <group> <file> <out> — run the bulk create, output to <out> (and +# to the terminal with -v; by default import_bulk renders one line per workflow +# from it instead). sg-cli exits 0 even when individual workflows fail, so +# callers must inspect the output ("Failed to create <name>: ..." lines). sgcli_bulk() { - "$SGCLI_BIN" workflow create --bulk --workflow-group "$1" --org "$ORG" "$2" 2>&1 | tee "$3" - return "${PIPESTATUS[0]}" + if [ "$VERBOSE" -eq 1 ]; then + "$SGCLI_BIN" workflow create --bulk --workflow-group "$1" --org "$ORG" "$2" 2>&1 | tee "$3" + return "${PIPESTATUS[0]}" + fi + "$SGCLI_BIN" workflow create --bulk --workflow-group "$1" --org "$ORG" "$2" >"$3" 2>&1 } # state_path_of <file> <wf> — the payload entry's TfStateFilePath ("" if none). @@ -757,30 +761,62 @@ state_path_of() { "$JQ_BIN" -r --arg n "$2" '.[] | select(.ResourceName == $n) | # upload_state <group> <wf> <file> <out> — upload the workflow's state file # ourselves and append a "[state] uploaded|failed|none <wf>[: why]" marker to -# <out> for do_import. Returns 1 only when the workflow has a state file that -# did not land. +# <out> for do_import. Prints nothing: US_RESULT (uploaded|none|failed) and +# US_WHY are left for wf_line. Returns 1 only when the workflow has a state +# file that did not land. +US_RESULT="" +US_WHY="" upload_state() { local grp="$1" name="$2" file="$3" out="$4" path why + US_WHY="" path="$(state_path_of "$file" "$name")" if [ -z "$path" ]; then + US_RESULT=none printf '[state] none %s\n' "$name" >>"$out" return 0 fi if [ ! -f "$path" ]; then - sg_warn " $name: state file missing: $(sg_rel "$path")" + US_RESULT=failed + US_WHY="state file missing: $(sg_rel "$path")" printf '[state] failed %s: state file missing (%s)\n' "$name" "$path" >>"$out" return 1 fi if why="$(sg_upload_tfstate "$grp" "$name" "$path")"; then - sg_log " $name: state file uploaded" + US_RESULT=uploaded printf '[state] uploaded %s\n' "$name" >>"$out" return 0 fi - sg_warn " $name: state file upload failed — $why" + US_RESULT=failed + US_WHY="$why" printf '[state] failed %s: %s\n' "$name" "$why" >>"$out" return 1 } +# wf_line <group> <wf> <created|updated> [uploaded|none|failed [why]] — the one +# status line a workflow gets in the import output ("✓ grp/wf created, state +# uploaded"); the state part defaults to upload_state's result. +wf_line() { + local grp="$1" name="$2" verb="$3" st="${4:-$US_RESULT}" why="${5:-$US_WHY}" + case "$st" in + uploaded) sg_ok "$grp/$name $verb, state uploaded" ;; + none) sg_ok "$grp/$name $verb" ;; + *) sg_note "$grp/$name $verb, state upload failed${why:+ — $why}" ;; + esac +} + +# wf_failed <group> <wf> <created|updated> <Failed to ... line> — the ✗ line for +# a rejected workflow, with the API body (the raw line goes to <out> for do_import). +wf_failed() { + local body + body="${4#*Failed to * "$2": }" + sg_bad "$1/$2 not $3 — $body" +} + +# Regex for the API's rejection of a Terraform version above SG's managed +# ceiling. SG bundles managed runtimes only up to the last MPL-licensed (FOSS) +# Terraform release; newer versions are BSL and are not shipped. +TF_CEILING_RE='Failed to create ([^:]+): 400: .*above the highest managed version \(([0-9.]+)\)' + # import_bulk <group> <file> <out> — import one payload file: sg-cli for the # workflows that have Terraform variables, a direct API POST for the ones # without — sg-cli drops an empty iacInputData.data and the API then rejects @@ -803,7 +839,7 @@ upload_state() { # not be made. import_bulk() { local grp="$1" file="$2" out="$3" rc=0 n_direct with_vars entry name err line cur="" cli_out pat known upd_names create_file - local -a redo=() exists=() updated=() + local -a redo=() exists=() updated=() seen=() st_ok=() st_none=() failed_lines=() : >"$out" # Workflows that already exist in the group (the plan's "update" rows) are @@ -815,17 +851,19 @@ import_bulk() { printf '%s' "$known" | "$JQ_BIN" -e 'type == "array"' >/dev/null 2>&1 || known='[]' upd_names="$("$JQ_BIN" -c --argjson ex "$known" '[.[].ResourceName | select(. as $n | $ex | index($n) != null)]' "$file")" if [ "$("$JQ_BIN" 'length' <<<"$upd_names")" -gt 0 ]; then - sg_log "$("$JQ_BIN" 'length' <<<"$upd_names") workflow(s) already exist in $grp — updating them" + [ "$VERBOSE" -eq 1 ] && sg_log "$("$JQ_BIN" 'length' <<<"$upd_names") workflow(s) already exist in $grp — updating them" while IFS= read -r name; do [ -n "$name" ] || continue entry="$("$JQ_BIN" -c --arg n "$name" 'first(.[] | select(.ResourceName == $n))' "$file")" if err="$(SG_NO_RETRY_RC=22 sg_retry "$RETRIES" "$RETRY_BASE" -- sg_update_workflow "$grp" "$entry" 2>/dev/null)"; then - sg_log " $name: updated" printf '[updated] %s\n' "$name" >>"$out" upload_state "$grp" "$name" "$file" "$out" || true + wf_line "$grp" "$name" updated else # Same shape as sg-cli's failure line (parsed by do_import). - printf 'Failed to update %s: %s\n' "$name" "$(tail -n1 <<<"$err")" | tee -a "$out" + line="Failed to update $name: $(tail -n1 <<<"$err")" + printf '%s\n' "$line" >>"$out" + wf_failed "$grp" "$name" updated "$line" fi done < <("$JQ_BIN" -r '.[]' <<<"$upd_names") create_file="$(mktemp)" @@ -844,32 +882,36 @@ import_bulk() { # sg-cli's own state upload, per workflow: trust its success, redo its # failures (it sends no x-ms-blob-type, and misreads anything but a # literal "HTTP/1.1 200 OK" as a failure). A 409 is an existing workflow. + # Everything else it printed is rendered as one line per workflow below. while IFS= read -r line; do case "$line" in - *"Processing workflow: "*) cur="${line##*Processing workflow: }" ;; + *"Processing workflow: "*) cur="${line##*Processing workflow: }"; seen+=("$cur") ;; *"Failed to create "*) cur="" name="${line##*Failed to create }" name="${name%%:*}" - case "$line" in *": 409: "* | *"not unique"*) exists+=("$name") ;; esac + case "$line" in + *": 409: "* | *"not unique"*) exists+=("$name") ;; + *) failed_lines+=("$line") ;; + esac ;; - *"State file uploaded successfully"*) [ -n "$cur" ] && printf '[state] uploaded %s\n' "$cur" >>"$out" ;; + *"State file uploaded successfully"*) [ -n "$cur" ] && { printf '[state] uploaded %s\n' "$cur" >>"$out"; st_ok+=("$cur"); } ;; *"Failed to upload state file for "*) name="${line##*Failed to upload state file for }"; redo+=("${name%%:*}") ;; *"cannot access state file"*) [ -n "$cur" ] && redo+=("$cur") ;; - *"TfStateFilePath not provided for "*) name="${line##*TfStateFilePath not provided for }"; printf '[state] none %s\n' "${name%%:*}" >>"$out" ;; + *"TfStateFilePath not provided for "*) name="${line##*TfStateFilePath not provided for }"; printf '[state] none %s\n' "${name%%:*}" >>"$out"; st_none+=("${name%%:*}") ;; esac done <"$cli_out" if [ "${#exists[@]}" -gt 0 ]; then - sg_log "${#exists[@]} workflow(s) already exist — updating them via the API (sg-cli's update path does not trigger on the 409)" + [ "$VERBOSE" -eq 1 ] && sg_log "${#exists[@]} workflow(s) already exist — updating them via the API (sg-cli's update path does not trigger on the 409)" for name in "${exists[@]}"; do entry="$("$JQ_BIN" -c --arg n "$name" 'first(.[] | select(.ResourceName == $n))' "$file")" if err="$(SG_NO_RETRY_RC=22 sg_retry "$RETRIES" "$RETRY_BASE" -- sg_update_workflow "$grp" "$entry" 2>/dev/null)"; then - sg_log " $name: updated" updated+=("$name") - redo+=("$name") printf '[updated] %s\n' "$name" >>"$out" + upload_state "$grp" "$name" "$file" "$out" || true + wf_line "$grp" "$name" updated else - sg_warn " $name: update failed — $(tail -n1 <<<"$err")" + sg_bad "$grp/$name not updated — $(tail -n1 <<<"$err")" fi done fi @@ -883,10 +925,31 @@ import_bulk() { cat "$cli_out" >>"$out" fi rm -f "$cli_out" - if [ "${#redo[@]}" -gt 0 ]; then - sg_log "uploading the state of ${#redo[@]} workflow(s) directly (sg-cli reported the upload failed, or did not attempt it)" - for name in "${redo[@]}"; do upload_state "$grp" "$name" "$file" "$out" || true; done - fi + # The created workflows, one line each: sg-cli's upload result, or our own + # upload when it failed or was not attempted. + for name in ${seen[@]+"${seen[@]}"}; do + case " ${exists[*]+"${exists[*]}"} " in *" $name "*) continue ;; esac + grep -q "Failed to create $name:" "$out" && continue # sg-cli prefixes its lines with " ✗ " + if [[ " ${redo[*]+"${redo[*]}"} " == *" $name "* ]]; then + [ "$VERBOSE" -eq 1 ] && sg_log "uploading the state of $name directly (sg-cli reported the upload failed, or did not attempt it)" + upload_state "$grp" "$name" "$file" "$out" || true + wf_line "$grp" "$name" created + elif [[ " ${st_ok[*]+"${st_ok[*]}"} " == *" $name "* ]]; then + wf_line "$grp" "$name" created uploaded + elif [[ " ${st_none[*]+"${st_none[*]}"} " == *" $name "* ]]; then + wf_line "$grp" "$name" created none + else + upload_state "$grp" "$name" "$file" "$out" || true + wf_line "$grp" "$name" created + fi + done + for line in ${failed_lines[@]+"${failed_lines[@]}"}; do + # A version above the managed ceiling is retried by do_import with the + # fallback version; its ✗ would only be noise here. + [[ "$line" =~ $TF_CEILING_RE ]] && continue + name="${line##*Failed to create }" + wf_failed "$grp" "${name%%:*}" created "$line" + done fi [ "$with_vars" != "$create_file" ] && rm -f "$with_vars" if [ "$n_direct" -eq 0 ]; then @@ -894,31 +957,29 @@ import_bulk() { return "$rc" fi - sg_log "$n_direct workflow(s) have no Terraform variables — creating them via the API directly (sg-cli drops an empty iacInputData.data)" + [ "$VERBOSE" -eq 1 ] && sg_log "$n_direct workflow(s) have no Terraform variables — creating them via the API directly (sg-cli drops an empty iacInputData.data)" while IFS= read -r entry; do name="$("$JQ_BIN" -r '.ResourceName' <<<"$entry")" if err="$(SG_NO_RETRY_RC=22 sg_retry "$RETRIES" "$RETRY_BASE" -- sg_create_workflow "$grp" "$entry" 2>/dev/null)"; then if [ "$(tail -n1 <<<"$err")" = "updated" ]; then - sg_log " $name: already existed — updated" printf '[updated] %s\n' "$name" >>"$out" + upload_state "$grp" "$name" "$file" "$out" || true + wf_line "$grp" "$name" updated else - sg_log " $name: created" + upload_state "$grp" "$name" "$file" "$out" || true + wf_line "$grp" "$name" created fi - upload_state "$grp" "$name" "$file" "$out" || true else # Same shape as sg-cli's failure line (parsed by do_import). - printf 'Failed to create %s: %s\n' "$name" "$(tail -n1 <<<"$err")" | tee -a "$out" + line="Failed to create $name: $(tail -n1 <<<"$err")" + printf '%s\n' "$line" >>"$out" + [[ "$line" =~ $TF_CEILING_RE ]] || wf_failed "$grp" "$name" created "$line" fi done < <("$JQ_BIN" -c '.[] | select((.VCSConfig.iacInputData.data // {}) | length == 0)' "$create_file") [ "$create_file" != "$file" ] && rm -f "$create_file" return "$rc" } -# Regex for the API's rejection of a Terraform version above SG's managed -# ceiling. SG bundles managed runtimes only up to the last MPL-licensed (FOSS) -# Terraform release; newer versions are BSL and are not shipped. -TF_CEILING_RE='Failed to create ([^:]+): 400: .*above the highest managed version \(([0-9.]+)\)' - # do_import <payload> — bulk-import one file. Workflows rejected because their # Terraform version is above the SG ceiling are re-imported with # SG_DEFAULT_TF_VERSION (the payload file is patched in place so re-runs and @@ -941,7 +1002,7 @@ do_import() { return 0 fi fi - sg_log "importing $(basename "$f") -> $grp" + [ "$VERBOSE" -eq 1 ] && sg_log "importing $(basename "$f") -> $grp" out="$(mktemp)" import_bulk "$grp" "$work" "$out" || rc=1 all_names="$("$JQ_BIN" -c '[.[].ResourceName]' "$work")" @@ -1047,11 +1108,7 @@ probe_import() { do_import "$probe" || true res="$EXPORT_DIR/.import-result.$seg.json" if [ -f "$res" ] && [ "$("$JQ_BIN" '(.failed | length) + (.state_failed | length)' "$res")" -eq 0 ]; then - if [ "$("$JQ_BIN" '.state_uploaded | length' "$res")" -gt 0 ]; then - sg_success "probe ok — $name is in SG with its state; importing the rest" - else - sg_success "probe ok — $name is in SG (no state file to upload); importing the rest" - fi + sg_success "probe ok — importing the rest" rm -rf "$dir" "$res" return 0 fi diff --git a/scripts/tools.sh b/scripts/tools.sh index 3100080..bab5d79 100755 --- a/scripts/tools.sh +++ b/scripts/tools.sh @@ -53,6 +53,11 @@ sg_err() { printf '%s[sg-migrate] ERROR%s %s\n' "$C_RED$C_BOLD" "$C_RESET" "$*" sg_success() { printf '%s[sg-migrate] ✓%s %s\n' "$C_GREEN$C_BOLD" "$C_RESET" "$*" >&2; } sg_step() { printf '\n%s==> %s%s\n' "$C_CYAN$C_BOLD" "$*" "$C_RESET" >&2; } sg_dim() { printf '%s %s%s\n' "$C_DIM" "$*" "$C_RESET" >&2; } +# sg_ok / sg_bad / sg_note — indented " ✓ ..." / " ✗ ..." / " ! ..." status +# lines (one per item: a workflow imported, a check passed). +sg_ok() { printf ' %s✓%s %s\n' "$C_GREEN" "$C_RESET" "$*" >&2; } +sg_bad() { printf ' %s✗%s %s\n' "$C_RED$C_BOLD" "$C_RESET" "$*" >&2; } +sg_note() { printf ' %s!%s %s\n' "$C_YELLOW" "$C_RESET" "$*" >&2; } # sg_row <label> <value> — an aligned " label value" line (review/summary tables). sg_row() { printf ' %s%-26s%s %s\n' "$C_BOLD" "$1" "$C_RESET" "$2" >&2; } From d2d6d4c2908c9a11cfd9cde0f14be68ffe12c727 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adis=20Halilovi=C4=87?= <adis.halilovic@stackguardian.io> Date: Mon, 14 Sep 2026 18:38:59 +0200 Subject: [PATCH 66/71] =?UTF-8?q?fix:=20checklist=20wording=20=E2=80=94=20?= =?UTF-8?q?preset=20version=20line=20reads=20plainly,=20state=20export=20a?= =?UTF-8?q?nd=20upload=20share=20one=20line=20when=20clean?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scripts/lib/checklist.sh | 19 ++++++++++++++----- scripts/lib/sg_api.sh | 5 +++-- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/scripts/lib/checklist.sh b/scripts/lib/checklist.sh index 431e7c7..3208530 100644 --- a/scripts/lib/checklist.sh +++ b/scripts/lib/checklist.sh @@ -191,16 +191,25 @@ write_checklist() { "secrets: set the real value of $n_secrets placeholder secret(s) (value CHANGE_ME)${i_unstubbed:+; $n_unstubbed sensitive var(s) have no stub}" _cl_status "$n_failed" "imports: none failed" "imports: $n_failed workflow(s) failed — fix and re-run '$PROG import'" if [ "$n_preset" -gt 0 ]; then - _cl_status "$n_preset" "" "Terraform version: all $n_preset workflow(s) take the execution preset's version (${preset_desc:-see the SG org settings}) instead of their TFC version — run a plan before relying on them" + _cl_status "$n_preset" "" "Terraform version: $n_preset workflow(s) run the org's execution preset (${preset_desc:-see the SG org settings}), not their TFC version — run a plan first" else _cl_status "$((n_fallback + n_unpinned))" "Terraform version: every workflow keeps its TFC version" \ "Terraform version: $((n_fallback + n_unpinned)) workflow(s) run ${SG_TF_FALLBACK_LABEL:-the fallback version} instead of what TFC used — run a plan before relying on them" fi _cl_status "$n_trig" "VCS triggers: registered for every workflow that had them" "VCS triggers: $n_trig workflow(s) without triggers — see the checklist, then '$PROG triggers'" - _cl_status "$((n_state + n_nonremote))" "state: exported for every selected workspace" \ - "state: $n_state workspace(s) without exported state${i_nonremote:+, $n_nonremote with non-remote execution} — upload by hand" - _cl_status "$n_stfail" "state: uploaded to SG for $n_stok workflow(s)" \ - "state: $n_stfail workflow(s) are in SG without their state (upload failed) — re-run '$PROG import' or upload by hand" + # Export and upload on one line when both are clean, one line each otherwise. + if [ "$((n_state + n_nonremote + n_stfail))" -eq 0 ]; then + if [ "$n_stok" -gt 0 ]; then + _cl_status 0 "state: exported and uploaded to SG for $n_stok workflow(s)" + else + _cl_status 0 "state: nothing to upload (no state files were exported)" + fi + else + _cl_status "$((n_state + n_nonremote))" "state: exported for every selected workspace" \ + "state: $n_state workspace(s) without exported state${i_nonremote:+, $n_nonremote with non-remote execution} — upload by hand" + _cl_status "$n_stfail" "state: uploaded to SG for $n_stok workflow(s)" \ + "state: $n_stfail workflow(s) are in SG without their state (upload failed) — re-run '$PROG import' or upload by hand" + fi [ "$n_renamed" -gt 0 ] && _cl_status "$n_renamed" "" "names: $n_renamed workflow(s) were renamed to valid SG names" # shellcheck disable=SC2034 CHECKLIST_OPEN=$((n_secrets + n_unstubbed + n_failed + n_fallback + n_unpinned + n_preset + n_trig + n_state + n_stfail + n_nonremote)) diff --git a/scripts/lib/sg_api.sh b/scripts/lib/sg_api.sh index b306656..c4330aa 100644 --- a/scripts/lib/sg_api.sh +++ b/scripts/lib/sg_api.sh @@ -259,10 +259,11 @@ sg_preset_version_desc() { } # sg_preset_desc <preset-json> — one line: "Terraform 1.5.7 on shared runners"; -# "none configured (platform defaults: managed Terraform 1.5.7 on shared runners)" for {}. +# "platform default: managed Terraform 1.5.7 on shared runners" for {} (no +# nested parentheses, the callers wrap it in their own). sg_preset_desc() { if [ -z "$1" ] || [ "$1" = "{}" ] || [ "$1" = "null" ]; then - printf 'none configured (platform defaults: managed Terraform 1.5.7 on shared runners)' + printf 'platform default: managed Terraform 1.5.7 on shared runners' else printf '%s on %s' "$(sg_preset_version_desc "$1")" "$(sg_preset_runner_desc "$1")" fi From a7f77f5fc8b04a2473f8efa87c6de29c618128e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adis=20Halilovi=C4=87?= <adis.halilovic@stackguardian.io> Date: Mon, 14 Sep 2026 18:39:00 +0200 Subject: [PATCH 67/71] fix: migration summary shows one 'Version & runners' row when both are left to the execution preset --- scripts/lib/report.sh | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/scripts/lib/report.sh b/scripts/lib/report.sh index ac50ccf..df41576 100644 --- a/scripts/lib/report.sh +++ b/scripts/lib/report.sh @@ -31,14 +31,20 @@ show_migration_summary() { local src def src="$("$jqb" -r '.terraformVersionSource // "carry"' "$f")" def="$("$jqb" -r '.terraformVersionDefault // empty' "$f")" - if [ "$src" = "preset" ]; then - sg_row "Terraform version" "none sent; the org's execution preset applies at import" - elif [ -z "$def" ]; then - sg_row "Terraform version" "carried from TFC; unpinned or rejected pins go to the execution preset" + local runners_preset=0 + [ "$("$jqb" -r '.runnerConstraintsSource // "config"' "$f")" = "preset" ] && runners_preset=1 + if [ "$src" = "preset" ] && [ "$runners_preset" -eq 1 ]; then + sg_row "Version & runners" "left to the org's execution preset (applied at import)" else - sg_row "Terraform version" "carried from TFC; fallback ${def#TERRAFORM-} for unpinned or rejected pins" + if [ "$src" = "preset" ]; then + sg_row "Terraform version" "none sent; the org's execution preset applies at import" + elif [ -z "$def" ]; then + sg_row "Terraform version" "carried from TFC; unpinned or rejected pins go to the execution preset" + else + sg_row "Terraform version" "carried from TFC; fallback ${def#TERRAFORM-} for unpinned or rejected pins" + fi + [ "$runners_preset" -eq 1 ] && sg_row "Runners" "none sent; the org's execution preset applies at import" fi - [ "$("$jqb" -r '.runnerConstraintsSource // "config"' "$f")" = "preset" ] && sg_row "Runners" "none sent; the org's execution preset applies at import" _summary_section "$f" '.skippedSensitiveVars' "Sensitive variables skipped" \ 'to_entries[] | "\(.key): \(.value | join(", "))"' "TFC never exposes their values; they become placeholder SG secrets after import" From 7136862cfa522d28387f08c41e637398e2ea7dd3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adis=20Halilovi=C4=87?= <adis.halilovic@stackguardian.io> Date: Mon, 14 Sep 2026 18:39:00 +0200 Subject: [PATCH 68/71] fix: the import plan legend only explains what the table shows --- CLAUDE.md | 2 +- scripts/lib/report.sh | 30 +++++++++++++++++++++++------- 2 files changed, 24 insertions(+), 8 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 18eed4d..0aac0b2 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -25,7 +25,7 @@ The five phases are wrapped by an orchestrator so users don't run them by hand: - `sg-migrate.sh` (host entrypoint, repo root) — runs `scripts/migrate.sh` **inside the Docker image** (`Dockerfile`), bind-mounting the repo at `/app` and forwarding `SG_API_TOKEN`/`SG_ORG`/`SG_BASE_URL`/`TFE_TOKEN`/`TF_TOKEN_*`. `--tfvars FILE` (or `SG_TFVARS`) is a host path: the wrapper strips the flag, resolves it and passes it on as `SG_TFVARS` — rewritten to `/app/<relative>` when the file is inside the checkout, otherwise mounted read-only at `/tmp/sg-run.tfvars`. For TFC auth it prefers a long-lived `TFE_TOKEN` (env); otherwise it mounts `~/.terraform.d/credentials.tfrc.json` read-only. Runs natively instead when `--native`/`--local` is passed, `SG_NATIVE=1` is set, the command is `clean`/`completion`/help (or no command is given), or Docker is absent. Exports `SG_PROG` so `migrate.sh` shows `./sg-migrate.sh` in its usage/hints. `update` is host-only too (`cmd_update` in the wrapper, never passed to `migrate.sh`): refuses a non-git checkout, a detached HEAD or dirty tracked files, then `git pull --ff-only` and rebuilds the image only when the `Dockerfile` changed between the old and new HEAD — this is how customers pick up fixes from a plain clone (untracked config/output survive). `migrate.sh` lists `update` in its usage/`SG_COMMANDS` and rejects it with a host-only hint. `.gitattributes` forces LF so a Git-for-Windows clone does not CRLF the scripts. - `scripts/migrate.sh` — the actual orchestrator (sources `scripts/lib/*.sh`, see below). Subcommands `init|preflight|apply|enrich|convert|validate|import|triggers|checklist|all|clean|completion` (no command prints the help menu; `enrich` runs in `all` unless `--no-variable-sets`). Flags beyond the basics: `--dry-run` (import, or all: the local export runs, then the plan is shown and nothing is created in SG), `--fresh` (ignore run state), the scope flags from `lib/scope.sh` — `--project NAME|SLUG`, `--workspace GLOB`, `--exclude-workspace GLOB`, `--tag NAME`, `--exclude-tag NAME`, all repeatable; tfvars holds the widest scope, include flags replace the tfvars list, exclude flags add to it; apply passes them as `-var` (`tfProjects`, `workspacenames`, `tfWorkspaceIgnoreNames`, `tfWorkspaceTags`, `tfWorkspaceIgnoreTags`), the later phases apply the same names/excludes to the payload entries (`ws_selected` in bash, `WS_SCOPE_JQ` + `WS_JQ_ARGS` in jq) and `--project` to the payload files by slug (`project_selected`); `--workspace '*'` alone is not a narrowing, so the unchanged-file skip still applies; a scope that matches nothing is an error —, `--tfvars FILE` (`TFVARS`, absolute; also `-var-file` for terraform and exported to the enrich script; `clean --all` only removes the module's own file), `--skip-preflight`, `--no-secret-stubs`, `--upgrade` (with `init`: `tfvars_upgrade`, append the settings an older tfvars lacks; preflight warns about them until then), and the run configuration flags `--set KEY=VALUE`, `--cloud-connector ID`, `--vcs-connector ID`, `--runner-group NAME|shared`, `--workflow-group NAME` (one-run overlay on the tfvars, see `scope.sh`; connector kinds looked up in SG; with `--project` they are that project's `projectOverrides`, else the `SGDefault*` values). `import`/`all` end by writing `export/run-result.json` + `run-summary.md` (`write_run_result`, outcomes planned/blocked/success/failed) for CI. Resolves each tool via `sg_resolve` (PATH first — the image installs them — else `sg_ensure_*` cache). `apply`/`all` first run `require_tfc_auth`, which resolves the token the tfe provider will use (`TFE_TOKEN`, `TF_TOKEN_<host>`, or the `terraform login` file for `tfHostname`) and verifies it with `GET /api/v2/account/details`, failing fast on a missing or rejected (expired) credential. Runs `convert` and `import` in parallel (`--concurrency`, default 4), raises `terraform apply -parallelism`, and retries `sg-cli` imports with backoff (`sg_retry`, `SG_RETRIES`). Import resolves each project's workflow group with `group_for` (legacy `.sg/workflow-groups.json` entry → the payload's `CLIConfiguration.WorkflowGroup.name` via `payload_group` → `tfc-<seg>`) and runs `plan_groups`: STATUS `reuse` (exists) / `create` (missing; created after the confirmation) / `missing!` (`--no-create-groups`) / `moved!`; it collects `PLAN_PROBLEMS` — a project whose workflows still live in the group recorded in `state.json` (or the default group) when the target changed (SG cannot move workflows), and two projects sharing a group with overlapping names — prints them as ✗ lines, still shows the per-workflow plan (also under `--dry-run`) and then dies. The skip set (`state_import_done`) is computed before the plan so the ACTION column can say `skip`/`update`/`create`. Groups are never PATCHed. Imports go through `import_bulk`: it first lists the group (`sg_list_workflows`) and PATCHes every workflow that already exists (`sg_update_workflow`, `[updated]` marker, state re-uploaded, failures as `Failed to update <wf>: ...`), so the plan's `update` rows never hit a create; the remaining, new workflows with Terraform variables go via `sg-cli workflow create --bulk`, workflows **without** via a direct `POST .../wfs/` (`sg_create_workflow` + `sg_upload_tfstate` in `sg_api.sh`) because sg-cli drops an empty `iacInputData.data` and the API rejects the workflow — a workaround (`TODO(sg-cli)` markers) until sg-cli ships with sg-sdk-go >= v1.5.7, which sends the empty object; the direct path prints sg-cli-style `Failed to create <name>: <code>: <body>` lines so the parsing below covers both. sg-cli's own output is shown only with `-v`; by default `import_bulk` renders one line per workflow from it — `wf_line` (`✓ <group>/<wf> created|updated[, state uploaded]`, a `!` line when the state upload failed) and `wf_failed` (`✗ <group>/<wf> not created|updated — <code>: <body>`, followed by `explain_api_error`'s hint) — while the raw lines still go to the parse file for `do_import`; `upload_state` prints nothing itself and leaves `US_RESULT`/`US_WHY` for `wf_line`; the `importing <file> -> <group>` and "N workflow(s) already exist / have no Terraform variables" lines are `-v` only. State files are the migrator's job on both paths: `import_bulk` tracks sg-cli's per-workflow upload lines (`State file uploaded successfully` / `Failed to upload state file for <wf>`), re-uploads the failures via `upload_state` → `sg_upload_tfstate` (adds the `x-ms-blob-type: BlockBlob` header Azure Blob requires; sg-cli omits it and also treats anything but a literal `HTTP/1.1 200 OK` as failure — second `TODO(sg-cli)`), and writes `[state] uploaded|failed|none <wf>` markers into the parsed output; `do_import` turns them into `state_uploaded`/`state_failed` in the result, a workflow without its state fails the file (and `state_import_done` re-tries it next run). Existing workflows come back from sg-cli as `Failed to create <wf>: 409: Workflow ID not unique` (its update branch matches the outdated text `Workflow name not unique` — third `TODO(sg-cli)`); `import_bulk` collects those, PATCHes them via `sg_update_workflow`, re-uploads their state and drops the failure line, and `sg_create_workflow` does the same fallback on the direct path (prints `updated`). Before the parallel import, `probe_import` imports one workflow alone (first selected of the first file) and dies unless the create and the state upload both succeeded — the fail-fast for environment problems. sg-cli exits 0 even when individual workflows fail, so `do_import` parses its output: a workflow rejected as above SG's managed Terraform ceiling (1.5.7, last MPL/FOSS release) is re-imported with `SGDefaultTerraformVersion` (read from `terraform.tfvars`; an explicit `null` there means the `terraformVersion` key is deleted instead, so the org's execution preset decides — `tfvars_is_null` tells an explicit null from a missing key), the payload patched in place, and the case logged to `export/terraform-version-fallbacks.log` plus a printed notice; any other per-workflow failure fails the run. `cmd_import` reads the org's execution preset (`sg_execution_preset`, `GET /orgs/{org}/` → `Settings.workflowDefaults`) and `preset_labels` (report.sh) turns it into the `preset (1.5.7)` / `preset (private:rg)` cells of the plan and the `SG_TF_FALLBACK_LABEL` used in messages. The trigger pass (`do_set_triggers` → `sg_set_vcs_triggers`) skips workflows that do not exist in SG, keeps the sha of each posted `{VCSConfig, VCSTriggers}` body in `triggers.<seg>.sha` and does not re-POST an unchanged, already-set workflow (`--fresh` does; the endpoint upserts anyway, a 4xx "already exists" from older builds counts as success), and `sg_api_post` returns 22 on 4xx so `sg_retry` (via `SG_NO_RETRY_RC`) does not retry definitive errors; `do_set_triggers` ends with one line per file that leaves out zero counts (`triggers set on N workflow(s), M unchanged, K without triggers, F failed, X not in SG`). `clean` removes local artifacts (`export/`, TF state, tool cache); `clean --all` also removes config. `completion bash|zsh` prints a completion script (`cmd_completion`; commands + descriptions come from `SG_COMMAND_DESCS` (`SG_COMMANDS` is derived from it), options from `SG_OPTIONS`; keep them in sync with the parser and the host flags; the zsh script also works autoloaded from `$fpath` as `_sg-migrate.sh`); like `clean` it always runs natively. Output is concise by default — `apply` captures terraform's init/plan output and shows only progress + the final summary (or the full log on failure, `TF_IN_AUTOMATION=1`/`-no-color`); `-v`/`--verbose` (`SG_VERBOSE`) streams everything and un-gates the convert detail lines. Colored logging via `sg_step`/`sg_log`/`sg_success`/`sg_warn`/`sg_err`/`sg_row`, plus the indented per-item status lines `sg_ok`/`sg_bad`/`sg_note` (✓/✗/!); paths shown relative via `sg_rel` (all auto-off when not a TTY / `NO_COLOR`). Long steps show a live progress line (`sg_run_quiet` for terraform init/apply, `run_parallel <fn> <max> <label> <items>` for convert/import/triggers — animated only when `SG_ANIMATE=1`, i.e. colors are on; plain log lines otherwise) and report their duration (`sg_fmt_secs`). Phases start with `phase_begin` — inside `all` the header is numbered (`Phase 2/5: ...`, `PHASE_TOTAL` set in `main`) and `phase_took` gives the elapsed time; `run_parallel` prints a job's output as-is when it is a single line and adds the `── <file> ──` header only for longer or failed output (always with `-v`). The import confirmation goes through `sg_confirm` (so `SG_ANSWERS_FILE` works; a declined prompt is a neutral "cancelled" line, not an error) and `import`/`all` end with `finish_line` (outcome, total time, open checklist items from `CHECKLIST_OPEN`). A Ctrl-C is trapped to clear the progress line and print a resume hint. -- `scripts/lib/` — sourced libraries (a `lib/` gitignore rule is negated for this dir): `prompt.sh` (`sg_ask`/`sg_confirm`/`sg_select`; tty or `SG_ANSWERS_FILE` for tests; `SG_NONINTERACTIVE=1`/`-y`/no TTY → defaults), `tfvars.sh` (`tfvars_get` cached hcl2json reads — `tfvars_json` merges the run overlay (`TFVARS_OVERLAY_JSON`) over the file, and `tfvars_get`/`tfvars_get_json` keep a `false` value instead of treating it like a missing key —, `tfvars_has`/`tfvars_is_null` to tell an explicit `null` from a missing key, `tfvars_write` renders the wizard's `W_*` vars — `W_TF_SOURCE`, and `W_TF_VERSION`/`W_RUNNER_JSON` may be the literal `null`; plain strings go through `_tfvars_str`, so quotes/backslashes in names stay valid HCL; `projectOverrides`/`workspaceOverrides` are rendered from `W_PROJECT_OVERRIDES_JSON`/`W_WS_OVERRIDES_JSON` by `_tfvars_map` — `"key" = { attr = <json> }`, which hcl2json round-trips — so hand-written blocks survive a re-run of `init` (`wizard_run` reads them before writing; inner comments are lost); `tfWorkspaceIgnoreNames` is kept the same way (`W_IGNORE_NAMES_JSON`, never asked, shown in the review's scope row), and every other setting the wizard does not ask about (`tfProjects`, `cloudAuthVarPatterns`, hand-added keys) is appended verbatim from `W_PREV_JSON` under "Kept from the previous file"; `tfvars_missing_keys` (keys of `terraform.tfvars.example` the file lacks) and `tfvars_upgrade` (appends those paragraphs of the example — comment + default — under a dated header, `.bak` kept, touches nothing else; behind `init --upgrade`, prompt-free)), `scope.sh` (the run scope: `PROJECT_FILTER`/`WS_FILTER`/`WS_EXCLUDE`/`TAG_FILTER`/`TAG_EXCLUDE` arrays, `names_json`, `slug_of` (the transformer's project-slug rule in bash), `project_selected <seg>`, `ws_narrowed`/`ws_filter_json`/`ws_exclude_json` (tfvars `tfWorkspaceIgnoreNames` ∪ flags, memoized)/`ws_selected <name>` (bash `case` globs), `WS_SCOPE_JQ` + `ws_jq_args` → `WS_JQ_ARGS` for the jq sites, `scope_tags_json`/`scope_ignore_tags_json`, `scope_tfvar_args` → `SCOPE_TFVAR_ARGS` for apply, `scope_describe`, `scope_sha_input` for the apply phase hash; and the **run configuration overlay**: `--set KEY=VALUE` (`SET_VARS`, values via `_overlay_value`: HCL/JSON literals through hcl2json, bare text stays a string), `--cloud-connector`/`--vcs-connector`/`--runner-group`/`--workflow-group` → `overlay_build` looks the connector kinds up with `sg_list_integrations`/`sg_integration_type`/`sg_vcs_kind_of`, resolves `--project` values to raw TFC names (`_overlay_project_name`, fatal for an unknown project, as-given with a warning when TFC is unreachable) and fills `TFVARS_OVERLAY_JSON` — `projectOverrides[<name>]` fields with `--project`, `SGDefault*` keys without; `tfvars_json` deep-merges it over the file so every reader sees it, `scope_tfvar_args` passes each overlay key as `-var`, `scope_sha_input` includes it, `write_run_result` records it under `configuration`; `main` refuses the flags for `init`/`clean`/`completion` and a standalone `import` warns that they only reach the workflows through the export (`RAN_APPLY`)), `tfc_api.sh` (`tfc_http`/`tfc_get_all` paginated, `tfc_list_{orgs,projects,workspaces}` — workspaces include `vcs_provider`/`vcs_url`/`vcs_identifier` —, `tfc_select_workspaces` (the module's name-glob / include-tags(all) / exclude-tags / exclude-name-globs filter, shared by wizard and preflight), `tfc_vcs_summary`/`tfc_vcs_kind_for`/`tfc_vcs_label_for`, `tfc_token` with provider precedence, `require_tfc_auth`), `sg_api.sh` (`_sg_api` 0/22/1 contract + `SG_HTTP_CODE`, `_sg_listall` paginated listall reader (the API pages at 50) behind `sg_list_workflows`/`sg_list_wfgrps`, `sg_set_vcs_triggers`, `sg_list_integrations`, `sg_integration_type`, `sg_vcs_kind_of` (connector type → `sourceConfigDestKind`), `sg_execution_preset` + `sg_preset_{desc,runner_desc,version_desc,runner_provided}` (the org's execution preset and how to describe it), `sg_*_exists`, `sg_list_workflows`, `sg_create_secret`, `sg_patch_workflow`, `wfgroup_*`), `wizard.sh` (`wizard_run`: 5 steps TFC → SG → projects → defaults → review; `wizard_projects` (with >1 selected project, from `W_SEL_PROJECTS_JSON`) asks whether the global connectors and `tfc-<project>` groups apply to all, else per project the cloud connector, VCS connector (kind + prefix follow) and group (default / existing via `sg_list_wfgrps` / typed), writing `projectOverrides`; "same as the default" deletes the key, unmanaged fields are kept, previous picks come first (`_w_first`); the defaults step also asks `stripCloudAuthVars`; the TFC step first asks which TFC projects to migrate (with >1 project: `all` or `pick` by name/slug, resolved to raw names, unknown names re-asked; a previous non-empty `tfProjects` makes `pick` the first item; free text without discovery) → `W_PROJECTS_JSON`, rendered as `tfProjects` (`[]` = all) and applied to the workspace selection before the scope question; it keeps the workspace list and derives, via `tfc_vcs_summary`, which VCS provider the selected workspaces use and the repositories' URL prefix — the SG step lists matching connectors first and takes the repo URL prefix from TFC (never from a previous tfvars written for another provider kind), warns on a kind mismatch, reads the org's execution preset and offers it as the runner choice; the defaults step asks whether to carry TFC versions or use the preset (`SGTerraformVersionSource`) and, for carry, which fallback (`SGDefaultTerraformVersion` or the preset = null); the review shows human-readable rows plus the `tfc-<project>` groups; `_w_select_lines` feeds menus line-by-line so names with spaces/globs survive), `preflight.sh` (`preflight_run <apply|import|all>`, once per process; ✓ lines are plain language and name the connector's role/kind — related passing checks share one line (connector + kind + repo prefix, the project inside the selection line, version + runners when both come from the preset, one summary for the override keys) and routine ones are only counted (`pf_pass`), while ✗/! lines always print and name the tfvars field to fix; connector ids, runner groups and DPC kinds are collected from `SGDefault*`, `projectOverrides` and `workspaceOverrides`; besides existence it checks consistency — VCS kind vs. the connector's type, globally and per `projectOverrides` entry (fail), repo URL prefix vs. where the TFC repositories live or the kind's well-known host (warn), `projectOverrides` keys vs. the TFC project names (warn), `--project` values vs. the project names/slugs (fail; a `tfProjects` typo warns), and the workspace-selection preview applies the CLI scope (names, excludes, tags, projects) so its count is what the apply will export; the import context states the group policy and warns when a project's group changed since its last import; it also reports the org's execution preset next to the version/runner policy and warns when `carry` meets a preset that mounts a runner-provided binary), `state.sh` (`.sg/state.json`: `phases.<name>.input_sha`, `import.<seg>.{group,imported,updated,failed,tf_fallback,state_uploaded,state_failed}` (`group` anchors the never-move check; `updated` = PATCHed on a 409), `triggers.<seg>.{set,unchanged,failed,missing,sha}`, `secrets.<name>`; `run_phase` in migrate.sh skips unchanged phases), `report.sh` (`show_migration_summary` after apply — incl. a state-files row —, `show_import_plan` per-workflow table with columns sized to their content — it keeps the rows in `PLAN_ROWS_TSV` (name, group, action, version, runner, triggers, vars, secrets, segment, workspace, project) —, `write_run_result <outcome>` → `export/run-result.json` + `run-summary.md` from `PLAN_ROWS_TSV`, `PLAN_GROUP_ROWS` (plain copy of the group table, set by `plan_groups`), `PLAN_PROBLEMS`, the merged state and `CHECKLIST_OPEN`; per workflow `result` is planned/skipped/failed/updated/created, with the probe workflow — planned `create`, PATCHed on its second pass — reported as created), `errors.sh` (`explain_api_error` regex→hint table, `SG_API_HINTS`), `checklist.sh` (`create_secret_stubs` → SG secret `tfc-<wf>-<VAR>` = `CHANGE_ME`, referenced as `${secret::<name>}` via PATCH + payload patch; `write_checklist` → `export/post-import-checklist.md`, printed to the terminal as one ✓/! status line per section plus the imported groups with UI links — the file has the details; section 5 covers both the TFC state export and the SG state upload (`state_failed` from the import results); sets `CHECKLIST_OPEN`). Parallel jobs (`run_parallel`) run in subshells, so `do_import`/`do_set_triggers` write `.import-result.<seg>.json` / `.triggers-result.<seg>.json` into the export dir and the caller merges them into state. +- `scripts/lib/` — sourced libraries (a `lib/` gitignore rule is negated for this dir): `prompt.sh` (`sg_ask`/`sg_confirm`/`sg_select`; tty or `SG_ANSWERS_FILE` for tests; `SG_NONINTERACTIVE=1`/`-y`/no TTY → defaults), `tfvars.sh` (`tfvars_get` cached hcl2json reads — `tfvars_json` merges the run overlay (`TFVARS_OVERLAY_JSON`) over the file, and `tfvars_get`/`tfvars_get_json` keep a `false` value instead of treating it like a missing key —, `tfvars_has`/`tfvars_is_null` to tell an explicit `null` from a missing key, `tfvars_write` renders the wizard's `W_*` vars — `W_TF_SOURCE`, and `W_TF_VERSION`/`W_RUNNER_JSON` may be the literal `null`; plain strings go through `_tfvars_str`, so quotes/backslashes in names stay valid HCL; `projectOverrides`/`workspaceOverrides` are rendered from `W_PROJECT_OVERRIDES_JSON`/`W_WS_OVERRIDES_JSON` by `_tfvars_map` — `"key" = { attr = <json> }`, which hcl2json round-trips — so hand-written blocks survive a re-run of `init` (`wizard_run` reads them before writing; inner comments are lost); `tfWorkspaceIgnoreNames` is kept the same way (`W_IGNORE_NAMES_JSON`, never asked, shown in the review's scope row), and every other setting the wizard does not ask about (`tfProjects`, `cloudAuthVarPatterns`, hand-added keys) is appended verbatim from `W_PREV_JSON` under "Kept from the previous file"; `tfvars_missing_keys` (keys of `terraform.tfvars.example` the file lacks) and `tfvars_upgrade` (appends those paragraphs of the example — comment + default — under a dated header, `.bak` kept, touches nothing else; behind `init --upgrade`, prompt-free)), `scope.sh` (the run scope: `PROJECT_FILTER`/`WS_FILTER`/`WS_EXCLUDE`/`TAG_FILTER`/`TAG_EXCLUDE` arrays, `names_json`, `slug_of` (the transformer's project-slug rule in bash), `project_selected <seg>`, `ws_narrowed`/`ws_filter_json`/`ws_exclude_json` (tfvars `tfWorkspaceIgnoreNames` ∪ flags, memoized)/`ws_selected <name>` (bash `case` globs), `WS_SCOPE_JQ` + `ws_jq_args` → `WS_JQ_ARGS` for the jq sites, `scope_tags_json`/`scope_ignore_tags_json`, `scope_tfvar_args` → `SCOPE_TFVAR_ARGS` for apply, `scope_describe`, `scope_sha_input` for the apply phase hash; and the **run configuration overlay**: `--set KEY=VALUE` (`SET_VARS`, values via `_overlay_value`: HCL/JSON literals through hcl2json, bare text stays a string), `--cloud-connector`/`--vcs-connector`/`--runner-group`/`--workflow-group` → `overlay_build` looks the connector kinds up with `sg_list_integrations`/`sg_integration_type`/`sg_vcs_kind_of`, resolves `--project` values to raw TFC names (`_overlay_project_name`, fatal for an unknown project, as-given with a warning when TFC is unreachable) and fills `TFVARS_OVERLAY_JSON` — `projectOverrides[<name>]` fields with `--project`, `SGDefault*` keys without; `tfvars_json` deep-merges it over the file so every reader sees it, `scope_tfvar_args` passes each overlay key as `-var`, `scope_sha_input` includes it, `write_run_result` records it under `configuration`; `main` refuses the flags for `init`/`clean`/`completion` and a standalone `import` warns that they only reach the workflows through the export (`RAN_APPLY`)), `tfc_api.sh` (`tfc_http`/`tfc_get_all` paginated, `tfc_list_{orgs,projects,workspaces}` — workspaces include `vcs_provider`/`vcs_url`/`vcs_identifier` —, `tfc_select_workspaces` (the module's name-glob / include-tags(all) / exclude-tags / exclude-name-globs filter, shared by wizard and preflight), `tfc_vcs_summary`/`tfc_vcs_kind_for`/`tfc_vcs_label_for`, `tfc_token` with provider precedence, `require_tfc_auth`), `sg_api.sh` (`_sg_api` 0/22/1 contract + `SG_HTTP_CODE`, `_sg_listall` paginated listall reader (the API pages at 50) behind `sg_list_workflows`/`sg_list_wfgrps`, `sg_set_vcs_triggers`, `sg_list_integrations`, `sg_integration_type`, `sg_vcs_kind_of` (connector type → `sourceConfigDestKind`), `sg_execution_preset` + `sg_preset_{desc,runner_desc,version_desc,runner_provided}` (the org's execution preset and how to describe it), `sg_*_exists`, `sg_list_workflows`, `sg_create_secret`, `sg_patch_workflow`, `wfgroup_*`), `wizard.sh` (`wizard_run`: 5 steps TFC → SG → projects → defaults → review; `wizard_projects` (with >1 selected project, from `W_SEL_PROJECTS_JSON`) asks whether the global connectors and `tfc-<project>` groups apply to all, else per project the cloud connector, VCS connector (kind + prefix follow) and group (default / existing via `sg_list_wfgrps` / typed), writing `projectOverrides`; "same as the default" deletes the key, unmanaged fields are kept, previous picks come first (`_w_first`); the defaults step also asks `stripCloudAuthVars`; the TFC step first asks which TFC projects to migrate (with >1 project: `all` or `pick` by name/slug, resolved to raw names, unknown names re-asked; a previous non-empty `tfProjects` makes `pick` the first item; free text without discovery) → `W_PROJECTS_JSON`, rendered as `tfProjects` (`[]` = all) and applied to the workspace selection before the scope question; it keeps the workspace list and derives, via `tfc_vcs_summary`, which VCS provider the selected workspaces use and the repositories' URL prefix — the SG step lists matching connectors first and takes the repo URL prefix from TFC (never from a previous tfvars written for another provider kind), warns on a kind mismatch, reads the org's execution preset and offers it as the runner choice; the defaults step asks whether to carry TFC versions or use the preset (`SGTerraformVersionSource`) and, for carry, which fallback (`SGDefaultTerraformVersion` or the preset = null); the review shows human-readable rows plus the `tfc-<project>` groups; `_w_select_lines` feeds menus line-by-line so names with spaces/globs survive), `preflight.sh` (`preflight_run <apply|import|all>`, once per process; ✓ lines are plain language and name the connector's role/kind — related passing checks share one line (connector + kind + repo prefix, the project inside the selection line, version + runners when both come from the preset, one summary for the override keys) and routine ones are only counted (`pf_pass`), while ✗/! lines always print and name the tfvars field to fix; connector ids, runner groups and DPC kinds are collected from `SGDefault*`, `projectOverrides` and `workspaceOverrides`; besides existence it checks consistency — VCS kind vs. the connector's type, globally and per `projectOverrides` entry (fail), repo URL prefix vs. where the TFC repositories live or the kind's well-known host (warn), `projectOverrides` keys vs. the TFC project names (warn), `--project` values vs. the project names/slugs (fail; a `tfProjects` typo warns), and the workspace-selection preview applies the CLI scope (names, excludes, tags, projects) so its count is what the apply will export; the import context states the group policy and warns when a project's group changed since its last import; it also reports the org's execution preset next to the version/runner policy and warns when `carry` meets a preset that mounts a runner-provided binary), `state.sh` (`.sg/state.json`: `phases.<name>.input_sha`, `import.<seg>.{group,imported,updated,failed,tf_fallback,state_uploaded,state_failed}` (`group` anchors the never-move check; `updated` = PATCHed on a 409), `triggers.<seg>.{set,unchanged,failed,missing,sha}`, `secrets.<name>`; `run_phase` in migrate.sh skips unchanged phases), `report.sh` (`show_migration_summary` after apply — incl. a state-files row, and one `Version & runners` row when both are left to the preset —, `show_import_plan` per-workflow table with columns sized to their content and a legend that only explains what the rows use (no ACTION line for create-only plans; the fallback/preset/SECRETS fragments only when a row shows them) — it keeps the rows in `PLAN_ROWS_TSV` (name, group, action, version, runner, triggers, vars, secrets, segment, workspace, project) —, `write_run_result <outcome>` → `export/run-result.json` + `run-summary.md` from `PLAN_ROWS_TSV`, `PLAN_GROUP_ROWS` (plain copy of the group table, set by `plan_groups`), `PLAN_PROBLEMS`, the merged state and `CHECKLIST_OPEN`; per workflow `result` is planned/skipped/failed/updated/created, with the probe workflow — planned `create`, PATCHed on its second pass — reported as created), `errors.sh` (`explain_api_error` regex→hint table, `SG_API_HINTS`), `checklist.sh` (`create_secret_stubs` → SG secret `tfc-<wf>-<VAR>` = `CHANGE_ME`, referenced as `${secret::<name>}` via PATCH + payload patch; `write_checklist` → `export/post-import-checklist.md`, printed to the terminal as one ✓/! status line per section plus the imported groups with UI links — the file has the details; section 5 covers both the TFC state export and the SG state upload (`state_failed` from the import results), shown as one `state:` line when both are clean; sets `CHECKLIST_OPEN`). `sg_preset_desc` describes an empty preset as `platform default: ...` without nested parentheses. Parallel jobs (`run_parallel`) run in subshells, so `do_import`/`do_set_triggers` write `.import-result.<seg>.json` / `.triggers-result.<seg>.json` into the export dir and the caller merges them into state. - `scripts/tools.sh` — sourced by the other scripts (sets `SG_REPO_ROOT` to its parent dir). Provides `sg_resolve` (PATH-or-cache), `sg_ensure_{jq,hcl2json,yajsv,sgcli}` (pinned downloads into `.sg/cached/`; `sg-cli` is the Go binary), `sg_retry`, the color vars + log helpers. The Docker image bundles these tools at build time (`yajsv` built from source so arm64 works), so in-container runs never download. - **Variable sets** — `scripts/enrich_variable_sets.sh <tfc-org> <payload...>` (run by `cmd_enrich`, which reads `tfOrg`/`tfHostname` from `terraform.tfvars` via `hcl2json` — variable sets live in the **TFC** org, not `SG_ORG`). Lists all sets + vars via the TFC API (Bearer token from the `terraform login` creds file or `TFE_TOKEN`), computes per-workspace effective vars (scope: global/project/workspace; precedence: priority sets > workspace vars > non-priority sets), and merges them into each workflow (matched by workspace name parsed from `TfStateFilePath`). Sensitive set vars and key conflicts are reported. Skipped by `--no-variable-sets`. - **Regions** — `--region eu|us` / `SG_REGION` (`region_urls`/`region_apply` in migrate.sh) pick `SG_BASE_URL` and `SG_UI_URL` together: eu = `api.app`/`app.stackguardian.io` (default), us = `api.us`/`us.stackguardian.io`, and the undocumented internal `dash` = `testapi.qa`/`dash.qa.stackguardian.io` (never mention it in user-facing docs). An explicit `SG_BASE_URL`/`SG_UI_URL` wins over the region. `init` stores `config.sg_region` in `.sg/state.json` (older files only have `sg_base_url`, still honoured); the plan header and `run-result.json` (`region`, `apiUrl`, `uiUrl`) show what a run used. The wrapper forwards `SG_REGION`. diff --git a/scripts/lib/report.sh b/scripts/lib/report.sh index df41576..c45b4be 100644 --- a/scripts/lib/report.sh +++ b/scripts/lib/report.sh @@ -162,19 +162,35 @@ show_import_plan() { rw=8 case "$all_rows" in *$'\t'preset$'\t'*) rw="$(sg_maxlen 8 "preset (${SG_PRESET_RUNNER_SHORT:-})")" ;; esac printf "\n %s%-${wn}s %-${gn}s %-7s %-28s %-${rw}s %-8s %-5s %s%s\n" "$C_BOLD" "WORKFLOW" "GROUP" "ACTION" "TERRAFORM" "RUNNER" "TRIGGERS" "VARS" "SECRETS" "$C_RESET" >&2 + # The legend below only explains what the table actually shows. + local has_create=0 has_update=0 has_skip=0 has_fallback=0 has_preset=0 has_secrets=0 legend="" while IFS=$'\t' read -r name grp action tfv runner trig vars secrets _; do [ -n "$name" ] || continue - if [ "$tfv" = "preset" ]; then tfv="preset${SG_PRESET_TFV:+ ($SG_PRESET_TFV)}" - elif tf_version_above_ceiling "$tfv"; then tfv="${tfv#TERRAFORM-} -> ${SG_TF_FALLBACK_SHORT:-${SG_DEFAULT_TF_VERSION#TERRAFORM-}} (fallback)" + if [ "$tfv" = "preset" ]; then tfv="preset${SG_PRESET_TFV:+ ($SG_PRESET_TFV)}"; has_preset=1 + elif tf_version_above_ceiling "$tfv"; then tfv="${tfv#TERRAFORM-} -> ${SG_TF_FALLBACK_SHORT:-${SG_DEFAULT_TF_VERSION#TERRAFORM-}} (fallback)"; has_fallback=1 else tfv="${tfv#TERRAFORM-}"; fi - [ "$runner" = "preset" ] && runner="preset${SG_PRESET_RUNNER_SHORT:+ ($SG_PRESET_RUNNER_SHORT)}" - [ "$secrets" = "0" ] && secrets="-" - case "$action" in create) action="${C_GREEN}create ${C_RESET}" ;; update) action="${C_YELLOW}update ${C_RESET}" ;; skip) action="${C_DIM}skip ${C_RESET}" ;; esac + [ "$runner" = "preset" ] && { runner="preset${SG_PRESET_RUNNER_SHORT:+ ($SG_PRESET_RUNNER_SHORT)}"; has_preset=1; } + if [ "$secrets" = "0" ]; then secrets="-"; else has_secrets=1; fi + case "$action" in + create) action="${C_GREEN}create ${C_RESET}"; has_create=1 ;; + update) action="${C_YELLOW}update ${C_RESET}"; has_update=1 ;; + skip) action="${C_DIM}skip ${C_RESET}"; has_skip=1 ;; + esac printf " %-${wn}s %-${gn}s %s %-28s %-${rw}s %-8s %-5s %s\n" "$name" "$grp" "$action" "$tfv" "$runner" "$trig" "$vars" "$secrets" >&2 done <<<"$all_rows" echo >&2 - sg_dim "ACTION create = new workflow; update = exists in the group, PATCHed with the current payload; skip = file already imported with identical content (--fresh re-imports)" - sg_dim "TERRAFORM '-> fallback' = pinned above SG's managed ceiling (1.5.7, last FOSS release); 'preset' = left to the org's execution preset at import; SECRETS = sensitive vars recreated as placeholder secrets" + if [ "$has_update" -eq 1 ] || [ "$has_skip" -eq 1 ]; then + [ "$has_create" -eq 1 ] && legend="create = new workflow" + [ "$has_update" -eq 1 ] && legend="$legend${legend:+; }update = exists in the group, PATCHed with the current payload" + [ "$has_skip" -eq 1 ] && legend="$legend${legend:+; }skip = file already imported with identical content (--fresh re-imports)" + sg_dim "ACTION $legend" + fi + legend="" + [ "$has_fallback" -eq 1 ] && legend="TERRAFORM '-> fallback' = pinned above SG's managed ceiling (1.5.7, last FOSS release)" + [ "$has_preset" -eq 1 ] && legend="$legend${legend:+; }'preset' = left to the org's execution preset at import" + [ "$has_secrets" -eq 1 ] && legend="$legend${legend:+; }SECRETS = sensitive vars recreated as placeholder secrets" + [ -n "$legend" ] && sg_dim "$legend" + return 0 } # write_run_result <outcome> — export/run-result.json and run-summary.md: what From ca40a6d07369b7afe895461d2a2a3c82e07711c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adis=20Halilovi=C4=87?= <adis.halilovic@stackguardian.io> Date: Mon, 14 Sep 2026 18:49:58 +0200 Subject: [PATCH 69/71] =?UTF-8?q?fix:=20drop=20the=20[sg-migrate]=20prefix?= =?UTF-8?q?=20=E2=80=94=20phase=20lines=20are=20plain,=20=E2=9C=93,=20!=20?= =?UTF-8?q?or=20=E2=9C=97?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scripts/tools.sh | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/scripts/tools.sh b/scripts/tools.sh index bab5d79..a6855ed 100755 --- a/scripts/tools.sh +++ b/scripts/tools.sh @@ -47,10 +47,12 @@ sg_rel() { esac } -sg_log() { printf '%s[sg-migrate]%s %s\n' "$C_CYAN" "$C_RESET" "$*" >&2; } -sg_warn() { printf '%s[sg-migrate] WARN%s %s\n' "$C_YELLOW" "$C_RESET" "$*" >&2; } -sg_err() { printf '%s[sg-migrate] ERROR%s %s\n' "$C_RED$C_BOLD" "$C_RESET" "$*" >&2; } -sg_success() { printf '%s[sg-migrate] ✓%s %s\n' "$C_GREEN$C_BOLD" "$C_RESET" "$*" >&2; } +# Phase-level lines, no prefix: plain progress, "! warning", "✗ error", "✓ done". +# (Item lines — one per workflow or check — use the indented sg_ok/sg_bad/sg_note below.) +sg_log() { printf '%s\n' "$*" >&2; } +sg_warn() { printf '%s!%s %s\n' "$C_YELLOW$C_BOLD" "$C_RESET" "$*" >&2; } +sg_err() { printf '%s✗%s %s\n' "$C_RED$C_BOLD" "$C_RESET" "$*" >&2; } +sg_success() { printf '%s✓%s %s\n' "$C_GREEN$C_BOLD" "$C_RESET" "$*" >&2; } sg_step() { printf '\n%s==> %s%s\n' "$C_CYAN$C_BOLD" "$*" "$C_RESET" >&2; } sg_dim() { printf '%s %s%s\n' "$C_DIM" "$*" "$C_RESET" >&2; } # sg_ok / sg_bad / sg_note — indented " ✓ ..." / " ✗ ..." / " ! ..." status @@ -87,7 +89,7 @@ _SG_SPIN_FRAMES=('⠋' '⠙' '⠹' '⠸' '⠼' '⠴' '⠦' '⠧' '⠇' '⠏') _SG_SPIN_I=0 sg_spin_frame() { [ "$SG_ANIMATE" = "1" ] || return 0 - printf '\r%s[sg-migrate]%s %s%s%s %s\033[K' "$C_CYAN" "$C_RESET" "$C_CYAN" "${_SG_SPIN_FRAMES[_SG_SPIN_I % 10]}" "$C_RESET" "$1" >&2 + printf '\r%s%s%s %s\033[K' "$C_CYAN" "${_SG_SPIN_FRAMES[_SG_SPIN_I % 10]}" "$C_RESET" "$1" >&2 _SG_SPIN_I=$((_SG_SPIN_I + 1)) } sg_spin_clear() { From b84572cb3dbcfe0e35e5fc6563d5da0aa6db6797 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adis=20Halilovi=C4=87?= <adis.halilovic@stackguardian.io> Date: Mon, 14 Sep 2026 18:49:58 +0200 Subject: [PATCH 70/71] =?UTF-8?q?fix:=20fold=20the=20remaining=20repeated?= =?UTF-8?q?=20lines=20=E2=80=94=20skipped=20phases,=20clean=20checklist=20?= =?UTF-8?q?sections,=20per-file=20=E2=9C=93s,=20the=20group=20table=20when?= =?UTF-8?q?=20every=20group=20is=20reused?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CLAUDE.md | 4 +- README.md | 2 +- scripts/lib/checklist.sh | 8 +++- scripts/migrate.sh | 81 ++++++++++++++++++++++++++++--------- scripts/validate_payload.sh | 2 +- sg-migrate.sh | 7 +++- 6 files changed, 79 insertions(+), 25 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 0aac0b2..b4307e7 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -24,8 +24,8 @@ The migration is a user-driven pipeline, not a single program: The five phases are wrapped by an orchestrator so users don't run them by hand: - `sg-migrate.sh` (host entrypoint, repo root) — runs `scripts/migrate.sh` **inside the Docker image** (`Dockerfile`), bind-mounting the repo at `/app` and forwarding `SG_API_TOKEN`/`SG_ORG`/`SG_BASE_URL`/`TFE_TOKEN`/`TF_TOKEN_*`. `--tfvars FILE` (or `SG_TFVARS`) is a host path: the wrapper strips the flag, resolves it and passes it on as `SG_TFVARS` — rewritten to `/app/<relative>` when the file is inside the checkout, otherwise mounted read-only at `/tmp/sg-run.tfvars`. For TFC auth it prefers a long-lived `TFE_TOKEN` (env); otherwise it mounts `~/.terraform.d/credentials.tfrc.json` read-only. Runs natively instead when `--native`/`--local` is passed, `SG_NATIVE=1` is set, the command is `clean`/`completion`/help (or no command is given), or Docker is absent. Exports `SG_PROG` so `migrate.sh` shows `./sg-migrate.sh` in its usage/hints. `update` is host-only too (`cmd_update` in the wrapper, never passed to `migrate.sh`): refuses a non-git checkout, a detached HEAD or dirty tracked files, then `git pull --ff-only` and rebuilds the image only when the `Dockerfile` changed between the old and new HEAD — this is how customers pick up fixes from a plain clone (untracked config/output survive). `migrate.sh` lists `update` in its usage/`SG_COMMANDS` and rejects it with a host-only hint. `.gitattributes` forces LF so a Git-for-Windows clone does not CRLF the scripts. -- `scripts/migrate.sh` — the actual orchestrator (sources `scripts/lib/*.sh`, see below). Subcommands `init|preflight|apply|enrich|convert|validate|import|triggers|checklist|all|clean|completion` (no command prints the help menu; `enrich` runs in `all` unless `--no-variable-sets`). Flags beyond the basics: `--dry-run` (import, or all: the local export runs, then the plan is shown and nothing is created in SG), `--fresh` (ignore run state), the scope flags from `lib/scope.sh` — `--project NAME|SLUG`, `--workspace GLOB`, `--exclude-workspace GLOB`, `--tag NAME`, `--exclude-tag NAME`, all repeatable; tfvars holds the widest scope, include flags replace the tfvars list, exclude flags add to it; apply passes them as `-var` (`tfProjects`, `workspacenames`, `tfWorkspaceIgnoreNames`, `tfWorkspaceTags`, `tfWorkspaceIgnoreTags`), the later phases apply the same names/excludes to the payload entries (`ws_selected` in bash, `WS_SCOPE_JQ` + `WS_JQ_ARGS` in jq) and `--project` to the payload files by slug (`project_selected`); `--workspace '*'` alone is not a narrowing, so the unchanged-file skip still applies; a scope that matches nothing is an error —, `--tfvars FILE` (`TFVARS`, absolute; also `-var-file` for terraform and exported to the enrich script; `clean --all` only removes the module's own file), `--skip-preflight`, `--no-secret-stubs`, `--upgrade` (with `init`: `tfvars_upgrade`, append the settings an older tfvars lacks; preflight warns about them until then), and the run configuration flags `--set KEY=VALUE`, `--cloud-connector ID`, `--vcs-connector ID`, `--runner-group NAME|shared`, `--workflow-group NAME` (one-run overlay on the tfvars, see `scope.sh`; connector kinds looked up in SG; with `--project` they are that project's `projectOverrides`, else the `SGDefault*` values). `import`/`all` end by writing `export/run-result.json` + `run-summary.md` (`write_run_result`, outcomes planned/blocked/success/failed) for CI. Resolves each tool via `sg_resolve` (PATH first — the image installs them — else `sg_ensure_*` cache). `apply`/`all` first run `require_tfc_auth`, which resolves the token the tfe provider will use (`TFE_TOKEN`, `TF_TOKEN_<host>`, or the `terraform login` file for `tfHostname`) and verifies it with `GET /api/v2/account/details`, failing fast on a missing or rejected (expired) credential. Runs `convert` and `import` in parallel (`--concurrency`, default 4), raises `terraform apply -parallelism`, and retries `sg-cli` imports with backoff (`sg_retry`, `SG_RETRIES`). Import resolves each project's workflow group with `group_for` (legacy `.sg/workflow-groups.json` entry → the payload's `CLIConfiguration.WorkflowGroup.name` via `payload_group` → `tfc-<seg>`) and runs `plan_groups`: STATUS `reuse` (exists) / `create` (missing; created after the confirmation) / `missing!` (`--no-create-groups`) / `moved!`; it collects `PLAN_PROBLEMS` — a project whose workflows still live in the group recorded in `state.json` (or the default group) when the target changed (SG cannot move workflows), and two projects sharing a group with overlapping names — prints them as ✗ lines, still shows the per-workflow plan (also under `--dry-run`) and then dies. The skip set (`state_import_done`) is computed before the plan so the ACTION column can say `skip`/`update`/`create`. Groups are never PATCHed. Imports go through `import_bulk`: it first lists the group (`sg_list_workflows`) and PATCHes every workflow that already exists (`sg_update_workflow`, `[updated]` marker, state re-uploaded, failures as `Failed to update <wf>: ...`), so the plan's `update` rows never hit a create; the remaining, new workflows with Terraform variables go via `sg-cli workflow create --bulk`, workflows **without** via a direct `POST .../wfs/` (`sg_create_workflow` + `sg_upload_tfstate` in `sg_api.sh`) because sg-cli drops an empty `iacInputData.data` and the API rejects the workflow — a workaround (`TODO(sg-cli)` markers) until sg-cli ships with sg-sdk-go >= v1.5.7, which sends the empty object; the direct path prints sg-cli-style `Failed to create <name>: <code>: <body>` lines so the parsing below covers both. sg-cli's own output is shown only with `-v`; by default `import_bulk` renders one line per workflow from it — `wf_line` (`✓ <group>/<wf> created|updated[, state uploaded]`, a `!` line when the state upload failed) and `wf_failed` (`✗ <group>/<wf> not created|updated — <code>: <body>`, followed by `explain_api_error`'s hint) — while the raw lines still go to the parse file for `do_import`; `upload_state` prints nothing itself and leaves `US_RESULT`/`US_WHY` for `wf_line`; the `importing <file> -> <group>` and "N workflow(s) already exist / have no Terraform variables" lines are `-v` only. State files are the migrator's job on both paths: `import_bulk` tracks sg-cli's per-workflow upload lines (`State file uploaded successfully` / `Failed to upload state file for <wf>`), re-uploads the failures via `upload_state` → `sg_upload_tfstate` (adds the `x-ms-blob-type: BlockBlob` header Azure Blob requires; sg-cli omits it and also treats anything but a literal `HTTP/1.1 200 OK` as failure — second `TODO(sg-cli)`), and writes `[state] uploaded|failed|none <wf>` markers into the parsed output; `do_import` turns them into `state_uploaded`/`state_failed` in the result, a workflow without its state fails the file (and `state_import_done` re-tries it next run). Existing workflows come back from sg-cli as `Failed to create <wf>: 409: Workflow ID not unique` (its update branch matches the outdated text `Workflow name not unique` — third `TODO(sg-cli)`); `import_bulk` collects those, PATCHes them via `sg_update_workflow`, re-uploads their state and drops the failure line, and `sg_create_workflow` does the same fallback on the direct path (prints `updated`). Before the parallel import, `probe_import` imports one workflow alone (first selected of the first file) and dies unless the create and the state upload both succeeded — the fail-fast for environment problems. sg-cli exits 0 even when individual workflows fail, so `do_import` parses its output: a workflow rejected as above SG's managed Terraform ceiling (1.5.7, last MPL/FOSS release) is re-imported with `SGDefaultTerraformVersion` (read from `terraform.tfvars`; an explicit `null` there means the `terraformVersion` key is deleted instead, so the org's execution preset decides — `tfvars_is_null` tells an explicit null from a missing key), the payload patched in place, and the case logged to `export/terraform-version-fallbacks.log` plus a printed notice; any other per-workflow failure fails the run. `cmd_import` reads the org's execution preset (`sg_execution_preset`, `GET /orgs/{org}/` → `Settings.workflowDefaults`) and `preset_labels` (report.sh) turns it into the `preset (1.5.7)` / `preset (private:rg)` cells of the plan and the `SG_TF_FALLBACK_LABEL` used in messages. The trigger pass (`do_set_triggers` → `sg_set_vcs_triggers`) skips workflows that do not exist in SG, keeps the sha of each posted `{VCSConfig, VCSTriggers}` body in `triggers.<seg>.sha` and does not re-POST an unchanged, already-set workflow (`--fresh` does; the endpoint upserts anyway, a 4xx "already exists" from older builds counts as success), and `sg_api_post` returns 22 on 4xx so `sg_retry` (via `SG_NO_RETRY_RC`) does not retry definitive errors; `do_set_triggers` ends with one line per file that leaves out zero counts (`triggers set on N workflow(s), M unchanged, K without triggers, F failed, X not in SG`). `clean` removes local artifacts (`export/`, TF state, tool cache); `clean --all` also removes config. `completion bash|zsh` prints a completion script (`cmd_completion`; commands + descriptions come from `SG_COMMAND_DESCS` (`SG_COMMANDS` is derived from it), options from `SG_OPTIONS`; keep them in sync with the parser and the host flags; the zsh script also works autoloaded from `$fpath` as `_sg-migrate.sh`); like `clean` it always runs natively. Output is concise by default — `apply` captures terraform's init/plan output and shows only progress + the final summary (or the full log on failure, `TF_IN_AUTOMATION=1`/`-no-color`); `-v`/`--verbose` (`SG_VERBOSE`) streams everything and un-gates the convert detail lines. Colored logging via `sg_step`/`sg_log`/`sg_success`/`sg_warn`/`sg_err`/`sg_row`, plus the indented per-item status lines `sg_ok`/`sg_bad`/`sg_note` (✓/✗/!); paths shown relative via `sg_rel` (all auto-off when not a TTY / `NO_COLOR`). Long steps show a live progress line (`sg_run_quiet` for terraform init/apply, `run_parallel <fn> <max> <label> <items>` for convert/import/triggers — animated only when `SG_ANIMATE=1`, i.e. colors are on; plain log lines otherwise) and report their duration (`sg_fmt_secs`). Phases start with `phase_begin` — inside `all` the header is numbered (`Phase 2/5: ...`, `PHASE_TOTAL` set in `main`) and `phase_took` gives the elapsed time; `run_parallel` prints a job's output as-is when it is a single line and adds the `── <file> ──` header only for longer or failed output (always with `-v`). The import confirmation goes through `sg_confirm` (so `SG_ANSWERS_FILE` works; a declined prompt is a neutral "cancelled" line, not an error) and `import`/`all` end with `finish_line` (outcome, total time, open checklist items from `CHECKLIST_OPEN`). A Ctrl-C is trapped to clear the progress line and print a resume hint. -- `scripts/lib/` — sourced libraries (a `lib/` gitignore rule is negated for this dir): `prompt.sh` (`sg_ask`/`sg_confirm`/`sg_select`; tty or `SG_ANSWERS_FILE` for tests; `SG_NONINTERACTIVE=1`/`-y`/no TTY → defaults), `tfvars.sh` (`tfvars_get` cached hcl2json reads — `tfvars_json` merges the run overlay (`TFVARS_OVERLAY_JSON`) over the file, and `tfvars_get`/`tfvars_get_json` keep a `false` value instead of treating it like a missing key —, `tfvars_has`/`tfvars_is_null` to tell an explicit `null` from a missing key, `tfvars_write` renders the wizard's `W_*` vars — `W_TF_SOURCE`, and `W_TF_VERSION`/`W_RUNNER_JSON` may be the literal `null`; plain strings go through `_tfvars_str`, so quotes/backslashes in names stay valid HCL; `projectOverrides`/`workspaceOverrides` are rendered from `W_PROJECT_OVERRIDES_JSON`/`W_WS_OVERRIDES_JSON` by `_tfvars_map` — `"key" = { attr = <json> }`, which hcl2json round-trips — so hand-written blocks survive a re-run of `init` (`wizard_run` reads them before writing; inner comments are lost); `tfWorkspaceIgnoreNames` is kept the same way (`W_IGNORE_NAMES_JSON`, never asked, shown in the review's scope row), and every other setting the wizard does not ask about (`tfProjects`, `cloudAuthVarPatterns`, hand-added keys) is appended verbatim from `W_PREV_JSON` under "Kept from the previous file"; `tfvars_missing_keys` (keys of `terraform.tfvars.example` the file lacks) and `tfvars_upgrade` (appends those paragraphs of the example — comment + default — under a dated header, `.bak` kept, touches nothing else; behind `init --upgrade`, prompt-free)), `scope.sh` (the run scope: `PROJECT_FILTER`/`WS_FILTER`/`WS_EXCLUDE`/`TAG_FILTER`/`TAG_EXCLUDE` arrays, `names_json`, `slug_of` (the transformer's project-slug rule in bash), `project_selected <seg>`, `ws_narrowed`/`ws_filter_json`/`ws_exclude_json` (tfvars `tfWorkspaceIgnoreNames` ∪ flags, memoized)/`ws_selected <name>` (bash `case` globs), `WS_SCOPE_JQ` + `ws_jq_args` → `WS_JQ_ARGS` for the jq sites, `scope_tags_json`/`scope_ignore_tags_json`, `scope_tfvar_args` → `SCOPE_TFVAR_ARGS` for apply, `scope_describe`, `scope_sha_input` for the apply phase hash; and the **run configuration overlay**: `--set KEY=VALUE` (`SET_VARS`, values via `_overlay_value`: HCL/JSON literals through hcl2json, bare text stays a string), `--cloud-connector`/`--vcs-connector`/`--runner-group`/`--workflow-group` → `overlay_build` looks the connector kinds up with `sg_list_integrations`/`sg_integration_type`/`sg_vcs_kind_of`, resolves `--project` values to raw TFC names (`_overlay_project_name`, fatal for an unknown project, as-given with a warning when TFC is unreachable) and fills `TFVARS_OVERLAY_JSON` — `projectOverrides[<name>]` fields with `--project`, `SGDefault*` keys without; `tfvars_json` deep-merges it over the file so every reader sees it, `scope_tfvar_args` passes each overlay key as `-var`, `scope_sha_input` includes it, `write_run_result` records it under `configuration`; `main` refuses the flags for `init`/`clean`/`completion` and a standalone `import` warns that they only reach the workflows through the export (`RAN_APPLY`)), `tfc_api.sh` (`tfc_http`/`tfc_get_all` paginated, `tfc_list_{orgs,projects,workspaces}` — workspaces include `vcs_provider`/`vcs_url`/`vcs_identifier` —, `tfc_select_workspaces` (the module's name-glob / include-tags(all) / exclude-tags / exclude-name-globs filter, shared by wizard and preflight), `tfc_vcs_summary`/`tfc_vcs_kind_for`/`tfc_vcs_label_for`, `tfc_token` with provider precedence, `require_tfc_auth`), `sg_api.sh` (`_sg_api` 0/22/1 contract + `SG_HTTP_CODE`, `_sg_listall` paginated listall reader (the API pages at 50) behind `sg_list_workflows`/`sg_list_wfgrps`, `sg_set_vcs_triggers`, `sg_list_integrations`, `sg_integration_type`, `sg_vcs_kind_of` (connector type → `sourceConfigDestKind`), `sg_execution_preset` + `sg_preset_{desc,runner_desc,version_desc,runner_provided}` (the org's execution preset and how to describe it), `sg_*_exists`, `sg_list_workflows`, `sg_create_secret`, `sg_patch_workflow`, `wfgroup_*`), `wizard.sh` (`wizard_run`: 5 steps TFC → SG → projects → defaults → review; `wizard_projects` (with >1 selected project, from `W_SEL_PROJECTS_JSON`) asks whether the global connectors and `tfc-<project>` groups apply to all, else per project the cloud connector, VCS connector (kind + prefix follow) and group (default / existing via `sg_list_wfgrps` / typed), writing `projectOverrides`; "same as the default" deletes the key, unmanaged fields are kept, previous picks come first (`_w_first`); the defaults step also asks `stripCloudAuthVars`; the TFC step first asks which TFC projects to migrate (with >1 project: `all` or `pick` by name/slug, resolved to raw names, unknown names re-asked; a previous non-empty `tfProjects` makes `pick` the first item; free text without discovery) → `W_PROJECTS_JSON`, rendered as `tfProjects` (`[]` = all) and applied to the workspace selection before the scope question; it keeps the workspace list and derives, via `tfc_vcs_summary`, which VCS provider the selected workspaces use and the repositories' URL prefix — the SG step lists matching connectors first and takes the repo URL prefix from TFC (never from a previous tfvars written for another provider kind), warns on a kind mismatch, reads the org's execution preset and offers it as the runner choice; the defaults step asks whether to carry TFC versions or use the preset (`SGTerraformVersionSource`) and, for carry, which fallback (`SGDefaultTerraformVersion` or the preset = null); the review shows human-readable rows plus the `tfc-<project>` groups; `_w_select_lines` feeds menus line-by-line so names with spaces/globs survive), `preflight.sh` (`preflight_run <apply|import|all>`, once per process; ✓ lines are plain language and name the connector's role/kind — related passing checks share one line (connector + kind + repo prefix, the project inside the selection line, version + runners when both come from the preset, one summary for the override keys) and routine ones are only counted (`pf_pass`), while ✗/! lines always print and name the tfvars field to fix; connector ids, runner groups and DPC kinds are collected from `SGDefault*`, `projectOverrides` and `workspaceOverrides`; besides existence it checks consistency — VCS kind vs. the connector's type, globally and per `projectOverrides` entry (fail), repo URL prefix vs. where the TFC repositories live or the kind's well-known host (warn), `projectOverrides` keys vs. the TFC project names (warn), `--project` values vs. the project names/slugs (fail; a `tfProjects` typo warns), and the workspace-selection preview applies the CLI scope (names, excludes, tags, projects) so its count is what the apply will export; the import context states the group policy and warns when a project's group changed since its last import; it also reports the org's execution preset next to the version/runner policy and warns when `carry` meets a preset that mounts a runner-provided binary), `state.sh` (`.sg/state.json`: `phases.<name>.input_sha`, `import.<seg>.{group,imported,updated,failed,tf_fallback,state_uploaded,state_failed}` (`group` anchors the never-move check; `updated` = PATCHed on a 409), `triggers.<seg>.{set,unchanged,failed,missing,sha}`, `secrets.<name>`; `run_phase` in migrate.sh skips unchanged phases), `report.sh` (`show_migration_summary` after apply — incl. a state-files row, and one `Version & runners` row when both are left to the preset —, `show_import_plan` per-workflow table with columns sized to their content and a legend that only explains what the rows use (no ACTION line for create-only plans; the fallback/preset/SECRETS fragments only when a row shows them) — it keeps the rows in `PLAN_ROWS_TSV` (name, group, action, version, runner, triggers, vars, secrets, segment, workspace, project) —, `write_run_result <outcome>` → `export/run-result.json` + `run-summary.md` from `PLAN_ROWS_TSV`, `PLAN_GROUP_ROWS` (plain copy of the group table, set by `plan_groups`), `PLAN_PROBLEMS`, the merged state and `CHECKLIST_OPEN`; per workflow `result` is planned/skipped/failed/updated/created, with the probe workflow — planned `create`, PATCHed on its second pass — reported as created), `errors.sh` (`explain_api_error` regex→hint table, `SG_API_HINTS`), `checklist.sh` (`create_secret_stubs` → SG secret `tfc-<wf>-<VAR>` = `CHANGE_ME`, referenced as `${secret::<name>}` via PATCH + payload patch; `write_checklist` → `export/post-import-checklist.md`, printed to the terminal as one ✓/! status line per section plus the imported groups with UI links — the file has the details; section 5 covers both the TFC state export and the SG state upload (`state_failed` from the import results), shown as one `state:` line when both are clean; sets `CHECKLIST_OPEN`). `sg_preset_desc` describes an empty preset as `platform default: ...` without nested parentheses. Parallel jobs (`run_parallel`) run in subshells, so `do_import`/`do_set_triggers` write `.import-result.<seg>.json` / `.triggers-result.<seg>.json` into the export dir and the caller merges them into state. +- `scripts/migrate.sh` — the actual orchestrator (sources `scripts/lib/*.sh`, see below). Subcommands `init|preflight|apply|enrich|convert|validate|import|triggers|checklist|all|clean|completion` (no command prints the help menu; `enrich` runs in `all` unless `--no-variable-sets`). Flags beyond the basics: `--dry-run` (import, or all: the local export runs, then the plan is shown and nothing is created in SG), `--fresh` (ignore run state), the scope flags from `lib/scope.sh` — `--project NAME|SLUG`, `--workspace GLOB`, `--exclude-workspace GLOB`, `--tag NAME`, `--exclude-tag NAME`, all repeatable; tfvars holds the widest scope, include flags replace the tfvars list, exclude flags add to it; apply passes them as `-var` (`tfProjects`, `workspacenames`, `tfWorkspaceIgnoreNames`, `tfWorkspaceTags`, `tfWorkspaceIgnoreTags`), the later phases apply the same names/excludes to the payload entries (`ws_selected` in bash, `WS_SCOPE_JQ` + `WS_JQ_ARGS` in jq) and `--project` to the payload files by slug (`project_selected`); `--workspace '*'` alone is not a narrowing, so the unchanged-file skip still applies; a scope that matches nothing is an error —, `--tfvars FILE` (`TFVARS`, absolute; also `-var-file` for terraform and exported to the enrich script; `clean --all` only removes the module's own file), `--skip-preflight`, `--no-secret-stubs`, `--upgrade` (with `init`: `tfvars_upgrade`, append the settings an older tfvars lacks; preflight warns about them until then), and the run configuration flags `--set KEY=VALUE`, `--cloud-connector ID`, `--vcs-connector ID`, `--runner-group NAME|shared`, `--workflow-group NAME` (one-run overlay on the tfvars, see `scope.sh`; connector kinds looked up in SG; with `--project` they are that project's `projectOverrides`, else the `SGDefault*` values). `import`/`all` end by writing `export/run-result.json` + `run-summary.md` (`write_run_result`, outcomes planned/blocked/success/failed) for CI. Resolves each tool via `sg_resolve` (PATH first — the image installs them — else `sg_ensure_*` cache). `apply`/`all` first run `require_tfc_auth`, which resolves the token the tfe provider will use (`TFE_TOKEN`, `TF_TOKEN_<host>`, or the `terraform login` file for `tfHostname`) and verifies it with `GET /api/v2/account/details`, failing fast on a missing or rejected (expired) credential. Runs `convert` and `import` in parallel (`--concurrency`, default 4), raises `terraform apply -parallelism`, and retries `sg-cli` imports with backoff (`sg_retry`, `SG_RETRIES`). Import resolves each project's workflow group with `group_for` (legacy `.sg/workflow-groups.json` entry → the payload's `CLIConfiguration.WorkflowGroup.name` via `payload_group` → `tfc-<seg>`) and runs `plan_groups`: STATUS `reuse` (exists) / `create` (missing; created after the confirmation) / `missing!` (`--no-create-groups`) / `moved!` — the FILE table is printed only when a group is not `reuse`, a problem was found or the legacy mapping applies, and the confirmation prompt names the groups to create; it collects `PLAN_PROBLEMS` — a project whose workflows still live in the group recorded in `state.json` (or the default group) when the target changed (SG cannot move workflows), and two projects sharing a group with overlapping names — prints them as ✗ lines, still shows the per-workflow plan (also under `--dry-run`) and then dies. The skip set (`state_import_done`) is computed before the plan so the ACTION column can say `skip`/`update`/`create`. Groups are never PATCHed. Imports go through `import_bulk`: it first lists the group (`sg_list_workflows`) and PATCHes every workflow that already exists (`sg_update_workflow`, `[updated]` marker, state re-uploaded, failures as `Failed to update <wf>: ...`), so the plan's `update` rows never hit a create; the remaining, new workflows with Terraform variables go via `sg-cli workflow create --bulk`, workflows **without** via a direct `POST .../wfs/` (`sg_create_workflow` + `sg_upload_tfstate` in `sg_api.sh`) because sg-cli drops an empty `iacInputData.data` and the API rejects the workflow — a workaround (`TODO(sg-cli)` markers) until sg-cli ships with sg-sdk-go >= v1.5.7, which sends the empty object; the direct path prints sg-cli-style `Failed to create <name>: <code>: <body>` lines so the parsing below covers both. sg-cli's own output is shown only with `-v`; by default `import_bulk` renders one line per workflow from it — `wf_line` (`✓ <group>/<wf> created|updated[, state uploaded]`, a `!` line when the state upload failed) and `wf_failed` (`✗ <group>/<wf> not created|updated — <code>: <body>`, followed by `explain_api_error`'s hint) — while the raw lines still go to the parse file for `do_import`; `upload_state` prints nothing itself and leaves `US_RESULT`/`US_WHY` for `wf_line`; the `importing <file> -> <group>` and "N workflow(s) already exist / have no Terraform variables" lines are `-v` only. State files are the migrator's job on both paths: `import_bulk` tracks sg-cli's per-workflow upload lines (`State file uploaded successfully` / `Failed to upload state file for <wf>`), re-uploads the failures via `upload_state` → `sg_upload_tfstate` (adds the `x-ms-blob-type: BlockBlob` header Azure Blob requires; sg-cli omits it and also treats anything but a literal `HTTP/1.1 200 OK` as failure — second `TODO(sg-cli)`), and writes `[state] uploaded|failed|none <wf>` markers into the parsed output; `do_import` turns them into `state_uploaded`/`state_failed` in the result, a workflow without its state fails the file (and `state_import_done` re-tries it next run). Existing workflows come back from sg-cli as `Failed to create <wf>: 409: Workflow ID not unique` (its update branch matches the outdated text `Workflow name not unique` — third `TODO(sg-cli)`); `import_bulk` collects those, PATCHes them via `sg_update_workflow`, re-uploads their state and drops the failure line, and `sg_create_workflow` does the same fallback on the direct path (prints `updated`). Before the parallel import, `probe_import` imports one workflow alone (first selected of the first file) and dies unless the create and the state upload both succeeded — the fail-fast for environment problems. sg-cli exits 0 even when individual workflows fail, so `do_import` parses its output: a workflow rejected as above SG's managed Terraform ceiling (1.5.7, last MPL/FOSS release) is re-imported with `SGDefaultTerraformVersion` (read from `terraform.tfvars`; an explicit `null` there means the `terraformVersion` key is deleted instead, so the org's execution preset decides — `tfvars_is_null` tells an explicit null from a missing key), the payload patched in place, and the case logged to `export/terraform-version-fallbacks.log` plus a printed notice; any other per-workflow failure fails the run. `cmd_import` reads the org's execution preset (`sg_execution_preset`, `GET /orgs/{org}/` → `Settings.workflowDefaults`) and `preset_labels` (report.sh) turns it into the `preset (1.5.7)` / `preset (private:rg)` cells of the plan and the `SG_TF_FALLBACK_LABEL` used in messages. The trigger pass (`do_set_triggers` → `sg_set_vcs_triggers`) skips workflows that do not exist in SG, keeps the sha of each posted `{VCSConfig, VCSTriggers}` body in `triggers.<seg>.sha` and does not re-POST an unchanged, already-set workflow (`--fresh` does; the endpoint upserts anyway, a 4xx "already exists" from older builds counts as success), and `sg_api_post` returns 22 on 4xx so `sg_retry` (via `SG_NO_RETRY_RC`) does not retry definitive errors; `do_set_triggers` ends with one line per file that leaves out zero counts (`triggers set on N workflow(s), M unchanged, K without triggers, F failed, X not in SG`). `clean` removes local artifacts (`export/`, TF state, tool cache); `clean --all` also removes config. `completion bash|zsh` prints a completion script (`cmd_completion`; commands + descriptions come from `SG_COMMAND_DESCS` (`SG_COMMANDS` is derived from it), options from `SG_OPTIONS`; keep them in sync with the parser and the host flags; the zsh script also works autoloaded from `$fpath` as `_sg-migrate.sh`); like `clean` it always runs natively. Output is concise by default — `apply` captures terraform's init/plan output and shows only progress + the final summary (or the full log on failure, `TF_IN_AUTOMATION=1`/`-no-color`; the `terraform providers ready` line is `-v` only); `convert` with a single file prints only the ✓ line (the file's own result line folded in) and `validate` prints one ✓ for all files (per-file ✓ with `-v`, ✗ always); `-v`/`--verbose` (`SG_VERBOSE`) streams everything and un-gates the convert detail lines. Colored logging via `sg_step`/`sg_log`/`sg_success`/`sg_warn`/`sg_err`/`sg_row` — phase-level lines carry no prefix (plain, `✓ `, `! `, `✗ `; the former `[sg-migrate]` tag was dropped since raw tool output is `-v` only) — plus the indented per-item status lines `sg_ok`/`sg_bad`/`sg_note` (✓/✗/!); paths shown relative via `sg_rel` (all auto-off when not a TTY / `NO_COLOR`). Long steps show a live progress line (`sg_run_quiet` for terraform init/apply, `run_parallel <fn> <max> <label> <items>` for convert/import/triggers — animated only when `SG_ANIMATE=1`, i.e. colors are on; plain log lines otherwise) and report their duration (`sg_fmt_secs`). Phases start with `phase_begin` — inside `all` the header is numbered (`Phase 2/5: ...`, `PHASE_TOTAL` set in `main`) and `phase_took` gives the elapsed time; `run_parallel` prints a job's output as-is when it is a single line and adds the `── <file> ──` header only for longer or failed output (always with `-v`). The import confirmation goes through `sg_confirm` (so `SG_ANSWERS_FILE` works; a declined prompt is a neutral "cancelled" line, not an error) and `import`/`all` end with `finish_line` (one line: outcome, total time, open checklist items from `CHECKLIST_OPEN`, and the checklist/run-result files). A Ctrl-C is trapped to clear the progress line and print a resume hint. +- `scripts/lib/` — sourced libraries (a `lib/` gitignore rule is negated for this dir): `prompt.sh` (`sg_ask`/`sg_confirm`/`sg_select`; tty or `SG_ANSWERS_FILE` for tests; `SG_NONINTERACTIVE=1`/`-y`/no TTY → defaults), `tfvars.sh` (`tfvars_get` cached hcl2json reads — `tfvars_json` merges the run overlay (`TFVARS_OVERLAY_JSON`) over the file, and `tfvars_get`/`tfvars_get_json` keep a `false` value instead of treating it like a missing key —, `tfvars_has`/`tfvars_is_null` to tell an explicit `null` from a missing key, `tfvars_write` renders the wizard's `W_*` vars — `W_TF_SOURCE`, and `W_TF_VERSION`/`W_RUNNER_JSON` may be the literal `null`; plain strings go through `_tfvars_str`, so quotes/backslashes in names stay valid HCL; `projectOverrides`/`workspaceOverrides` are rendered from `W_PROJECT_OVERRIDES_JSON`/`W_WS_OVERRIDES_JSON` by `_tfvars_map` — `"key" = { attr = <json> }`, which hcl2json round-trips — so hand-written blocks survive a re-run of `init` (`wizard_run` reads them before writing; inner comments are lost); `tfWorkspaceIgnoreNames` is kept the same way (`W_IGNORE_NAMES_JSON`, never asked, shown in the review's scope row), and every other setting the wizard does not ask about (`tfProjects`, `cloudAuthVarPatterns`, hand-added keys) is appended verbatim from `W_PREV_JSON` under "Kept from the previous file"; `tfvars_missing_keys` (keys of `terraform.tfvars.example` the file lacks) and `tfvars_upgrade` (appends those paragraphs of the example — comment + default — under a dated header, `.bak` kept, touches nothing else; behind `init --upgrade`, prompt-free)), `scope.sh` (the run scope: `PROJECT_FILTER`/`WS_FILTER`/`WS_EXCLUDE`/`TAG_FILTER`/`TAG_EXCLUDE` arrays, `names_json`, `slug_of` (the transformer's project-slug rule in bash), `project_selected <seg>`, `ws_narrowed`/`ws_filter_json`/`ws_exclude_json` (tfvars `tfWorkspaceIgnoreNames` ∪ flags, memoized)/`ws_selected <name>` (bash `case` globs), `WS_SCOPE_JQ` + `ws_jq_args` → `WS_JQ_ARGS` for the jq sites, `scope_tags_json`/`scope_ignore_tags_json`, `scope_tfvar_args` → `SCOPE_TFVAR_ARGS` for apply, `scope_describe`, `scope_sha_input` for the apply phase hash; and the **run configuration overlay**: `--set KEY=VALUE` (`SET_VARS`, values via `_overlay_value`: HCL/JSON literals through hcl2json, bare text stays a string), `--cloud-connector`/`--vcs-connector`/`--runner-group`/`--workflow-group` → `overlay_build` looks the connector kinds up with `sg_list_integrations`/`sg_integration_type`/`sg_vcs_kind_of`, resolves `--project` values to raw TFC names (`_overlay_project_name`, fatal for an unknown project, as-given with a warning when TFC is unreachable) and fills `TFVARS_OVERLAY_JSON` — `projectOverrides[<name>]` fields with `--project`, `SGDefault*` keys without; `tfvars_json` deep-merges it over the file so every reader sees it, `scope_tfvar_args` passes each overlay key as `-var`, `scope_sha_input` includes it, `write_run_result` records it under `configuration`; `main` refuses the flags for `init`/`clean`/`completion` and a standalone `import` warns that they only reach the workflows through the export (`RAN_APPLY`)), `tfc_api.sh` (`tfc_http`/`tfc_get_all` paginated, `tfc_list_{orgs,projects,workspaces}` — workspaces include `vcs_provider`/`vcs_url`/`vcs_identifier` —, `tfc_select_workspaces` (the module's name-glob / include-tags(all) / exclude-tags / exclude-name-globs filter, shared by wizard and preflight), `tfc_vcs_summary`/`tfc_vcs_kind_for`/`tfc_vcs_label_for`, `tfc_token` with provider precedence, `require_tfc_auth`), `sg_api.sh` (`_sg_api` 0/22/1 contract + `SG_HTTP_CODE`, `_sg_listall` paginated listall reader (the API pages at 50) behind `sg_list_workflows`/`sg_list_wfgrps`, `sg_set_vcs_triggers`, `sg_list_integrations`, `sg_integration_type`, `sg_vcs_kind_of` (connector type → `sourceConfigDestKind`), `sg_execution_preset` + `sg_preset_{desc,runner_desc,version_desc,runner_provided}` (the org's execution preset and how to describe it), `sg_*_exists`, `sg_list_workflows`, `sg_create_secret`, `sg_patch_workflow`, `wfgroup_*`), `wizard.sh` (`wizard_run`: 5 steps TFC → SG → projects → defaults → review; `wizard_projects` (with >1 selected project, from `W_SEL_PROJECTS_JSON`) asks whether the global connectors and `tfc-<project>` groups apply to all, else per project the cloud connector, VCS connector (kind + prefix follow) and group (default / existing via `sg_list_wfgrps` / typed), writing `projectOverrides`; "same as the default" deletes the key, unmanaged fields are kept, previous picks come first (`_w_first`); the defaults step also asks `stripCloudAuthVars`; the TFC step first asks which TFC projects to migrate (with >1 project: `all` or `pick` by name/slug, resolved to raw names, unknown names re-asked; a previous non-empty `tfProjects` makes `pick` the first item; free text without discovery) → `W_PROJECTS_JSON`, rendered as `tfProjects` (`[]` = all) and applied to the workspace selection before the scope question; it keeps the workspace list and derives, via `tfc_vcs_summary`, which VCS provider the selected workspaces use and the repositories' URL prefix — the SG step lists matching connectors first and takes the repo URL prefix from TFC (never from a previous tfvars written for another provider kind), warns on a kind mismatch, reads the org's execution preset and offers it as the runner choice; the defaults step asks whether to carry TFC versions or use the preset (`SGTerraformVersionSource`) and, for carry, which fallback (`SGDefaultTerraformVersion` or the preset = null); the review shows human-readable rows plus the `tfc-<project>` groups; `_w_select_lines` feeds menus line-by-line so names with spaces/globs survive), `preflight.sh` (`preflight_run <apply|import|all>`, once per process; ✓ lines are plain language and name the connector's role/kind — related passing checks share one line (connector + kind + repo prefix, the project inside the selection line, version + runners when both come from the preset, one summary for the override keys) and routine ones are only counted (`pf_pass`), while ✗/! lines always print and name the tfvars field to fix; connector ids, runner groups and DPC kinds are collected from `SGDefault*`, `projectOverrides` and `workspaceOverrides`; besides existence it checks consistency — VCS kind vs. the connector's type, globally and per `projectOverrides` entry (fail), repo URL prefix vs. where the TFC repositories live or the kind's well-known host (warn), `projectOverrides` keys vs. the TFC project names (warn), `--project` values vs. the project names/slugs (fail; a `tfProjects` typo warns), and the workspace-selection preview applies the CLI scope (names, excludes, tags, projects) so its count is what the apply will export; the import context states the group policy and warns when a project's group changed since its last import; it also reports the org's execution preset next to the version/runner policy and warns when `carry` meets a preset that mounts a runner-provided binary), `state.sh` (`.sg/state.json`: `phases.<name>.input_sha`, `import.<seg>.{group,imported,updated,failed,tf_fallback,state_uploaded,state_failed}` (`group` anchors the never-move check; `updated` = PATCHed on a 409), `triggers.<seg>.{set,unchanged,failed,missing,sha}`, `secrets.<name>`; `run_phase` in migrate.sh skips unchanged phases and reports consecutive skips on one line via `skipped_phases_flush` — `phases 1-4/5 (apply, enrich, convert, validate) unchanged since ... — skipped`), `report.sh` (`show_migration_summary` after apply — incl. a state-files row, and one `Version & runners` row when both are left to the preset —, `show_import_plan` per-workflow table with columns sized to their content and a legend that only explains what the rows use (no ACTION line for create-only plans; the fallback/preset/SECRETS fragments only when a row shows them) — it keeps the rows in `PLAN_ROWS_TSV` (name, group, action, version, runner, triggers, vars, secrets, segment, workspace, project) —, `write_run_result <outcome>` → `export/run-result.json` + `run-summary.md` from `PLAN_ROWS_TSV`, `PLAN_GROUP_ROWS` (plain copy of the group table, set by `plan_groups`), `PLAN_PROBLEMS`, the merged state and `CHECKLIST_OPEN`; per workflow `result` is planned/skipped/failed/updated/created, with the probe workflow — planned `create`, PATCHed on its second pass — reported as created), `errors.sh` (`explain_api_error` regex→hint table, `SG_API_HINTS`), `checklist.sh` (`create_secret_stubs` → SG secret `tfc-<wf>-<VAR>` = `CHANGE_ME`, referenced as `${secret::<name>}` via PATCH + payload patch; `write_checklist` → `export/post-import-checklist.md`, printed to the terminal as the imported groups with UI links, one `!` line per section that still needs a human and one `✓ secrets, imports, ...: nothing left to do` line for the clean sections (`_cl_status` collects them in `CL_CLEAN`) — the file has the details; section 5 covers both the TFC state export and the SG state upload (`state_failed` from the import results), shown as one `state:` line when both are clean; sets `CHECKLIST_OPEN`). `sg_preset_desc` describes an empty preset as `platform default: ...` without nested parentheses. Parallel jobs (`run_parallel`) run in subshells, so `do_import`/`do_set_triggers` write `.import-result.<seg>.json` / `.triggers-result.<seg>.json` into the export dir and the caller merges them into state. - `scripts/tools.sh` — sourced by the other scripts (sets `SG_REPO_ROOT` to its parent dir). Provides `sg_resolve` (PATH-or-cache), `sg_ensure_{jq,hcl2json,yajsv,sgcli}` (pinned downloads into `.sg/cached/`; `sg-cli` is the Go binary), `sg_retry`, the color vars + log helpers. The Docker image bundles these tools at build time (`yajsv` built from source so arm64 works), so in-container runs never download. - **Variable sets** — `scripts/enrich_variable_sets.sh <tfc-org> <payload...>` (run by `cmd_enrich`, which reads `tfOrg`/`tfHostname` from `terraform.tfvars` via `hcl2json` — variable sets live in the **TFC** org, not `SG_ORG`). Lists all sets + vars via the TFC API (Bearer token from the `terraform login` creds file or `TFE_TOKEN`), computes per-workspace effective vars (scope: global/project/workspace; precedence: priority sets > workspace vars > non-priority sets), and merges them into each workflow (matched by workspace name parsed from `TfStateFilePath`). Sensitive set vars and key conflicts are reported. Skipped by `--no-variable-sets`. - **Regions** — `--region eu|us` / `SG_REGION` (`region_urls`/`region_apply` in migrate.sh) pick `SG_BASE_URL` and `SG_UI_URL` together: eu = `api.app`/`app.stackguardian.io` (default), us = `api.us`/`us.stackguardian.io`, and the undocumented internal `dash` = `testapi.qa`/`dash.qa.stackguardian.io` (never mention it in user-facing docs). An explicit `SG_BASE_URL`/`SG_UI_URL` wins over the region. `init` stores `config.sg_region` in `.sg/state.json` (older files only have `sg_base_url`, still honoured); the plan header and `run-result.json` (`region`, `apiUrl`, `uiUrl`) show what a run used. The wrapper forwards `SG_REGION`. diff --git a/README.md b/README.md index d4f451c..fd39ca5 100644 --- a/README.md +++ b/README.md @@ -195,4 +195,4 @@ To update workflows with different details, re-run the sg-cli command with the m - **State export is idempotent.** Re-running `terraform apply` only pulls state for workspaces not yet exported. Set `forceStateRefresh = true` to re-pull everything. Workspaces not using `remote` execution may export incomplete state — see the summary. - **Workflow naming.** `ResourceName` mirrors the TFC workspace name, sanitized to StackGuardian's rules (1-100 chars, `[-a-zA-Z0-9_]`); any rename is listed in the summary and the checklist. - **Preflight.** `apply`, `import` and `all` first verify the TFC and SG tokens, the TFC org and workspace selection, and that every connector, secret and runner group referenced in `terraform.tfvars` (globally, per project and per workspace) exists and agrees with the VCS kinds; `projectOverrides`/`workspaceOverrides` keys are checked against the TFC projects and workspaces. Fix what it reports (or re-run `init`); `--skip-preflight` bypasses it. -- **Post-import checklist.** `export/post-import-checklist.md` collects the secrets to fill in, failed imports, Terraform version fallbacks, failed VCS triggers, missing state exports and renames, with links into the SG UI. The terminal shows one status line per section (plus links to the imported workflow groups); the file has the details. +- **Post-import checklist.** `export/post-import-checklist.md` collects the secrets to fill in, failed imports, Terraform version fallbacks, failed VCS triggers, missing state exports and renames, with links into the SG UI. The terminal lists the sections that still need a human, one line each, folds the clean ones into a single ✓ line and links the imported workflow groups; the file has the details. diff --git a/scripts/lib/checklist.sh b/scripts/lib/checklist.sh index 3208530..9e13b7f 100644 --- a/scripts/lib/checklist.sh +++ b/scripts/lib/checklist.sh @@ -95,9 +95,13 @@ _cl_count() { } # _cl_status <count> <ok-text> <attention-text> — one terminal line per section. +# _cl_status <count> <ok-text> <open-text> — an open section prints its "!" +# line; a clean one is only collected (CL_CLEAN) and the terminal view ends +# with one "✓ secrets, imports, ...: nothing left to do" line for all of them. +CL_CLEAN="" _cl_status() { if [ "${1:-0}" -gt 0 ]; then printf ' %s!%s %s\n' "$C_YELLOW" "$C_RESET" "$3" >&2 - else printf ' %s✓%s %s\n' "$C_GREEN" "$C_RESET" "$2" >&2; fi + else CL_CLEAN="$CL_CLEAN${CL_CLEAN:+, }${2%%:*}"; fi } # _cl_block <items> — the markdown list for a section, or "- None." @@ -112,6 +116,7 @@ write_checklist() { local n_secrets n_unstubbed n_failed n_fallback n_unpinned n_preset n_trig n_state n_stfail n_stok n_nonremote n_renamed local f seg grp n total=0 gw preset_desc="" local -a glines=() + CL_CLEAN="" st="$(state_read)" [ -n "${SG_PRESET_JSON:-}" ] && preset_desc="$(sg_preset_desc "$SG_PRESET_JSON")" @@ -211,6 +216,7 @@ write_checklist() { "state: $n_stfail workflow(s) are in SG without their state (upload failed) — re-run '$PROG import' or upload by hand" fi [ "$n_renamed" -gt 0 ] && _cl_status "$n_renamed" "" "names: $n_renamed workflow(s) were renamed to valid SG names" + [ -n "$CL_CLEAN" ] && printf ' %s✓%s %s\n' "$C_GREEN" "$C_RESET" "$CL_CLEAN: nothing left to do" >&2 # shellcheck disable=SC2034 CHECKLIST_OPEN=$((n_secrets + n_unstubbed + n_failed + n_fallback + n_unpinned + n_preset + n_trig + n_state + n_stfail + n_nonremote)) sg_dim "full checklist with links: $(sg_rel "$out")" diff --git a/scripts/migrate.sh b/scripts/migrate.sh index 24ac957..8335163 100755 --- a/scripts/migrate.sh +++ b/scripts/migrate.sh @@ -300,19 +300,38 @@ payload_sha() { # rewrites the payloads, so a payload hash could never match again. convert and # validate record the post-run payload hash, so an unchanged export is # recognised on the next run. +# Consecutive skipped phases are reported on one line ("phases 1-4 (apply, +# enrich, convert, validate) unchanged since ... — skipped"), flushed by +# skipped_phases_flush before the next phase that runs and before the import. +SKIPPED_PHASES="" +SKIPPED_FIRST=0 +SKIPPED_LAST=0 +SKIPPED_AT="" +skipped_phases_flush() { + [ -n "$SKIPPED_PHASES" ] || return 0 + local range="phase $SKIPPED_FIRST" + [ "$SKIPPED_LAST" -gt "$SKIPPED_FIRST" ] && range="phases $SKIPPED_FIRST-$SKIPPED_LAST" + if [ "$PHASE_TOTAL" -gt 0 ]; then + sg_log "$range/$PHASE_TOTAL (${SKIPPED_PHASES// /, }) unchanged since $SKIPPED_AT — skipped (--fresh to redo)" + else + sg_log "${SKIPPED_PHASES// /, } unchanged since $SKIPPED_AT — skipped (--fresh to redo)" + fi + SKIPPED_PHASES="" +} run_phase() { local name="$1" sha="$2" fn="$3" at if at="$(state_phase_done "$name" "$sha")" && [ -n "$at" ]; then at="${at%:*}" at="${at/T/ } UTC" - if [ "$PHASE_TOTAL" -gt 0 ]; then - PHASE_N=$((PHASE_N + 1)) - sg_log "skipping phase $PHASE_N/$PHASE_TOTAL ($name) — inputs unchanged since $at (--fresh to redo)" - else - sg_log "skipping $name — inputs unchanged since $at (--fresh to redo)" - fi + PHASE_N=$((PHASE_N + 1)) + [ -n "$SKIPPED_PHASES" ] || { SKIPPED_FIRST=$PHASE_N; SKIPPED_AT="$at"; } + SKIPPED_LAST=$PHASE_N + SKIPPED_PHASES="$SKIPPED_PHASES${SKIPPED_PHASES:+ }$name" + # The oldest timestamp is the one that matters when they differ. + [[ "$at" < "$SKIPPED_AT" ]] && SKIPPED_AT="$at" return 0 fi + skipped_phases_flush "$fn" || return $? case "$name" in apply | enrich) state_mark_phase "$name" "$sha" ;; @@ -518,13 +537,20 @@ plan_groups() { | select(($dups | length) > 0) | "workflow name(s) \($dups | join(", ")) appear in more than one project mapped to group \u0027\($g)\u0027 (\([.[].seg] | join(", "))) — a workflow name is unique within a group; give one of the projects its own workflowGroup"') - fw="$(sg_maxlen 4 "${files[@]}")" - gw="$(sg_maxlen 14 "${groups[@]}")" printf '%sImport plan%s (org: %s%s%s, %s)\n' "$C_BOLD" "$C_RESET" "$C_CYAN" "$ORG" "$C_RESET" "$SG_BASE_URL" >&2 - printf " %s%-${fw}s %-${gw}s %-9s %s%s\n" "$C_BOLD" "FILE" "WORKFLOW GROUP" "WORKFLOWS" "STATUS" "$C_RESET" >&2 - for ((i = 0; i < ${#files[@]}; i++)); do - printf " %-${fw}s %-${gw}s %-9s %s\n" "${files[i]}" "${groups[i]}" "${counts[i]}" "${statuses[i]}" >&2 - done + # The group table only when it says something the workflow table does not: + # a group to create, or a problem. All-reuse plans skip it (the GROUP column + # names the groups; the confirmation names the ones to create). + local all_reuse=1 + for plain in "${plains[@]}"; do [ "$plain" = reuse ] || all_reuse=0; done + if [ "$all_reuse" -eq 0 ] || [ "${#PLAN_PROBLEMS[@]}" -gt 0 ] || [ "$legacy" -gt 0 ]; then + fw="$(sg_maxlen 4 "${files[@]}")" + gw="$(sg_maxlen 14 "${groups[@]}")" + printf " %s%-${fw}s %-${gw}s %-9s %s%s\n" "$C_BOLD" "FILE" "WORKFLOW GROUP" "WORKFLOWS" "STATUS" "$C_RESET" >&2 + for ((i = 0; i < ${#files[@]}; i++)); do + printf " %-${fw}s %-${gw}s %-9s %s\n" "${files[i]}" "${groups[i]}" "${counts[i]}" "${statuses[i]}" >&2 + done + fi if [ "$legacy" -gt 0 ]; then sg_warn "$(sg_rel "$MAPPING") overrides the group of $legacy project(s) — this file is deprecated; set projectOverrides.\"<project>\".workflowGroup in $(sg_rel "$TFVARS") instead (project names: workspaceProjects in $(sg_rel "$EXPORT_DIR")/migration-summary.json) and re-run 'apply'" fi @@ -688,7 +714,7 @@ cmd_apply() { # Quiet: terraform's init/plan output goes to a log that is shown only on # failure; the terminal gets a live progress line per step instead. tflog="$(mktemp)" - sg_run_quiet "initializing terraform providers" "terraform providers ready" "$tflog" tf_init -no-color || rc=$? + sg_run_quiet "initializing terraform providers" "$([ "$VERBOSE" -eq 1 ] && echo "terraform providers ready")" "$tflog" tf_init -no-color || rc=$? if [ "$rc" -eq 0 ]; then sg_run_quiet "reading workspaces, generating payloads, exporting state" "workspaces read, payloads generated, state exported" "$tflog" \ tf_apply -no-color ${tfvar_args[@]+"${tfvar_args[@]}"} || rc=$? @@ -724,6 +750,17 @@ cmd_convert() { phase_begin "convert (HCL → JSON)" payload_files [ "${#PF[@]}" -gt 0 ] || die "No payload files in $(sg_rel "$EXPORT_DIR") (run 'apply' first)." + local one="" + if [ "${#PF[@]}" -eq 1 ] && [ "$VERBOSE" -ne 1 ]; then + # One file: its result line and the ✓ line would say the same — keep the ✓. + if one="$(do_convert "${PF[0]}" 2>&1)"; then + sg_success "${one#*: } in $(phase_took)" + return 0 + fi + printf '%s\n' "$one" >&2 + sg_err "conversion failed for $(basename "${PF[0]}")" + return 1 + fi if run_parallel do_convert "$CONC" "converting ${#PF[@]} payload file(s)" "${PF[@]}"; then sg_success "converted ${#PF[@]} payload file(s) in $(phase_took)" else @@ -737,7 +774,7 @@ cmd_validate() { payload_files [ "${#PF[@]}" -gt 0 ] || die "No payload files in $(sg_rel "$EXPORT_DIR")." if "$SCRIPT_DIR/validate_payload.sh" "${PF[@]}"; then - sg_success "${#PF[@]} payload file(s) valid against schema/sg-payload.schema.json" + sg_success "${#PF[@]} payload file(s) valid" else sg_err "validation failed — fix the payload(s) above (or the transformer) and re-run '$PROG validate'" return 1 @@ -1201,7 +1238,11 @@ cmd_import() { if [ "$ASSUME_YES" -ne 1 ]; then q="Import $PLAN_TOTAL_WF workflow(s) into $ORG" - [ "$PLAN_N_CREATE" -gt 0 ] && q="$q and create $PLAN_N_CREATE workflow group(s)" + if [ "$PLAN_N_CREATE" -gt 0 ]; then + local names_to_create="" + for grp in $PLAN_TO_CREATE; do names_to_create="$names_to_create${names_to_create:+, }$grp"; done + q="$q, creating workflow group(s) $names_to_create" + fi sg_interactive || die "no terminal to confirm the import — re-run with -y to import without a prompt" if ! sg_confirm "$q?" N; then sg_warn "import cancelled — nothing was changed in $ORG" @@ -1256,14 +1297,15 @@ cmd_import() { finish_line() { local open="${CHECKLIST_OPEN:-0}" took took="$(sg_fmt_secs $((SECONDS - RUN_T0)))" + local where + where="files: $(sg_rel "$EXPORT_DIR")/post-import-checklist.md, run-result.json, run-summary.md" if [ "$1" -ne 0 ]; then - sg_err "finished with failures in $took — fix what is reported above and re-run '$PROG import' (only failed or changed files are retried)" + sg_err "finished with failures in $took — fix what is reported above and re-run '$PROG import' (only failed or changed files are retried); $where" elif [ "$open" -gt 0 ]; then - sg_success "migration complete in $took — $open item(s) still need a human, see $(sg_rel "$EXPORT_DIR")/post-import-checklist.md" + sg_success "migration complete in $took — $open item(s) still need a human; $where" else - sg_success "migration complete in $took — nothing left to do by hand" + sg_success "migration complete in $took — nothing left to do by hand; $where" fi - sg_dim "run result: $(sg_rel "$EXPORT_DIR")/run-result.json, run-summary.md (markdown, e.g. for a CI job summary)" } # Single source of truth for shell completion (keep in sync with the parser below @@ -1578,6 +1620,7 @@ main() { if [ "$ENRICH_VARSETS" -eq 1 ]; then run_phase enrich "$apply_sha" cmd_enrich; fi run_phase convert "$(payload_sha)" cmd_convert run_phase validate "$(payload_sha)" cmd_validate + skipped_phases_flush RAN_APPLY=1 # the export ran, or its inputs (flags included) were unchanged cmd_import ;; diff --git a/scripts/validate_payload.sh b/scripts/validate_payload.sh index d8545ba..4f9d76d 100755 --- a/scripts/validate_payload.sh +++ b/scripts/validate_payload.sh @@ -31,7 +31,7 @@ set +o pipefail continue fi case "$line" in - *": pass") printf ' %s✓%s %s\n' "$C_GREEN" "$C_RESET" "$(basename "${line%: pass}")" >&2 ;; + *": pass") ;; # the orchestrator prints one ✓ line for all files *": fail: "*) printf ' %s✗%s %s: %s\n' "$C_RED$C_BOLD" "$C_RESET" "$(basename "${line%%: fail: *}")" "${line#*: fail: }" >&2 ;; *": error: "*) printf ' %s✗%s %s: %s\n' "$C_RED$C_BOLD" "$C_RESET" "$(basename "${line%%: error: *}")" "${line#*: error: }" >&2 ;; *) printf ' %s\n' "$line" >&2 ;; diff --git a/sg-migrate.sh b/sg-migrate.sh index 43c9e67..a2f02a1 100755 --- a/sg-migrate.sh +++ b/sg-migrate.sh @@ -119,7 +119,12 @@ if [ "$NATIVE" = "1" ] || ! command -v docker >/dev/null 2>&1; then [ "$NATIVE" = "1" ] || sg_warn "docker not found; running natively" exec "$SCRIPT_DIR/scripts/migrate.sh" ${ARGS[@]+"${ARGS[@]}"} fi -sg_dim "running in Docker ($IMAGE); pass --native to run on this machine instead" +# The --native hint only until the checkout has run state (a first run). +if [ -f "$SCRIPT_DIR/.sg/state.json" ]; then + sg_dim "running in Docker ($IMAGE)" +else + sg_dim "running in Docker ($IMAGE); pass --native to run on this machine instead" +fi if [ "$BUILD" = "1" ] || ! docker image inspect "$IMAGE" >/dev/null 2>&1; then sg_log "building image $IMAGE ..." From 6fd9cf980af5c082ba2cab87937c15aea2a1123e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adis=20Halilovi=C4=87?= <adis.halilovic@stackguardian.io> Date: Mon, 14 Sep 2026 19:21:40 +0200 Subject: [PATCH 71/71] fix: no stray '(3s)' line for the terraform init step --- scripts/tools.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/tools.sh b/scripts/tools.sh index a6855ed..b7c0c95 100755 --- a/scripts/tools.sh +++ b/scripts/tools.sh @@ -99,7 +99,8 @@ sg_spin_clear() { # sg_run_quiet <running-label> <done-label> <logfile> <cmd...> — run cmd with # stdout+stderr captured in logfile, showing "<running-label> (elapsed)" as a -# live line meanwhile, then a "<done-label> (took Ns)" log line. Returns the +# live line meanwhile, then a "<done-label> (took Ns)" log line — none when +# the done label is empty (a step not worth a line by default). Returns the # command's exit code; the caller decides what to do with the log. sg_run_quiet() { local running="$1" done_label="$2" log="$3" pid rc=0 t0=$SECONDS @@ -116,7 +117,7 @@ sg_run_quiet() { sg_log "$running..." fi wait "$pid" || rc=$? - [ "$rc" -eq 0 ] && sg_log "$done_label $C_DIM($(sg_fmt_secs $((SECONDS - t0))))$C_RESET" + [ "$rc" -eq 0 ] && [ -n "$done_label" ] && sg_log "$done_label $C_DIM($(sg_fmt_secs $((SECONDS - t0))))$C_RESET" return "$rc" }