From cbb90b6aed23033f487b27783c40b90b0cc8c7f7 Mon Sep 17 00:00:00 2001 From: Chenyue Ma Date: Tue, 8 Sep 2026 21:50:41 +0000 Subject: [PATCH] Add `pipelines describe` command (hidden) Show a pipeline's configuration and current state, plus the result of its most recent update. The pipeline can be addressed by bundle KEY (resolved via the bundle) or by PIPELINE_ID (a UUID), which is resolved directly and needs no bundle. Backed by the public Get and GetUpdate APIs, called in the same request- struct style as `pipelines run`. Hidden for now because output is limited to basic info; unhide once richer output (datasets, DAG, lineage) lands. Exports LooksLikeUUID from cmd/pipelines and reuses it in the workspace `stop` override instead of a second copy. Adds a describe-by-id acceptance test. Co-authored-by: Isaac --- .../pipelines/describe/basic/out.test.toml | 2 + .../pipelines/describe/basic/output.txt | 57 +++++++++ acceptance/pipelines/describe/basic/script | 5 + acceptance/pipelines/describe/basic/test.toml | 39 +++++++ cmd/pipelines/commands.go | 1 + cmd/pipelines/describe.go | 109 ++++++++++++++++++ cmd/pipelines/describe_test.go | 47 ++++++++ cmd/pipelines/templates.go | 56 +++++++++ cmd/workspace/pipelines/overrides.go | 10 +- cmd/workspace/pipelines/overrides_test.go | 15 --- 10 files changed, 317 insertions(+), 24 deletions(-) create mode 100644 acceptance/pipelines/describe/basic/out.test.toml create mode 100644 acceptance/pipelines/describe/basic/output.txt create mode 100644 acceptance/pipelines/describe/basic/script create mode 100644 acceptance/pipelines/describe/basic/test.toml create mode 100644 cmd/pipelines/describe.go create mode 100644 cmd/pipelines/describe_test.go delete mode 100644 cmd/workspace/pipelines/overrides_test.go diff --git a/acceptance/pipelines/describe/basic/out.test.toml b/acceptance/pipelines/describe/basic/out.test.toml new file mode 100644 index 00000000000..0938e678987 --- /dev/null +++ b/acceptance/pipelines/describe/basic/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/pipelines/describe/basic/output.txt b/acceptance/pipelines/describe/basic/output.txt new file mode 100644 index 00000000000..4fd8a78bac9 --- /dev/null +++ b/acceptance/pipelines/describe/basic/output.txt @@ -0,0 +1,57 @@ + +=== describe by pipeline id + +>>> [CLI] pipelines describe [UUID] +Pipeline: My Pipeline +ID: [UUID] +State: IDLE +Health: HEALTHY +Creator: alice@example.com +Run as: alice@example.com +Target: main.sales +Mode: Triggered, Development +Compute: Serverless +Channel: CURRENT + +Last run: + Update ID: upd-9 + State: COMPLETED + Started: [TIMESTAMP] + Refreshed: [orders, customers] + Cause: API_CALL + +=== json output + +>>> [CLI] pipelines describe [UUID] --output json +{ + "pipeline": { + "creator_user_name": "alice@example.com", + "health": "HEALTHY", + "latest_updates": [ + { + "update_id": "upd-9" + } + ], + "name": "My Pipeline", + "pipeline_id": "[UUID]", + "run_as_user_name": "alice@example.com", + "spec": { + "catalog": "main", + "channel": "CURRENT", + "development": true, + "schema": "sales", + "serverless": true + }, + "state": "IDLE" + }, + "last_update": { + "cause": "API_CALL", + "creation_time": [NUMID], + "refresh_selection": [ + "orders", + "customers" + ], + "state": "COMPLETED", + "update_id": "upd-9" + } +} diff --git a/acceptance/pipelines/describe/basic/script b/acceptance/pipelines/describe/basic/script new file mode 100644 index 00000000000..7f2fca19d3c --- /dev/null +++ b/acceptance/pipelines/describe/basic/script @@ -0,0 +1,5 @@ +title "describe by pipeline id\n" +trace $CLI pipelines describe 3fb8e5a1-0d2c-4a6b-9f1e-2c7d8e9f0a1b + +title "json output\n" +trace $CLI pipelines describe 3fb8e5a1-0d2c-4a6b-9f1e-2c7d8e9f0a1b --output json diff --git a/acceptance/pipelines/describe/basic/test.toml b/acceptance/pipelines/describe/basic/test.toml new file mode 100644 index 00000000000..d01dbd5f6bd --- /dev/null +++ b/acceptance/pipelines/describe/basic/test.toml @@ -0,0 +1,39 @@ +Cloud = false + +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] + +# describe by PIPELINE_ID fetches the pipeline, then its most recent update. +[[Server]] +Pattern = "GET /api/2.0/pipelines/{pipeline_id}" +Response.Body = ''' +{ + "pipeline_id": "3fb8e5a1-0d2c-4a6b-9f1e-2c7d8e9f0a1b", + "name": "My Pipeline", + "state": "IDLE", + "health": "HEALTHY", + "creator_user_name": "alice@example.com", + "run_as_user_name": "alice@example.com", + "latest_updates": [ { "update_id": "upd-9" } ], + "spec": { + "catalog": "main", + "schema": "sales", + "development": true, + "serverless": true, + "channel": "CURRENT" + } +} +''' + +[[Server]] +Pattern = "GET /api/2.0/pipelines/{pipeline_id}/updates/{update_id}" +Response.Body = ''' +{ + "update": { + "update_id": "upd-9", + "state": "COMPLETED", + "creation_time": 1640995200000, + "refresh_selection": ["orders", "customers"], + "cause": "API_CALL" + } +} +''' diff --git a/cmd/pipelines/commands.go b/cmd/pipelines/commands.go index 6a6b6931313..4b9b4babebe 100644 --- a/cmd/pipelines/commands.go +++ b/cmd/pipelines/commands.go @@ -19,6 +19,7 @@ func Commands() []*cobra.Command { dryRunCommand(), historyCommand(), logsCommand(), + describeCommand(), openCommand(), showCommand(), } diff --git a/cmd/pipelines/describe.go b/cmd/pipelines/describe.go new file mode 100644 index 00000000000..37b91835cb5 --- /dev/null +++ b/cmd/pipelines/describe.go @@ -0,0 +1,109 @@ +package pipelines + +import ( + "context" + "fmt" + "regexp" + + "github.com/databricks/cli/cmd/bundle/utils" + "github.com/databricks/cli/cmd/root" + "github.com/databricks/cli/libs/cmdctx" + "github.com/databricks/cli/libs/cmdio" + databricks "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/service/pipelines" + "github.com/spf13/cobra" +) + +var uuidRegex = regexp.MustCompile(`^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$`) + +// LooksLikeUUID reports whether s is a pipeline ID (UUID) rather than a bundle KEY. +func LooksLikeUUID(s string) bool { + return uuidRegex.MatchString(s) +} + +// pipelineDescribeData is the payload rendered by `pipelines describe`: the +// pipeline definition and current state, plus its most recent update (if any). +type pipelineDescribeData struct { + Key string `json:"key,omitempty"` + Pipeline *pipelines.GetPipelineResponse `json:"pipeline"` + LastUpdate *pipelines.UpdateInfo `json:"last_update,omitempty"` +} + +func describeCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "describe [flags] KEY|PIPELINE_ID", + Args: root.MaximumNArgs(1), + Short: "Show a summary of a pipeline", + Long: `Show a pipeline's configuration and current state, plus the result of its +most recent update. Identify the pipeline by bundle KEY, or by PIPELINE_ID +(a UUID) to describe any pipeline in the workspace without a bundle.`, + // Hidden while the command is still limited to basic pipeline info; + // unhide once richer output (datasets, DAG, lineage) is available. + Hidden: true, + } + + cmd.RunE = func(cmd *cobra.Command, args []string) error { + // A raw pipeline ID (UUID) is addressed directly, without a bundle, so + // describe works for pipelines not defined in the current bundle. + if len(args) == 1 && LooksLikeUUID(args[0]) { + if err := root.MustWorkspaceClient(cmd, args); err != nil { + return err + } + ctx := cmd.Context() + w := cmdctx.WorkspaceClient(ctx) + return describePipeline(ctx, w, "", args[0]) + } + + b, err := utils.ProcessBundle(cmd, utils.ProcessOptions{ + InitIDs: true, + }) + if err != nil { + return err + } + ctx := cmd.Context() + + key, err := resolvePipelineArgument(ctx, b, args) + if err != nil { + return err + } + + pipelineId, err := resolvePipelineIdFromKey(ctx, b, key) + if err != nil { + return err + } + + w := b.WorkspaceClient(ctx) + return describePipeline(ctx, w, key, pipelineId) + } + + return cmd +} + +// describePipeline fetches the pipeline and its most recent update (if any) and +// renders the summary. key is the bundle KEY when addressed that way, or empty +// when addressed by pipeline ID. +func describePipeline(ctx context.Context, w *databricks.WorkspaceClient, key, pipelineId string) error { + pipeline, err := w.Pipelines.Get(ctx, pipelines.GetPipelineRequest{PipelineId: pipelineId}) + if err != nil { + return fmt.Errorf("failed to get pipeline %s: %w", pipelineId, err) + } + + // Enrich with the most recent update, when the pipeline has run before. + // LatestUpdates is ordered newest-first. + var lastUpdate *pipelines.UpdateInfo + if len(pipeline.LatestUpdates) > 0 { + updateId := pipeline.LatestUpdates[0].UpdateId + resp, err := w.Pipelines.GetUpdate(ctx, pipelines.GetUpdateRequest{PipelineId: pipelineId, UpdateId: updateId}) + if err != nil { + return fmt.Errorf("failed to get latest update %s: %w", updateId, err) + } + lastUpdate = resp.Update + } + + data := pipelineDescribeData{ + Key: key, + Pipeline: pipeline, + LastUpdate: lastUpdate, + } + return cmdio.RenderWithTemplate(ctx, data, "", pipelineDescribeTemplate) +} diff --git a/cmd/pipelines/describe_test.go b/cmd/pipelines/describe_test.go new file mode 100644 index 00000000000..a96507eabe4 --- /dev/null +++ b/cmd/pipelines/describe_test.go @@ -0,0 +1,47 @@ +package pipelines + +import ( + "testing" + + "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/databricks-sdk-go/service/pipelines" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestLooksLikeUUID(t *testing.T) { + tests := []struct { + in string + want bool + }{ + {"3fb8e5a1-0d2c-4a6b-9f1e-2c7d8e9f0a1b", true}, + {"my_pipeline", false}, + {"my-pipeline", false}, + {"3FB8E5A1-0D2C-4A6B-9F1E-2C7D8E9F0A1B", false}, // uppercase is treated as a KEY + {"3fb8e5a1-0d2c-4a6b-9f1e-2c7d8e9f0a1", false}, // too short + {"", false}, + } + for _, tt := range tests { + assert.Equal(t, tt.want, LooksLikeUUID(tt.in), "looksLikeUUID(%q)", tt.in) + } +} + +// The populated render is golden-tested in acceptance/pipelines/describe/basic; +// this covers only the has-never-run branch. +func TestPipelineDescribeTemplateNoRuns(t *testing.T) { + data := pipelineDescribeData{ + Pipeline: &pipelines.GetPipelineResponse{ + Name: "Fresh Pipeline", + PipelineId: "def-456", + Spec: &pipelines.PipelineSpec{}, + }, + } + + ctx, out := cmdio.NewTestContextWithStdout(t.Context()) + require.NoError(t, cmdio.RenderWithTemplate(ctx, data, "", pipelineDescribeTemplate)) + + got := out.String() + assert.Contains(t, got, "Pipeline: Fresh Pipeline") + assert.Contains(t, got, "No runs yet.") + assert.NotContains(t, got, "Update ID:") +} diff --git a/cmd/pipelines/templates.go b/cmd/pipelines/templates.go index 3c471b366bb..f1bd2e334b3 100644 --- a/cmd/pipelines/templates.go +++ b/cmd/pipelines/templates.go @@ -72,6 +72,62 @@ Pipeline configurations for this update: {{- end }} ` +// pipelineDescribeTemplate renders the `pipelines describe` summary: pipeline +// configuration and state, followed by the most recent update (if any). +const pipelineDescribeTemplate = `Pipeline: {{if .Pipeline.Name}}{{.Pipeline.Name}}{{else}}{{.Pipeline.PipelineId}}{{end}} +{{- if .Key}} +Key: {{.Key}} +{{- end}} +ID: {{.Pipeline.PipelineId}} +{{- if .Pipeline.State}} +State: {{.Pipeline.State}} +{{- end}} +{{- if .Pipeline.Health}} +Health: {{.Pipeline.Health}} +{{- end}} +{{- if .Pipeline.CreatorUserName}} +Creator: {{.Pipeline.CreatorUserName}} +{{- end}} +{{- if .Pipeline.RunAsUserName}} +Run as: {{.Pipeline.RunAsUserName}} +{{- end}} +{{- with .Pipeline.Spec}} +{{- if .Catalog}} +Target: {{.Catalog}}{{if .Schema}}.{{.Schema}}{{end}} +{{- end}} +Mode: {{if .Continuous}}Continuous{{else}}Triggered{{end}}, {{if .Development}}Development{{else}}Production{{end}} +Compute: {{if .Serverless}}Serverless{{else if $.Pipeline.ClusterId}}Classic ({{$.Pipeline.ClusterId}}){{else}}Classic{{end}} +{{- if .Channel}} +Channel: {{.Channel}} +{{- end}} +{{- end}} + +Last run: +{{- if .LastUpdate}} + Update ID: {{.LastUpdate.UpdateId}} +{{- if .LastUpdate.State}} + State: {{.LastUpdate.State}} +{{- end}} +{{- if .LastUpdate.CreationTime}} + Started: {{.LastUpdate.CreationTime | pretty_UTC_date_from_millis}} +{{- end}} +{{- if .LastUpdate.FullRefresh}} + Full refresh: all tables +{{- end}} +{{- if .LastUpdate.RefreshSelection}} + Refreshed: [{{join .LastUpdate.RefreshSelection ", "}}] +{{- end}} +{{- if .LastUpdate.FullRefreshSelection}} + Full refreshed: [{{join .LastUpdate.FullRefreshSelection ", "}}] +{{- end}} +{{- if .LastUpdate.Cause}} + Cause: {{.LastUpdate.Cause}} +{{- end}} +{{- else}} + No runs yet. +{{- end}} +` + // progressEventsTemplate is the template for displaying progress events const progressEventsTemplate = `{{- if .ProgressEvents }} {{ printf "%-25s %s\n" "Run Phase" "Duration" }} diff --git a/cmd/workspace/pipelines/overrides.go b/cmd/workspace/pipelines/overrides.go index 08c36deabe2..02f1916e7d2 100644 --- a/cmd/workspace/pipelines/overrides.go +++ b/cmd/workspace/pipelines/overrides.go @@ -1,7 +1,6 @@ package pipelines import ( - "regexp" "slices" pipelinesCli "github.com/databricks/cli/cmd/pipelines" @@ -47,7 +46,7 @@ func init() { originalRunE := cmd.RunE cmd.RunE = func(cmd *cobra.Command, args []string) error { // For compatibility, if argument looks like pipeline ID, use API - if len(args) > 0 && looksLikeUUID(args[0]) { + if len(args) > 0 && pipelinesCli.LooksLikeUUID(args[0]) { return originalRunE(cmd, args) } // Looks like a bundle key or no args - use Lakeflow stop @@ -70,10 +69,3 @@ If there is only one pipeline in the bundle, KEY is optional. With a PIPELINE_ID: Stops the pipeline identified by the UUID using the API.` }) } - -var uuidRegex = regexp.MustCompile(`^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$`) - -// looksLikeUUID checks if a string matches the UUID format with lowercase hex digits -func looksLikeUUID(s string) bool { - return uuidRegex.MatchString(s) -} diff --git a/cmd/workspace/pipelines/overrides_test.go b/cmd/workspace/pipelines/overrides_test.go deleted file mode 100644 index 2e70cf4845b..00000000000 --- a/cmd/workspace/pipelines/overrides_test.go +++ /dev/null @@ -1,15 +0,0 @@ -package pipelines - -import ( - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestLooksLikeUUID(t *testing.T) { - assert.True(t, looksLikeUUID("a12cd3e4-0ab1-1abc-1a2b-1a2bcd3e4f05")) -} - -func TestLooksLikeUUID_resourceName(t *testing.T) { - assert.False(t, looksLikeUUID("my-pipeline-key")) -}