From 55902586c920f827e7d5883f940ea243d2a1a7b6 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Fri, 24 Jul 2026 01:41:10 +0200 Subject: [PATCH 001/125] bundle: record and read deployment state via DMS with server-generated IDs Wire the direct engine into the Deployment Metadata Service (DMS) so that a `record_deployment_history`-enabled bundle records each deploy/destroy as a version and can read its resource state back from DMS. The deployment ID is now assigned by the server: the first deploy calls CreateDeployment with an empty ID, reads the assigned ID back from the response, and persists it in the direct-engine state header (Header.DeploymentID). Later deploys pass the stored ID back, so a bundle maps one-to-one to a DMS deployment even after the local cache is deleted (the ID rides along in the workspace-synced state file). - libs/dms: Recorder creates the deployment (server-assigned ID) + version, heartbeats the lease, completes it, and deletes the deployment on destroy. - bundle/direct: operationRecorder reports each applied resource operation; the wire resource_key drops the CLI-internal "resources." prefix. - bundle/direct/dstate: Open takes a DMS client and overlays DMS resource state when DMS holds a successful version; deployment ID persisted in the header. - bundle/phases: create the version after plan approval, complete it under the lock, record operations during apply. - libs/testserver: stateful fake DMS (deployments/versions/operations/resources) with server-generated IDs; acceptance test covers deploy, cache-loss redeploy, and destroy. Co-authored-by: Isaac --- acceptance/bundle/dms/record/databricks.yml | 10 + acceptance/bundle/dms/record/out.test.toml | 3 + acceptance/bundle/dms/record/output.txt | 140 +++++++++++ acceptance/bundle/dms/record/script | 15 ++ acceptance/bundle/dms/test.toml | 13 + bundle/configsync/diff.go | 2 +- bundle/configsync/variables.go | 2 +- bundle/direct/bind.go | 12 +- bundle/direct/bundle_apply.go | 13 + bundle/direct/dstate/dms.go | 104 ++++++++ bundle/direct/dstate/state.go | 43 +++- bundle/direct/dstate/state_test.go | 45 +++- bundle/direct/oprecorder.go | 107 ++++++++ bundle/direct/oprecorder_test.go | 84 +++++++ bundle/direct/pkg.go | 5 + bundle/phases/deploy.go | 32 +++ bundle/phases/destroy.go | 23 ++ bundle/phases/dms.go | 37 +++ cmd/bundle/generate/dashboard.go | 2 +- cmd/bundle/generate/genie_space.go | 2 +- cmd/bundle/utils/process.go | 12 +- libs/dms/recorder.go | 255 ++++++++++++++++++++ libs/dms/recorder_test.go | 165 +++++++++++++ libs/testserver/bundle.go | 228 +++++++++++++++++ libs/testserver/fake_workspace.go | 5 + libs/testserver/handlers.go | 29 +++ 26 files changed, 1363 insertions(+), 25 deletions(-) create mode 100644 acceptance/bundle/dms/record/databricks.yml create mode 100644 acceptance/bundle/dms/record/out.test.toml create mode 100644 acceptance/bundle/dms/record/output.txt create mode 100644 acceptance/bundle/dms/record/script create mode 100644 acceptance/bundle/dms/test.toml create mode 100644 bundle/direct/dstate/dms.go create mode 100644 bundle/direct/oprecorder.go create mode 100644 bundle/direct/oprecorder_test.go create mode 100644 bundle/phases/dms.go create mode 100644 libs/dms/recorder.go create mode 100644 libs/dms/recorder_test.go create mode 100644 libs/testserver/bundle.go diff --git a/acceptance/bundle/dms/record/databricks.yml b/acceptance/bundle/dms/record/databricks.yml new file mode 100644 index 00000000000..b20e6274310 --- /dev/null +++ b/acceptance/bundle/dms/record/databricks.yml @@ -0,0 +1,10 @@ +bundle: + name: dms-record + +experimental: + record_deployment_history: true + +resources: + jobs: + foo: + name: foo diff --git a/acceptance/bundle/dms/record/out.test.toml b/acceptance/bundle/dms/record/out.test.toml new file mode 100644 index 00000000000..e90b6d5d1ba --- /dev/null +++ b/acceptance/bundle/dms/record/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/dms/record/output.txt b/acceptance/bundle/dms/record/output.txt new file mode 100644 index 00000000000..5c0317f38cc --- /dev/null +++ b/acceptance/bundle/dms/record/output.txt @@ -0,0 +1,140 @@ + +=== Deploy: the server assigns the deployment ID, and a version + create operation are recorded +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-record/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> print_requests.py //api/2.0/bundle --sort +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments", + "body": { + "target_name": "default" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[UUID]/versions", + "q": { + "version_id": "1" + }, + "body": { + "cli_version": "[CLI_VERSION]", + "target_name": "default", + "version_type": "VERSION_TYPE_DEPLOY" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[UUID]/versions/1/complete", + "body": { + "completion_reason": "VERSION_COMPLETE_SUCCESS" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[UUID]/versions/1/operations", + "q": { + "resource_key": "jobs.foo" + }, + "body": { + "action_type": "OPERATION_ACTION_TYPE_CREATE", + "resource_id": "[NUMID]", + "resource_key": "jobs.foo", + "state": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-record/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "foo", + "queue": { + "enabled": true + } + }, + "status": "OPERATION_STATUS_SUCCEEDED" + } +} + +=== The server-assigned deployment ID is persisted in the local state file +>>> jq .deployment_id .databricks/bundle/default/resources.json +"[UUID]" + +=== Redeploy after deleting the local cache: the deployment ID is recovered from remote state, the same deployment is reused, and the version increments (no new CreateDeployment) +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-record/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> print_requests.py //api/2.0/bundle --sort +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[UUID]/versions", + "q": { + "version_id": "2" + }, + "body": { + "cli_version": "[CLI_VERSION]", + "target_name": "default", + "version_type": "VERSION_TYPE_DEPLOY" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[UUID]/versions/2/complete", + "body": { + "completion_reason": "VERSION_COMPLETE_SUCCESS" + } +} + +=== Destroy: a destroy version and delete operation are recorded, then the deployment is deleted +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.jobs.foo + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/dms-record/default + +Deleting files... +Destroy complete! + +>>> print_requests.py //api/2.0/bundle --sort +{ + "method": "DELETE", + "path": "/api/2.0/bundle/deployments/[UUID]" +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[UUID]/versions", + "q": { + "version_id": "3" + }, + "body": { + "cli_version": "[CLI_VERSION]", + "target_name": "default", + "version_type": "VERSION_TYPE_DESTROY" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[UUID]/versions/3/complete", + "body": { + "completion_reason": "VERSION_COMPLETE_SUCCESS" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[UUID]/versions/3/operations", + "q": { + "resource_key": "jobs.foo" + }, + "body": { + "action_type": "OPERATION_ACTION_TYPE_DELETE", + "resource_key": "jobs.foo", + "status": "OPERATION_STATUS_SUCCEEDED" + } +} diff --git a/acceptance/bundle/dms/record/script b/acceptance/bundle/dms/record/script new file mode 100644 index 00000000000..ab59d38afb4 --- /dev/null +++ b/acceptance/bundle/dms/record/script @@ -0,0 +1,15 @@ +title "Deploy: the server assigns the deployment ID, and a version + create operation are recorded" +trace $CLI bundle deploy +trace print_requests.py //api/2.0/bundle --sort + +title "The server-assigned deployment ID is persisted in the local state file" +trace jq .deployment_id .databricks/bundle/default/resources.json + +title "Redeploy after deleting the local cache: the deployment ID is recovered from remote state, the same deployment is reused, and the version increments (no new CreateDeployment)" +rm -rf .databricks +trace $CLI bundle deploy +trace print_requests.py //api/2.0/bundle --sort + +title "Destroy: a destroy version and delete operation are recorded, then the deployment is deleted" +trace $CLI bundle destroy --auto-approve +trace print_requests.py //api/2.0/bundle --sort diff --git a/acceptance/bundle/dms/test.toml b/acceptance/bundle/dms/test.toml new file mode 100644 index 00000000000..24ce9756629 --- /dev/null +++ b/acceptance/bundle/dms/test.toml @@ -0,0 +1,13 @@ +Local = true +Cloud = false + +# Deployment Metadata Service (DMS) recording is only meaningful in the direct +# engine, where the deployment ID is stored in and read from the direct-engine +# state. +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] + +RecordRequests = true + +Ignore = [ + '.databricks', +] diff --git a/bundle/configsync/diff.go b/bundle/configsync/diff.go index ea45903508b..ca5b2c9410b 100644 --- a/bundle/configsync/diff.go +++ b/bundle/configsync/diff.go @@ -149,7 +149,7 @@ func OpenDeploymentState(ctx context.Context, b *bundle.Bundle, engine engine.En deployBundle := &direct.DeploymentBundle{} _, statePath := b.StateFilenameConfigSnapshot(ctx) - if err := deployBundle.StateDB.Open(ctx, statePath, dstate.WithRecovery(true), dstate.WithWrite(false)); err != nil { + if err := deployBundle.StateDB.Open(ctx, statePath, dstate.WithRecovery(true), dstate.WithWrite(false), nil); err != nil { return nil, fmt.Errorf("failed to open state: %w", err) } return deployBundle, nil diff --git a/bundle/configsync/variables.go b/bundle/configsync/variables.go index 055a47dc934..433b607a037 100644 --- a/bundle/configsync/variables.go +++ b/bundle/configsync/variables.go @@ -147,7 +147,7 @@ func resourceIDLookup(ctx context.Context, b *bundle.Bundle) func(string) string } _, statePath := b.StateFilenameConfigSnapshot(ctx) db := &dstate.DeploymentState{} - if err := db.Open(ctx, statePath, dstate.WithRecovery(false), dstate.WithWrite(false)); err != nil { + if err := db.Open(ctx, statePath, dstate.WithRecovery(false), dstate.WithWrite(false), nil); err != nil { log.Debugf(ctx, "variable restoration: failed to open state DB at %s: %v", statePath, err) return nil } diff --git a/bundle/direct/bind.go b/bundle/direct/bind.go index 9760ce95666..ec910b2734e 100644 --- a/bundle/direct/bind.go +++ b/bundle/direct/bind.go @@ -62,7 +62,7 @@ type BindResult struct { func (b *DeploymentBundle) Bind(ctx context.Context, client *databricks.WorkspaceClient, configRoot *config.Root, statePath, resourceKey, resourceID string) (*BindResult, error) { // Check if the resource is already managed (bound to a different ID) var checkStateDB dstate.DeploymentState - if err := checkStateDB.Open(ctx, statePath, dstate.WithRecovery(true), dstate.WithWrite(false)); err == nil { + if err := checkStateDB.Open(ctx, statePath, dstate.WithRecovery(true), dstate.WithWrite(false), nil); err == nil { existingID := checkStateDB.GetResourceID(resourceKey) if _, err := checkStateDB.Finalize(ctx); err != nil { log.Warnf(ctx, "failed to finalize state: %v", err) @@ -86,7 +86,7 @@ func (b *DeploymentBundle) Bind(ctx context.Context, client *databricks.Workspac } // Open temp state - err := b.StateDB.Open(ctx, tmpStatePath, dstate.WithRecovery(false), dstate.WithWrite(true)) + err := b.StateDB.Open(ctx, tmpStatePath, dstate.WithRecovery(false), dstate.WithWrite(true), nil) if err != nil { os.Remove(tmpStatePath) return nil, err @@ -109,7 +109,7 @@ func (b *DeploymentBundle) Bind(ctx context.Context, client *databricks.Workspac log.Infof(ctx, "Bound %s to id=%s (in temp state)", resourceKey, resourceID) // First plan + update: populate state with resolved config - err = b.StateDB.Open(ctx, tmpStatePath, dstate.WithRecovery(true), dstate.WithWrite(false)) + err = b.StateDB.Open(ctx, tmpStatePath, dstate.WithRecovery(true), dstate.WithWrite(false), nil) if err != nil { os.Remove(tmpStatePath) return nil, err @@ -145,7 +145,7 @@ func (b *DeploymentBundle) Bind(ctx context.Context, client *databricks.Workspac } } - err = b.StateDB.Open(ctx, tmpStatePath, dstate.WithRecovery(true), dstate.WithWrite(true)) + err = b.StateDB.Open(ctx, tmpStatePath, dstate.WithRecovery(true), dstate.WithWrite(true), nil) if err != nil { os.Remove(tmpStatePath) return nil, err @@ -165,7 +165,7 @@ func (b *DeploymentBundle) Bind(ctx context.Context, client *databricks.Workspac } // Second plan: this is the plan to present to the user (change between remote resource and config) - err = b.StateDB.Open(ctx, tmpStatePath, dstate.WithRecovery(true), dstate.WithWrite(false)) + err = b.StateDB.Open(ctx, tmpStatePath, dstate.WithRecovery(true), dstate.WithWrite(false), nil) if err != nil { os.Remove(tmpStatePath) return nil, err @@ -215,7 +215,7 @@ func (result *BindResult) Cancel() { // Unbind removes a resource from direct engine state without deleting // the workspace resource. Also removes associated permissions/grants entries. func (b *DeploymentBundle) Unbind(ctx context.Context, statePath, resourceKey string) error { - err := b.StateDB.Open(ctx, statePath, dstate.WithRecovery(true), dstate.WithWrite(true)) + err := b.StateDB.Open(ctx, statePath, dstate.WithRecovery(true), dstate.WithWrite(true), nil) if err != nil { return err } diff --git a/bundle/direct/bundle_apply.go b/bundle/direct/bundle_apply.go index c4178c4e601..afef2367e5b 100644 --- a/bundle/direct/bundle_apply.go +++ b/bundle/direct/bundle_apply.go @@ -88,6 +88,11 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa logdiag.LogError(ctx, fmt.Errorf("%s: %w", errorPrefix, err)) return false } + // Record the delete with DMS. State is nil: the resource is gone. + if err := b.recordOperation(ctx, resourceKey, action, "", nil); err != nil { + logdiag.LogError(ctx, fmt.Errorf("%s: %w", errorPrefix, err)) + return false + } return true } @@ -116,6 +121,14 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa logdiag.LogError(ctx, fmt.Errorf("%s: %w", errorPrefix, err)) return false } + + // Record the operation with DMS. The resource ID and applied config + // (sv.Value) come from the write just performed; GetResourceID reads + // the ID assigned by Deploy. + if err := b.recordOperation(ctx, resourceKey, action, b.StateDB.GetResourceID(resourceKey), sv.Value); err != nil { + logdiag.LogError(ctx, fmt.Errorf("%s: %w", errorPrefix, err)) + return false + } } // TODO: Note, we only really need remote state if there are remote references. diff --git a/bundle/direct/dstate/dms.go b/bundle/direct/dstate/dms.go new file mode 100644 index 00000000000..1d19d1fe214 --- /dev/null +++ b/bundle/direct/dstate/dms.go @@ -0,0 +1,104 @@ +package dstate + +import ( + "context" + "encoding/json" + "errors" + "fmt" + + "github.com/databricks/databricks-sdk-go/apierr" + "github.com/databricks/databricks-sdk-go/service/bundledeployments" +) + +// overlayDMSState replaces the file-derived resource state with the state +// recorded in the deployment metadata service (DMS), when DMS owns this +// deployment. Once DMS is authoritative its resource set is trusted even when +// empty (a successful deploy with no resources); the file's resources are only +// used when DMS has no successful version, or when the user opts out of +// recording deployment history. The caller holds db.mu and has already +// populated db.Data from the file, including the DeploymentID. +func (db *DeploymentState) overlayDMSState(ctx context.Context, client bundledeployments.BundleDeploymentsInterface) error { + authoritative, err := deploymentHasSuccessfulVersion(ctx, client, db.Data.DeploymentID) + if err != nil { + return err + } + if !authoritative { + // DMS has no completed version for this deployment: a prior direct deploy + // that has not yet successfully recorded to DMS. Keep the file state. + return nil + } + + resources, err := fetchDeploymentResources(ctx, client, db.Data.DeploymentID) + if err != nil { + return err + } + + db.Data.State = resources + db.stateIDs = make(map[string]string, len(resources)) + for key, entry := range resources { + db.stateIDs[key] = entry.ID + } + return nil +} + +// deploymentHasSuccessfulVersion reports whether DMS holds a successfully +// completed version for the deployment. It is the signal that DMS owns the +// state: if the deployment was never recorded to DMS, or its initial DMS deploy +// did not complete successfully, DMS state is absent or partial and Open keeps +// the local file's resources instead. +func deploymentHasSuccessfulVersion(ctx context.Context, client bundledeployments.BundleDeploymentsInterface, deploymentID string) (bool, error) { + // Versions are listed newest-first and fetched page by page, and we stop at + // the first successful one, so a deployment with a long version history does + // not require reading the whole list (typically just the first page). + it := client.ListVersions(ctx, bundledeployments.ListVersionsRequest{ + Parent: "deployments/" + deploymentID, + }) + for it.HasNext(ctx) { + v, err := it.Next(ctx) + if err != nil { + // A deployment that was never recorded to DMS is not an error here: it + // just means DMS is not (yet) the source of truth. + if errors.Is(err, apierr.ErrNotFound) { + return false, nil + } + return false, fmt.Errorf("listing versions from deployment metadata service: %w", err) + } + if v.Status == bundledeployments.VersionStatusVersionStatusCompleted && + v.CompletionReason == bundledeployments.VersionCompleteVersionCompleteSuccess { + return true, nil + } + } + return false, nil +} + +// fetchDeploymentResources lists every resource recorded for the deployment in +// DMS and maps them into state entries keyed by the fully-qualified resource key. +func fetchDeploymentResources(ctx context.Context, client bundledeployments.BundleDeploymentsInterface, deploymentID string) (map[string]ResourceEntry, error) { + it := client.ListResources(ctx, bundledeployments.ListResourcesRequest{ + Parent: "deployments/" + deploymentID, + }) + + out := make(map[string]ResourceEntry) + for it.HasNext(ctx) { + res, err := it.Next(ctx) + if err != nil { + return nil, fmt.Errorf("listing resources from deployment metadata service: %w", err) + } + + // DMS reports resource keys without the "resources." prefix (e.g. + // "jobs.foo"), but the state DB keys are fully qualified + // ("resources.jobs.foo"), so prepend it here. + key := "resources." + res.ResourceKey + + var state json.RawMessage + if res.State != nil { + state = *res.State + } + + out[key] = ResourceEntry{ + ID: res.ResourceId, + State: state, + } + } + return out, nil +} diff --git a/bundle/direct/dstate/state.go b/bundle/direct/dstate/state.go index f6c8fc8ba3c..64fc050bdc0 100644 --- a/bundle/direct/dstate/state.go +++ b/bundle/direct/dstate/state.go @@ -19,6 +19,7 @@ import ( "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/structs/structwalk" + "github.com/databricks/databricks-sdk-go/service/bundledeployments" "github.com/google/uuid" ) @@ -80,6 +81,13 @@ type Header struct { Lineage string `json:"lineage"` Serial int `json:"serial"` + // DeploymentID is the ID the deployment metadata service (DMS) assigned to + // this deployment. Unlike Lineage (a locally generated identifier for the + // state file), it is minted server-side by CreateDeployment and stored here so + // later deploys can find the same DMS deployment record and read its state. + // Empty/omitted until the bundle first records to DMS. + DeploymentID string `json:"deployment_id,omitempty"` + // Features maps each feature flag this state depends on to a (currently empty) // value. This CLI writes no features; it only reads the field to detect a state // that depends on features it lacks and refuse it (see migrateState). It is a @@ -209,6 +217,25 @@ func (db *DeploymentState) GetOrInitLineage() string { return db.Data.Lineage } +// GetDeploymentID returns the DMS deployment ID recorded in the state, or an +// empty string if this bundle has not yet recorded a deployment to DMS. +func (db *DeploymentState) GetDeploymentID() string { + db.mu.Lock() + defer db.mu.Unlock() + return db.Data.DeploymentID +} + +// SetDeploymentID stores the DMS-assigned deployment ID in the in-memory state +// header. It is set during deploy, after CreateDeployment returns the +// server-generated ID, and persisted to the state file by Finalize. Storing it +// on db.Data (not the WAL header, which is written before the ID is known) +// means the subsequent state write carries it forward. +func (db *DeploymentState) SetDeploymentID(id string) { + db.mu.Lock() + defer db.mu.Unlock() + db.Data.DeploymentID = id +} + type ( // If true, then Open reads the WAL and merges it in the state. If false, and WAL is present, Open returns an error. WithRecovery bool @@ -218,7 +245,15 @@ type ( WithWrite bool ) -func (db *DeploymentState) Open(ctx context.Context, path string, withRecovery WithRecovery, withWrite WithWrite) error { +// Open reads the deployment state from disk (and recovers the WAL when +// withRecovery is set). When dmsClient is non-nil, the deployment metadata +// service is the source of truth for resource state: if DMS holds a +// successfully completed version for this deployment, the resources read from +// the file are replaced with the ones recorded in DMS. The local identity +// (lineage, serial, and deployment ID) always comes from the file, since that +// is what the write path increments and carries forward. A nil dmsClient keeps +// the behavior file-only. +func (db *DeploymentState) Open(ctx context.Context, path string, withRecovery WithRecovery, withWrite WithWrite, dmsClient bundledeployments.BundleDeploymentsInterface) error { db.mu.Lock() defer db.mu.Unlock() @@ -266,6 +301,12 @@ func (db *DeploymentState) Open(ctx context.Context, path string, withRecovery W return fmt.Errorf("migrating state %s: %w", path, err) } + if dmsClient != nil && db.Data.DeploymentID != "" { + if err := db.overlayDMSState(ctx, dmsClient); err != nil { + return err + } + } + if withWrite { if err := os.MkdirAll(filepath.Dir(walPath), 0o755); err != nil { return fmt.Errorf("failed to create state directory: %w", err) diff --git a/bundle/direct/dstate/state_test.go b/bundle/direct/dstate/state_test.go index 11589944472..e95ad1b0224 100644 --- a/bundle/direct/dstate/state_test.go +++ b/bundle/direct/dstate/state_test.go @@ -20,24 +20,43 @@ func TestOpenSaveFinalizeRoundTrip(t *testing.T) { path := filepath.Join(t.TempDir(), "state.json") var db DeploymentState - require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true))) + require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil)) require.NoError(t, db.SaveState("jobs.my_job", "123", map[string]string{"key": "val"}, nil)) mustFinalize(t, &db) // Re-open and verify persisted data. var db2 DeploymentState - require.NoError(t, db2.Open(t.Context(), path, WithRecovery(false), WithWrite(false))) + require.NoError(t, db2.Open(t.Context(), path, WithRecovery(false), WithWrite(false), nil)) assert.Equal(t, 1, db2.Data.Serial) assert.Equal(t, "123", db2.GetResourceID("jobs.my_job")) mustFinalize(t, &db2) } +func TestDeploymentIDPersistsAcrossOpen(t *testing.T) { + path := filepath.Join(t.TempDir(), "state.json") + + var db DeploymentState + require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil)) + assert.Empty(t, db.GetDeploymentID()) + + // The deployment ID is set during deploy (after CreateDeployment) and + // persisted by Finalize even though it is not part of the WAL header. + db.SetDeploymentID("server-assigned-id") + require.NoError(t, db.SaveState("jobs.my_job", "123", map[string]string{}, nil)) + mustFinalize(t, &db) + + var reopened DeploymentState + require.NoError(t, reopened.Open(t.Context(), path, WithRecovery(false), WithWrite(false), nil)) + assert.Equal(t, "server-assigned-id", reopened.GetDeploymentID()) + mustFinalize(t, &reopened) +} + func TestFinalizeWithNoEntriesDoesNotWriteStateFile(t *testing.T) { path := filepath.Join(t.TempDir(), "state.json") var db DeploymentState - require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true))) + require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil)) mustFinalize(t, &db) _, err := os.Stat(path) @@ -93,10 +112,10 @@ func TestPanicOnDoubleOpen(t *testing.T) { path := filepath.Join(t.TempDir(), "state.json") var db DeploymentState - require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true))) + require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil)) assert.Panics(t, func() { - _ = db.Open(t.Context(), path, WithRecovery(true), WithWrite(true)) + _ = db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil) }) mustFinalize(t, &db) } @@ -107,12 +126,12 @@ func TestHeaderOnlyWALRecoveryDoesNotAdvanceSerial(t *testing.T) { // Commit serial 1 with one resource. var db DeploymentState - require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true))) + require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil)) require.NoError(t, db.SaveState("jobs.my_job", "123", map[string]string{}, nil)) mustFinalize(t, &db) var committed DeploymentState - require.NoError(t, committed.Open(t.Context(), path, WithRecovery(false), WithWrite(false))) + require.NoError(t, committed.Open(t.Context(), path, WithRecovery(false), WithWrite(false), nil)) lineage := committed.Data.Lineage require.Equal(t, 1, committed.Data.Serial) mustFinalize(t, &committed) @@ -128,7 +147,7 @@ func TestHeaderOnlyWALRecoveryDoesNotAdvanceSerial(t *testing.T) { require.NoError(t, os.WriteFile(walPath, append(headerLine, '\n'), 0o600)) var recovered DeploymentState - require.NoError(t, recovered.Open(t.Context(), path, WithRecovery(true), WithWrite(false))) + require.NoError(t, recovered.Open(t.Context(), path, WithRecovery(true), WithWrite(false), nil)) assert.Equal(t, 1, recovered.Data.Serial) assert.Equal(t, "123", recovered.GetResourceID("jobs.my_job")) assert.NoFileExists(t, walPath) @@ -171,17 +190,17 @@ func TestDeleteState(t *testing.T) { path := filepath.Join(t.TempDir(), "state.json") var db DeploymentState - require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true))) + require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil)) require.NoError(t, db.SaveState("jobs.my_job", "123", map[string]string{}, nil)) mustFinalize(t, &db) var db2 DeploymentState - require.NoError(t, db2.Open(t.Context(), path, WithRecovery(true), WithWrite(true))) + require.NoError(t, db2.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil)) require.NoError(t, db2.DeleteState("jobs.my_job")) mustFinalize(t, &db2) var db3 DeploymentState - require.NoError(t, db3.Open(t.Context(), path, WithRecovery(false), WithWrite(false))) + require.NoError(t, db3.Open(t.Context(), path, WithRecovery(false), WithWrite(false), nil)) assert.Equal(t, 2, db3.Data.Serial) assert.Empty(t, db3.GetResourceID("jobs.my_job")) mustFinalize(t, &db3) @@ -193,7 +212,7 @@ func TestGetOrInitLineageReadableBeforeWriteAndPersisted(t *testing.T) { // Fresh state opened read-only, as the deploy does before planning: no // lineage yet. var db DeploymentState - require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(false))) + require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(false), nil)) require.Empty(t, db.Data.Lineage) // GetOrInitLineage initializes the lineage and makes it readable before any @@ -210,7 +229,7 @@ func TestGetOrInitLineageReadableBeforeWriteAndPersisted(t *testing.T) { // Re-open: the persisted lineage matches the one read before the write. var reopened DeploymentState - require.NoError(t, reopened.Open(t.Context(), path, WithRecovery(false), WithWrite(false))) + require.NoError(t, reopened.Open(t.Context(), path, WithRecovery(false), WithWrite(false), nil)) assert.Equal(t, lineage, reopened.Data.Lineage) mustFinalize(t, &reopened) } diff --git a/bundle/direct/oprecorder.go b/bundle/direct/oprecorder.go new file mode 100644 index 00000000000..467f8ac648c --- /dev/null +++ b/bundle/direct/oprecorder.go @@ -0,0 +1,107 @@ +package direct + +import ( + "context" + "encoding/json" + "fmt" + "strings" + + "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/databricks-sdk-go/service/bundledeployments" +) + +// opRecorder records a resource operation with the deployment metadata service +// (DMS) after it has been applied to the workspace. state is the serialized +// local config after the operation and must be nil for delete operations. +type opRecorder interface { + record(ctx context.Context, resourceKey string, action deployplan.ActionType, resourceID string, state any) error +} + +// recordOperation reports an applied resource operation to DMS. It is a no-op +// unless the bundle opted into recording deployment history (OpRec is set). +// state is the serialized local config after the operation and must be nil for +// delete operations. +func (b *DeploymentBundle) recordOperation(ctx context.Context, resourceKey string, action deployplan.ActionType, resourceID string, state any) error { + if b.OpRec == nil { + return nil + } + return b.OpRec.record(ctx, resourceKey, action, resourceID, state) +} + +// operationRecorder records operations via the DMS CreateOperation API. +type operationRecorder struct { + client bundledeployments.BundleDeploymentsInterface + // parent is the version the operations are recorded under, formatted as + // "deployments/{deployment_id}/versions/{version_id}". + parent string +} + +// NewOperationRecorder returns an opRecorder backed by the DMS CreateOperation +// API. deploymentID and version identify the deployment version assigned by DMS +// that the operations are recorded under. +func NewOperationRecorder(client bundledeployments.BundleDeploymentsInterface, deploymentID string, version int64) opRecorder { + return &operationRecorder{ + client: client, + parent: fmt.Sprintf("deployments/%s/versions/%d", deploymentID, version), + } +} + +func (r *operationRecorder) record(ctx context.Context, resourceKey string, action deployplan.ActionType, resourceID string, state any) error { + actionType, err := deployActionToSDK(action) + if err != nil { + return err + } + + // DMS resource keys are unprefixed (e.g. "jobs.foo"), while the CLI's state + // keys carry a leading "resources." (e.g. "resources.jobs.foo"). Strip it on + // the way out; the read path re-adds it (see dstate.fetchDeploymentResources). + dmsKey := strings.TrimPrefix(resourceKey, "resources.") + + op := bundledeployments.Operation{ + ActionType: actionType, + ResourceId: resourceID, + ResourceKey: dmsKey, + Status: bundledeployments.OperationStatusOperationStatusSucceeded, + } + + // The DMS Operation.State field carries the serialized config so the backend + // can serve it as resource state. It is intentionally left unset for delete, + // where the resource no longer exists. + if state != nil { + raw, err := json.Marshal(state) + if err != nil { + return fmt.Errorf("serializing state: %w", err) + } + msg := json.RawMessage(raw) + op.State = &msg + } + + _, err = r.client.CreateOperation(ctx, bundledeployments.CreateOperationRequest{ + Parent: r.parent, + ResourceKey: dmsKey, + Operation: op, + }) + return err +} + +// deployActionToSDK maps a deployplan action to its DMS operation action type. +// Only actions that mutate a resource are recordable; Skip and Undefined never +// reach a recorder and are rejected rather than silently coerced. +func deployActionToSDK(a deployplan.ActionType) (bundledeployments.OperationActionType, error) { + switch a { + case deployplan.Create: + return bundledeployments.OperationActionTypeOperationActionTypeCreate, nil + case deployplan.Update: + return bundledeployments.OperationActionTypeOperationActionTypeUpdate, nil + case deployplan.UpdateWithID: + return bundledeployments.OperationActionTypeOperationActionTypeUpdateWithId, nil + case deployplan.Recreate: + return bundledeployments.OperationActionTypeOperationActionTypeRecreate, nil + case deployplan.Resize: + return bundledeployments.OperationActionTypeOperationActionTypeResize, nil + case deployplan.Delete: + return bundledeployments.OperationActionTypeOperationActionTypeDelete, nil + default: + return "", fmt.Errorf("cannot record operation: unsupported action %q", a) + } +} diff --git a/bundle/direct/oprecorder_test.go b/bundle/direct/oprecorder_test.go new file mode 100644 index 00000000000..56d860de3e6 --- /dev/null +++ b/bundle/direct/oprecorder_test.go @@ -0,0 +1,84 @@ +package direct + +import ( + "context" + "testing" + + "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/databricks-sdk-go/service/bundledeployments" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type fakeOpClient struct { + bundledeployments.BundleDeploymentsInterface + requests []bundledeployments.CreateOperationRequest +} + +func (f *fakeOpClient) CreateOperation(ctx context.Context, req bundledeployments.CreateOperationRequest) (*bundledeployments.Operation, error) { + f.requests = append(f.requests, req) + return &bundledeployments.Operation{}, nil +} + +func TestOperationRecorderStripsResourcePrefix(t *testing.T) { + f := &fakeOpClient{} + r := NewOperationRecorder(f, "dep-1", 2) + + err := r.record(t.Context(), "resources.jobs.foo", deployplan.Create, "job-123", map[string]string{"name": "foo"}) + require.NoError(t, err) + + require.Len(t, f.requests, 1) + req := f.requests[0] + // The wire key drops the CLI-internal "resources." prefix, both in the query + // param and the operation body. + assert.Equal(t, "jobs.foo", req.ResourceKey) + assert.Equal(t, "jobs.foo", req.Operation.ResourceKey) + assert.Equal(t, "deployments/dep-1/versions/2", req.Parent) + assert.Equal(t, bundledeployments.OperationActionTypeOperationActionTypeCreate, req.Operation.ActionType) + assert.Equal(t, "job-123", req.Operation.ResourceId) + require.NotNil(t, req.Operation.State) +} + +func TestOperationRecorderDeleteHasNoState(t *testing.T) { + f := &fakeOpClient{} + r := NewOperationRecorder(f, "dep-1", 3) + + err := r.record(t.Context(), "resources.jobs.foo", deployplan.Delete, "", nil) + require.NoError(t, err) + + require.Len(t, f.requests, 1) + assert.Equal(t, bundledeployments.OperationActionTypeOperationActionTypeDelete, f.requests[0].Operation.ActionType) + // Delete operations carry no serialized state. + assert.Nil(t, f.requests[0].Operation.State) +} + +func TestDeployActionToSDK(t *testing.T) { + cases := []struct { + action deployplan.ActionType + want bundledeployments.OperationActionType + }{ + {deployplan.Create, bundledeployments.OperationActionTypeOperationActionTypeCreate}, + {deployplan.Update, bundledeployments.OperationActionTypeOperationActionTypeUpdate}, + {deployplan.UpdateWithID, bundledeployments.OperationActionTypeOperationActionTypeUpdateWithId}, + {deployplan.Recreate, bundledeployments.OperationActionTypeOperationActionTypeRecreate}, + {deployplan.Resize, bundledeployments.OperationActionTypeOperationActionTypeResize}, + {deployplan.Delete, bundledeployments.OperationActionTypeOperationActionTypeDelete}, + } + for _, c := range cases { + got, err := deployActionToSDK(c.action) + require.NoError(t, err) + assert.Equal(t, c.want, got) + } + + // Skip and Undefined never reach a recorder and are rejected. + _, err := deployActionToSDK(deployplan.Skip) + assert.Error(t, err) + _, err = deployActionToSDK(deployplan.Undefined) + assert.Error(t, err) +} + +func TestRecordOperationNoOpWithoutRecorder(t *testing.T) { + b := &DeploymentBundle{} + // No OpRec set: recording is a no-op. + assert.NoError(t, b.recordOperation(t.Context(), "resources.jobs.foo", deployplan.Create, "id", struct{}{})) +} diff --git a/bundle/direct/pkg.go b/bundle/direct/pkg.go index 48a9c5a2ff7..f95b515f726 100644 --- a/bundle/direct/pkg.go +++ b/bundle/direct/pkg.go @@ -44,6 +44,11 @@ type DeploymentBundle struct { Plan *deployplan.Plan RemoteStateCache sync.Map StateCache structvar.Cache + + // OpRec records each applied resource operation with the deployment metadata + // service (DMS). It is nil unless the bundle opts into recording deployment + // history, in which case the phases package sets it after CreateVersion. + OpRec opRecorder } // SetRemoteState updates the remote state with type validation and marks as fresh. diff --git a/bundle/phases/deploy.go b/bundle/phases/deploy.go index f65e50a940e..792c016f963 100644 --- a/bundle/phases/deploy.go +++ b/bundle/phases/deploy.go @@ -17,6 +17,7 @@ import ( "github.com/databricks/cli/bundle/deploy/snapshot" "github.com/databricks/cli/bundle/deploy/terraform" "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/cli/bundle/direct" "github.com/databricks/cli/bundle/libraries" "github.com/databricks/cli/bundle/metrics" "github.com/databricks/cli/bundle/permissions" @@ -24,6 +25,7 @@ import ( "github.com/databricks/cli/bundle/statemgmt" "github.com/databricks/cli/libs/agent" "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/dms" "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/logdiag" "github.com/databricks/cli/libs/sync" @@ -161,7 +163,17 @@ func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHand } // lock is acquired here + // + // Set up DMS recording of this deployment as a version. The version is not + // created until the plan is approved (below), so a cancelled deploy records + // nothing; the deferred CompleteVersion is a no-op until CreateVersion runs. + // CompleteVersion is deferred before lock.Release so it runs while the lock + // is still held (defers run last-in-first-out). + recorder := newDeploymentRecorder(ctx, b, stateEngine, dms.VersionTypeDeploy) defer func() { + if err := recorder.CompleteVersion(ctx, !logdiag.HasError(ctx)); err != nil { + logdiag.LogError(ctx, err) + } bundle.ApplyContext(ctx, b, lock.Release(lock.GoalDeploy)) }() @@ -255,6 +267,26 @@ func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHand return } if haveApproval { + // Record the DMS version now that the plan is approved and the state WAL + // has been opened. CreateVersion requests version_id == last_version_id + 1; + // the server returns ABORTED if a concurrent deploy advanced the deployment + // since the plan was computed, so a stale plan is not applied. + if err := recorder.CreateVersion(ctx); err != nil { + logdiag.LogError(ctx, err) + return + } + if recorder != nil { + // On a first deploy the server assigned the deployment ID; persist it in + // state (Finalize writes it to disk) so later deploys reuse the record. + // Record operations under the version just created so DMS holds the + // deployed resource state. + b.DeploymentBundle.StateDB.SetDeploymentID(recorder.DeploymentID()) + b.DeploymentBundle.OpRec = direct.NewOperationRecorder( + b.WorkspaceClient(ctx).BundleDeployments, + recorder.DeploymentID(), + recorder.Version(), + ) + } deployCore(ctx, b, plan, stateEngine, requestedEngine) } else { cmdio.LogString(ctx, "Deployment cancelled!") diff --git a/bundle/phases/destroy.go b/bundle/phases/destroy.go index 2496c7033ad..244f593476f 100644 --- a/bundle/phases/destroy.go +++ b/bundle/phases/destroy.go @@ -13,8 +13,10 @@ import ( "github.com/databricks/cli/bundle/deploy/lock" "github.com/databricks/cli/bundle/deploy/terraform" "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/cli/bundle/direct" "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dms" "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/logdiag" "github.com/databricks/databricks-sdk-go/apierr" @@ -131,7 +133,15 @@ func Destroy(ctx context.Context, b *bundle.Bundle, engine engine.EngineType) { return } + // Set up DMS recording of this destroy as a version. The version is not + // created until the destroy is approved (below), so a cancelled destroy + // records nothing; the deferred CompleteVersion is a no-op until then. It is + // deferred before lock.Release so it runs while the lock is still held. + recorder := newDeploymentRecorder(ctx, b, engine, dms.VersionTypeDestroy) defer func() { + if err := recorder.CompleteVersion(ctx, !logdiag.HasError(ctx)); err != nil { + logdiag.LogError(ctx, err) + } bundle.ApplyContext(ctx, b, lock.Release(lock.GoalDestroy)) }() @@ -188,6 +198,19 @@ func Destroy(ctx context.Context, b *bundle.Bundle, engine engine.EngineType) { return } } + // Record the DMS version now that the destroy is approved and the state WAL + // has been opened, then record each delete operation under it. + if err := recorder.CreateVersion(ctx); err != nil { + logdiag.LogError(ctx, err) + return + } + if recorder != nil { + b.DeploymentBundle.OpRec = direct.NewOperationRecorder( + b.WorkspaceClient(ctx).BundleDeployments, + recorder.DeploymentID(), + recorder.Version(), + ) + } destroyCore(ctx, b, plan, engine) } else { cmdio.LogString(ctx, "Destroy cancelled!") diff --git a/bundle/phases/dms.go b/bundle/phases/dms.go new file mode 100644 index 00000000000..667ef8627aa --- /dev/null +++ b/bundle/phases/dms.go @@ -0,0 +1,37 @@ +package phases + +import ( + "context" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config/engine" + "github.com/databricks/cli/libs/dms" +) + +// newDeploymentRecorder returns a dms.Recorder for the current deployment, or +// nil when DMS recording does not apply. A nil recorder is a no-op, so callers +// do not need to branch on it. +// +// Recording is enabled only when experimental.record_deployment_history is set +// AND the engine is direct: DMS resource state is tracked per direct-engine +// deployment, and only the direct engine opens the state DB where the +// deployment ID is stored. Returning nil for terraform leaves those deployments +// untouched. +// +// The deployment ID passed to the recorder is the one persisted in state from a +// previous deploy; it is empty on a bundle's first recorded deploy, in which +// case the recorder creates the deployment and the server assigns the ID. +func newDeploymentRecorder(ctx context.Context, b *bundle.Bundle, eng engine.EngineType, versionType dms.VersionType) *dms.Recorder { + if b.Config.Experimental == nil || !b.Config.Experimental.RecordDeploymentHistory { + return nil + } + if !eng.IsDirect() { + return nil + } + return dms.NewRecorder( + b.WorkspaceClient(ctx).BundleDeployments, + b.DeploymentBundle.StateDB.GetDeploymentID(), + b.Config.Bundle.Target, + versionType, + ) +} diff --git a/cmd/bundle/generate/dashboard.go b/cmd/bundle/generate/dashboard.go index 086ec1d600a..2b286bcad3d 100644 --- a/cmd/bundle/generate/dashboard.go +++ b/cmd/bundle/generate/dashboard.go @@ -404,7 +404,7 @@ func (d *dashboard) runForResource(ctx context.Context, b *bundle.Bundle) { var state statemgmt.ExportedResourcesMap if stateDesc.Engine.IsDirect() { _, localPath := b.StateFilenameDirect(ctx) - if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WithRecovery(true), dstate.WithWrite(false)); err != nil { + if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WithRecovery(true), dstate.WithWrite(false), nil); err != nil { logdiag.LogError(ctx, err) return } diff --git a/cmd/bundle/generate/genie_space.go b/cmd/bundle/generate/genie_space.go index 6d938c5e03d..48ecc92a6cd 100644 --- a/cmd/bundle/generate/genie_space.go +++ b/cmd/bundle/generate/genie_space.go @@ -322,7 +322,7 @@ func (g *genieSpace) runForResource(ctx context.Context, b *bundle.Bundle) { var state statemgmt.ExportedResourcesMap if stateDesc.Engine.IsDirect() { _, localPath := b.StateFilenameDirect(ctx) - if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WithRecovery(true), dstate.WithWrite(false)); err != nil { + if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WithRecovery(true), dstate.WithWrite(false), nil); err != nil { logdiag.LogError(ctx, err) return } diff --git a/cmd/bundle/utils/process.go b/cmd/bundle/utils/process.go index e4f232605ce..2815f591b22 100644 --- a/cmd/bundle/utils/process.go +++ b/cmd/bundle/utils/process.go @@ -25,6 +25,7 @@ import ( "github.com/databricks/cli/libs/logdiag" "github.com/databricks/cli/libs/sync" "github.com/databricks/cli/libs/telemetry/protos" + "github.com/databricks/databricks-sdk-go/service/bundledeployments" "github.com/spf13/cobra" ) @@ -211,7 +212,16 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle needDirectState := stateDesc.Engine.IsDirect() && (opts.InitIDs || opts.ErrorOnEmptyState || opts.Deploy || opts.ReadPlanPath != "" || opts.PreDeployChecks || opts.PostStateFunc != nil) if needDirectState { _, localPath := b.StateFilenameDirect(ctx) - if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WithRecovery(true), dstate.WithWrite(false)); err != nil { + + // When the bundle records deployment history, the deployment metadata + // service owns resource state, so hand Open its client to overlay DMS + // state on top of the local identity (lineage/serial/deployment ID). + // Reads open the state write-disabled, so no lineage is minted here. + var dmsClient bundledeployments.BundleDeploymentsInterface + if b.Config.Experimental != nil && b.Config.Experimental.RecordDeploymentHistory { + dmsClient = b.WorkspaceClient(ctx).BundleDeployments + } + if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WithRecovery(true), dstate.WithWrite(false), dmsClient); err != nil { logdiag.LogError(ctx, err) return b, stateDesc, root.ErrAlreadyPrinted } diff --git a/libs/dms/recorder.go b/libs/dms/recorder.go new file mode 100644 index 00000000000..eed8485f2c2 --- /dev/null +++ b/libs/dms/recorder.go @@ -0,0 +1,255 @@ +// Package dms records bundle deployments as versions with the Deployment +// Metadata Service (DMS). +// +// It is intentionally independent of the deployment lock: a Recorder does not +// acquire or hold any lock. Callers are responsible for serializing concurrent +// deployments (today via the workspace-filesystem lock). The server-side +// version counter — CreateVersion only succeeds when the requested version is +// last_version_id + 1 — provides the concurrency control for the records +// themselves. +package dms + +import ( + "context" + "errors" + "fmt" + "net/http" + "strconv" + "strings" + "time" + + "github.com/databricks/cli/internal/build" + "github.com/databricks/cli/libs/log" + "github.com/databricks/databricks-sdk-go/apierr" + "github.com/databricks/databricks-sdk-go/service/bundledeployments" +) + +// The server expires a version's lease if it does not receive a heartbeat +// within a 2-minute TTL; we heartbeat well inside that window. +const defaultHeartbeatInterval = 30 * time.Second + +// VersionType identifies the kind of deployment a version records. +type VersionType = bundledeployments.VersionType + +const ( + VersionTypeDeploy VersionType = bundledeployments.VersionTypeVersionTypeDeploy + VersionTypeDestroy VersionType = bundledeployments.VersionTypeVersionTypeDestroy +) + +// Recorder records a single deploy/destroy as a version with DMS. +// +// The deployment ID is assigned by the server on the first deploy: NewRecorder +// is given the ID persisted in state (empty on a bundle's first-ever recorded +// deploy), and CreateVersion creates the deployment record when that ID is +// empty and exposes the server-assigned ID via DeploymentID so the caller can +// persist it. Later deploys pass the stored ID back in and reuse the record. +type Recorder struct { + svc bundledeployments.BundleDeploymentsInterface + deploymentID string + targetName string + versionType VersionType + + // populated by CreateVersion + versionNum int64 + stopHeartbeat context.CancelFunc +} + +// NewRecorder returns a Recorder for the given deployment. deploymentID is the +// DMS deployment ID persisted in state, or empty if this bundle has not yet +// recorded a deployment (the server assigns one during CreateVersion). +func NewRecorder(svc bundledeployments.BundleDeploymentsInterface, deploymentID, targetName string, versionType VersionType) *Recorder { + return &Recorder{ + svc: svc, + deploymentID: deploymentID, + targetName: targetName, + versionType: versionType, + } +} + +// DeploymentID returns the DMS deployment ID this recorder is bound to. It is +// empty until CreateVersion has created the deployment record (on a first +// deploy) and non-empty afterwards, so callers persist it once CreateVersion +// succeeds. +func (r *Recorder) DeploymentID() string { + if r == nil { + return "" + } + return r.deploymentID +} + +// Version returns the version number claimed by CreateVersion. It is zero until +// CreateVersion has run; callers use it to parent operations under the version. +func (r *Recorder) Version() int64 { + if r == nil { + return 0 + } + return r.versionNum +} + +// CreateVersion registers a new version with DMS, claiming it for the duration +// of the deployment. A nil Recorder is a no-op, so callers can leave it nil +// when recording is disabled. +func (r *Recorder) CreateVersion(ctx context.Context) error { + if r == nil { + return nil + } + + versionID, err := r.createDeploymentVersion(ctx) + if err != nil { + return err + } + + versionNum, err := strconv.ParseInt(versionID, 10, 64) + if err != nil { + return fmt.Errorf("failed to parse version ID %q: %w", versionID, err) + } + r.versionNum = versionNum + r.stopHeartbeat = startHeartbeat(ctx, r.svc, r.deploymentID, versionID) + return nil +} + +// CompleteVersion finalizes the version created by CreateVersion. A nil +// Recorder, or one whose CreateVersion never ran, is a no-op. +func (r *Recorder) CompleteVersion(ctx context.Context, success bool) error { + if r == nil || r.stopHeartbeat == nil { + return nil + } + + r.stopHeartbeat() + + versionIDStr := strconv.FormatInt(r.versionNum, 10) + versionName := fmt.Sprintf("deployments/%s/versions/%s", r.deploymentID, versionIDStr) + + reason := bundledeployments.VersionCompleteVersionCompleteSuccess + if !success { + reason = bundledeployments.VersionCompleteVersionCompleteFailure + } + + _, err := r.svc.CompleteVersion(ctx, bundledeployments.CompleteVersionRequest{ + Name: versionName, + CompletionReason: reason, + }) + if err != nil { + return err + } + log.Infof(ctx, "Completed deployment version: deployment=%s version=%s reason=%s", r.deploymentID, versionIDStr, reason) + + // For destroy operations, delete the deployment record after the version + // completes successfully. + if success && r.versionType == VersionTypeDestroy { + err = r.svc.DeleteDeployment(ctx, bundledeployments.DeleteDeploymentRequest{ + Name: "deployments/" + r.deploymentID, + }) + if err != nil { + return fmt.Errorf("failed to delete deployment: %w", err) + } + } + + return nil +} + +// createDeploymentVersion ensures the deployment record exists, then creates a +// new version under it. On a first deploy (no stored deployment ID) it creates +// the deployment and lets the server assign the ID; otherwise it reads the +// existing deployment to compute the next version number. +func (r *Recorder) createDeploymentVersion(ctx context.Context) (versionID string, err error) { + if r.deploymentID == "" { + // First deploy: create the deployment with an empty ID so the server + // assigns one, then start at version 1. + dep, createErr := r.svc.CreateDeployment(ctx, bundledeployments.CreateDeploymentRequest{ + Deployment: bundledeployments.Deployment{ + TargetName: r.targetName, + }, + }) + if createErr != nil { + return "", fmt.Errorf("failed to create deployment: %w", createErr) + } + id, idErr := deploymentIDFromName(dep.Name) + if idErr != nil { + return "", idErr + } + r.deploymentID = id + versionID = "1" + } else { + // Existing deployment: read it to compute the next version number. + dep, getErr := r.svc.GetDeployment(ctx, bundledeployments.GetDeploymentRequest{ + Name: "deployments/" + r.deploymentID, + }) + if getErr != nil { + return "", fmt.Errorf("failed to get deployment: %w", getErr) + } + lastVersion, parseErr := strconv.ParseInt(dep.LastVersionId, 10, 64) + if parseErr != nil { + return "", fmt.Errorf("failed to parse last_version_id %q: %w", dep.LastVersionId, parseErr) + } + versionID = strconv.FormatInt(lastVersion+1, 10) + } + + // The server validates that versionID equals last_version_id + 1 and returns + // ABORTED otherwise (e.g. a concurrent deploy already created this version). + version, versionErr := r.svc.CreateVersion(ctx, bundledeployments.CreateVersionRequest{ + Parent: "deployments/" + r.deploymentID, + VersionId: versionID, + Version: bundledeployments.Version{ + CliVersion: build.GetInfo().Version, + VersionType: r.versionType, + TargetName: r.targetName, + }, + }) + if versionErr != nil { + return "", fmt.Errorf("failed to create deployment version: %w", versionErr) + } + + log.Infof(ctx, "Created deployment version: deployment=%s version=%s", r.deploymentID, version.VersionId) + return versionID, nil +} + +// deploymentIDFromName extracts the deployment ID from a DMS resource name of +// the form "deployments/{deployment_id}". +func deploymentIDFromName(name string) (string, error) { + id, ok := strings.CutPrefix(name, "deployments/") + if !ok || id == "" { + return "", fmt.Errorf("unexpected deployment name %q from deployment metadata service", name) + } + return id, nil +} + +// startHeartbeat starts a background goroutine that sends heartbeats to keep +// the deployment version's lease alive. Returns a cancel function to stop it. +func startHeartbeat(ctx context.Context, svc bundledeployments.BundleDeploymentsInterface, deploymentID, versionID string) context.CancelFunc { + ctx, cancel := context.WithCancel(ctx) + versionName := fmt.Sprintf("deployments/%s/versions/%s", deploymentID, versionID) + + go func() { + ticker := time.NewTicker(defaultHeartbeatInterval) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + _, err := svc.Heartbeat(ctx, bundledeployments.HeartbeatRequest{Name: versionName}) + if err != nil { + // A 409 ABORTED is expected if the version was completed + // between the ticker firing and the heartbeat. + if isAbortedErr(err) { + log.Debugf(ctx, "Heartbeat stopped: version already completed") + return + } + log.Warnf(ctx, "Failed to send deployment heartbeat: %v", err) + } else { + log.Debugf(ctx, "Deployment heartbeat sent: deployment=%s version=%s", deploymentID, versionID) + } + } + } + }() + + return cancel +} + +// isAbortedErr reports whether err is an HTTP 409 ABORTED from the DMS API. +func isAbortedErr(err error) bool { + apiErr, ok := errors.AsType[*apierr.APIError](err) + return ok && apiErr.StatusCode == http.StatusConflict && apiErr.ErrorCode == "ABORTED" +} diff --git a/libs/dms/recorder_test.go b/libs/dms/recorder_test.go new file mode 100644 index 00000000000..91848f74a70 --- /dev/null +++ b/libs/dms/recorder_test.go @@ -0,0 +1,165 @@ +package dms + +import ( + "context" + "testing" + + "github.com/databricks/databricks-sdk-go/service/bundledeployments" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// fakeDMS records the calls the recorder makes and lets a test script the +// server-side responses. It embeds the SDK interface so it satisfies it while +// only overriding the methods the recorder uses. +type fakeDMS struct { + bundledeployments.BundleDeploymentsInterface + + // scripted behavior + getDeployment func(id string) (*bundledeployments.Deployment, error) + + // assigned deployment ID for CreateDeployment (server-generated flow) + assignedID string + + // captured requests + created []bundledeployments.CreateDeploymentRequest + versions []bundledeployments.CreateVersionRequest + completed []bundledeployments.CompleteVersionRequest + deleted []string +} + +func (f *fakeDMS) CreateDeployment(ctx context.Context, req bundledeployments.CreateDeploymentRequest) (*bundledeployments.Deployment, error) { + f.created = append(f.created, req) + id := req.DeploymentId + if id == "" { + id = f.assignedID + } + return &bundledeployments.Deployment{Name: "deployments/" + id}, nil +} + +func (f *fakeDMS) GetDeployment(ctx context.Context, req bundledeployments.GetDeploymentRequest) (*bundledeployments.Deployment, error) { + id := req.Name[len("deployments/"):] + return f.getDeployment(id) +} + +func (f *fakeDMS) CreateVersion(ctx context.Context, req bundledeployments.CreateVersionRequest) (*bundledeployments.Version, error) { + f.versions = append(f.versions, req) + return &bundledeployments.Version{VersionId: req.VersionId}, nil +} + +func (f *fakeDMS) CompleteVersion(ctx context.Context, req bundledeployments.CompleteVersionRequest) (*bundledeployments.Version, error) { + f.completed = append(f.completed, req) + return &bundledeployments.Version{}, nil +} + +func (f *fakeDMS) DeleteDeployment(ctx context.Context, req bundledeployments.DeleteDeploymentRequest) error { + f.deleted = append(f.deleted, req.Name) + return nil +} + +func (f *fakeDMS) Heartbeat(ctx context.Context, req bundledeployments.HeartbeatRequest) (*bundledeployments.HeartbeatResponse, error) { + return &bundledeployments.HeartbeatResponse{}, nil +} + +func TestRecorderFirstDeployCreatesDeploymentWithServerAssignedID(t *testing.T) { + f := &fakeDMS{assignedID: "server-generated-id"} + // A first deploy has no stored deployment ID. + r := NewRecorder(f, "", "dev", VersionTypeDeploy) + + require.NoError(t, r.CreateVersion(t.Context())) + + // The deployment was created with an empty ID so the server assigns one, and + // the recorder exposes the assigned ID for the caller to persist. + require.Len(t, f.created, 1) + assert.Empty(t, f.created[0].DeploymentId) + assert.Equal(t, "server-generated-id", r.DeploymentID()) + + // The first version is 1, parented under the assigned deployment. + require.Len(t, f.versions, 1) + assert.Equal(t, "1", f.versions[0].VersionId) + assert.Equal(t, "deployments/server-generated-id", f.versions[0].Parent) + assert.Equal(t, int64(1), r.Version()) + + require.NoError(t, r.CompleteVersion(t.Context(), true)) + require.Len(t, f.completed, 1) + assert.Equal(t, bundledeployments.VersionCompleteVersionCompleteSuccess, f.completed[0].CompletionReason) + assert.Empty(t, f.deleted) +} + +func TestRecorderSubsequentDeployReusesDeploymentAndIncrementsVersion(t *testing.T) { + f := &fakeDMS{ + getDeployment: func(id string) (*bundledeployments.Deployment, error) { + return &bundledeployments.Deployment{Name: "deployments/" + id, LastVersionId: "4"}, nil + }, + } + // A subsequent deploy passes the stored deployment ID. + r := NewRecorder(f, "stored-id", "dev", VersionTypeDeploy) + + require.NoError(t, r.CreateVersion(t.Context())) + + // No new deployment is created; the version increments to last_version_id + 1. + assert.Empty(t, f.created) + require.Len(t, f.versions, 1) + assert.Equal(t, "5", f.versions[0].VersionId) + assert.Equal(t, "stored-id", r.DeploymentID()) +} + +func TestRecorderDestroyDeletesDeploymentOnSuccess(t *testing.T) { + f := &fakeDMS{ + getDeployment: func(id string) (*bundledeployments.Deployment, error) { + return &bundledeployments.Deployment{Name: "deployments/" + id, LastVersionId: "2"}, nil + }, + } + r := NewRecorder(f, "stored-id", "dev", VersionTypeDestroy) + + require.NoError(t, r.CreateVersion(t.Context())) + assert.Equal(t, bundledeployments.VersionTypeVersionTypeDestroy, f.versions[0].Version.VersionType) + + require.NoError(t, r.CompleteVersion(t.Context(), true)) + // A successful destroy deletes the deployment record. + require.Equal(t, []string{"deployments/stored-id"}, f.deleted) +} + +func TestRecorderFailedDestroyKeepsDeployment(t *testing.T) { + f := &fakeDMS{ + getDeployment: func(id string) (*bundledeployments.Deployment, error) { + return &bundledeployments.Deployment{Name: "deployments/" + id, LastVersionId: "2"}, nil + }, + } + r := NewRecorder(f, "stored-id", "dev", VersionTypeDestroy) + + require.NoError(t, r.CreateVersion(t.Context())) + require.NoError(t, r.CompleteVersion(t.Context(), false)) + + assert.Equal(t, bundledeployments.VersionCompleteVersionCompleteFailure, f.completed[0].CompletionReason) + // A failed destroy leaves the deployment in place. + assert.Empty(t, f.deleted) +} + +func TestNilRecorderIsNoOp(t *testing.T) { + var r *Recorder + assert.NoError(t, r.CreateVersion(t.Context())) + assert.NoError(t, r.CompleteVersion(t.Context(), true)) + assert.Empty(t, r.DeploymentID()) + assert.Zero(t, r.Version()) +} + +func TestRecorderCompleteVersionNoOpWithoutCreateVersion(t *testing.T) { + f := &fakeDMS{} + r := NewRecorder(f, "stored-id", "dev", VersionTypeDeploy) + // CompleteVersion before CreateVersion is a no-op (nothing was claimed). + require.NoError(t, r.CompleteVersion(t.Context(), true)) + assert.Empty(t, f.completed) +} + +func TestDeploymentIDFromName(t *testing.T) { + id, err := deploymentIDFromName("deployments/abc-123") + require.NoError(t, err) + assert.Equal(t, "abc-123", id) + + _, err = deploymentIDFromName("abc-123") + assert.Error(t, err) + + _, err = deploymentIDFromName("deployments/") + assert.Error(t, err) +} diff --git a/libs/testserver/bundle.go b/libs/testserver/bundle.go new file mode 100644 index 00000000000..5f2c7e0cfd1 --- /dev/null +++ b/libs/testserver/bundle.go @@ -0,0 +1,228 @@ +package testserver + +import ( + "encoding/json" + "slices" + "strconv" + + "github.com/databricks/databricks-sdk-go/service/bundledeployments" +) + +// Handlers for the Deployment Metadata Service (DMS) API under /api/2.0/bundle. +// State is kept in FakeWorkspace.dmsDeployments, keyed by deployment ID. + +// dmsDeployment holds a deployment record together with the versions and +// resources recorded under it, so the read APIs (ListVersions/ListResources) +// can serve back what deploys wrote. +type dmsDeployment struct { + deployment bundledeployments.Deployment + versions map[string]*bundledeployments.Version + // resources is the latest resource state per resource key, updated as + // operations are recorded. + resources map[string]bundledeployments.Resource +} + +func (s *FakeWorkspace) CreateDeployment(req Request) Response { + // The client either supplies the deployment ID or, in the server-generated + // flow, leaves it empty for the server to mint one. + deploymentID := req.URL.Query().Get("deployment_id") + if deploymentID == "" { + deploymentID = nextUUID() + } + + var dep bundledeployments.Deployment + if err := json.Unmarshal(req.Body, &dep); err != nil { + return Response{StatusCode: 400, Body: map[string]string{"message": err.Error()}} + } + + defer s.LockUnlock()() + + dep.Name = "deployments/" + deploymentID + dep.Status = bundledeployments.DeploymentStatusDeploymentStatusActive + s.dmsDeployments[deploymentID] = &dmsDeployment{ + deployment: dep, + versions: map[string]*bundledeployments.Version{}, + resources: map[string]bundledeployments.Resource{}, + } + return Response{Body: dep} +} + +func (s *FakeWorkspace) GetDeployment(deploymentID string) Response { + defer s.LockUnlock()() + + d, ok := s.dmsDeployments[deploymentID] + if !ok { + return dmsNotFound("deployment " + deploymentID) + } + return Response{Body: d.deployment} +} + +func (s *FakeWorkspace) DeleteDeployment(deploymentID string) Response { + defer s.LockUnlock()() + + delete(s.dmsDeployments, deploymentID) + return Response{Body: map[string]any{}} +} + +func (s *FakeWorkspace) CreateVersion(req Request, deploymentID string) Response { + versionID := req.URL.Query().Get("version_id") + + var version bundledeployments.Version + if err := json.Unmarshal(req.Body, &version); err != nil { + return Response{StatusCode: 400, Body: map[string]string{"message": err.Error()}} + } + + defer s.LockUnlock()() + + d, ok := s.dmsDeployments[deploymentID] + if !ok { + return dmsNotFound("deployment " + deploymentID) + } + + // Mirror the server-side optimistic concurrency check: the new version must + // be exactly last_version_id + 1. + want := "1" + if d.deployment.LastVersionId != "" { + last, _ := strconv.ParseInt(d.deployment.LastVersionId, 10, 64) + want = strconv.FormatInt(last+1, 10) + } + if versionID != want { + return dmsAborted("expected version " + want + ", got " + versionID) + } + + d.deployment.LastVersionId = versionID + version.Name = "deployments/" + deploymentID + "/versions/" + versionID + version.VersionId = versionID + version.Status = bundledeployments.VersionStatusVersionStatusInProgress + d.versions[versionID] = &version + return Response{Body: version} +} + +func (s *FakeWorkspace) CompleteVersion(req Request, deploymentID, versionID string) Response { + var completeReq bundledeployments.CompleteVersionRequest + if err := json.Unmarshal(req.Body, &completeReq); err != nil { + return Response{StatusCode: 400, Body: map[string]string{"message": err.Error()}} + } + + defer s.LockUnlock()() + + d, ok := s.dmsDeployments[deploymentID] + if !ok { + return dmsNotFound("deployment " + deploymentID) + } + v, ok := d.versions[versionID] + if !ok { + return dmsNotFound("version " + versionID) + } + + v.Status = bundledeployments.VersionStatusVersionStatusCompleted + v.CompletionReason = completeReq.CompletionReason + return Response{Body: *v} +} + +func (s *FakeWorkspace) Heartbeat() Response { + return Response{Body: bundledeployments.HeartbeatResponse{}} +} + +func (s *FakeWorkspace) CreateOperation(req Request, deploymentID, versionID string) Response { + resourceKey := req.URL.Query().Get("resource_key") + + var op bundledeployments.Operation + if err := json.Unmarshal(req.Body, &op); err != nil { + return Response{StatusCode: 400, Body: map[string]string{"message": err.Error()}} + } + + defer s.LockUnlock()() + + d, ok := s.dmsDeployments[deploymentID] + if !ok { + return dmsNotFound("deployment " + deploymentID) + } + + op.Name = "deployments/" + deploymentID + "/versions/" + versionID + "/operations/" + resourceKey + op.ResourceKey = resourceKey + + // Reflect the operation onto the deployment-level resource set the way the + // backend does: a delete removes the resource, anything else upserts it. + if op.ActionType == bundledeployments.OperationActionTypeOperationActionTypeDelete { + delete(d.resources, resourceKey) + } else { + d.resources[resourceKey] = bundledeployments.Resource{ + Name: "deployments/" + deploymentID + "/resources/" + resourceKey, + ResourceKey: resourceKey, + ResourceId: op.ResourceId, + ResourceType: op.ResourceType, + LastActionType: op.ActionType, + LastVersionId: versionID, + State: op.State, + } + } + return Response{Body: op} +} + +func (s *FakeWorkspace) ListVersions(deploymentID string) Response { + defer s.LockUnlock()() + + d, ok := s.dmsDeployments[deploymentID] + if !ok { + return dmsNotFound("deployment " + deploymentID) + } + + // The API returns versions newest-first (descending version_id). + ids := make([]int64, 0, len(d.versions)) + for id := range d.versions { + n, _ := strconv.ParseInt(id, 10, 64) + ids = append(ids, n) + } + slices.SortFunc(ids, func(a, b int64) int { return int(b - a) }) + + versions := make([]bundledeployments.Version, 0, len(ids)) + for _, id := range ids { + versions = append(versions, *d.versions[strconv.FormatInt(id, 10)]) + } + return Response{Body: bundledeployments.ListVersionsResponse{Versions: versions}} +} + +func (s *FakeWorkspace) ListResources(deploymentID string) Response { + defer s.LockUnlock()() + + d, ok := s.dmsDeployments[deploymentID] + if !ok { + return dmsNotFound("deployment " + deploymentID) + } + + // Sort by resource key so the response order is deterministic. + keys := make([]string, 0, len(d.resources)) + for key := range d.resources { + keys = append(keys, key) + } + slices.Sort(keys) + + resources := make([]bundledeployments.Resource, 0, len(keys)) + for _, key := range keys { + resources = append(resources, d.resources[key]) + } + return Response{Body: bundledeployments.ListResourcesResponse{Resources: resources}} +} + +// dmsNotFound returns the RESOURCE_DOES_NOT_EXIST error shape the DMS API uses, +// which the SDK maps to apierr.ErrNotFound. +func dmsNotFound(what string) Response { + return Response{ + StatusCode: 404, + Body: map[string]string{ + "error_code": "RESOURCE_DOES_NOT_EXIST", + "message": what + " does not exist", + }, + } +} + +// dmsAborted returns the 409 ABORTED error the server uses for the version +// optimistic-concurrency check. +func dmsAborted(message string) Response { + return Response{ + StatusCode: 409, + Headers: map[string][]string{"Content-Type": {"application/json"}}, + Body: map[string]string{"error_code": "ABORTED", "message": message}, + } +} diff --git a/libs/testserver/fake_workspace.go b/libs/testserver/fake_workspace.go index 8d6e8ee0dd3..a3c4519ccc4 100644 --- a/libs/testserver/fake_workspace.go +++ b/libs/testserver/fake_workspace.go @@ -227,6 +227,10 @@ type FakeWorkspace struct { // clusterVenvs caches Python venvs per existing cluster ID, // matching cloud behavior where libraries are cached on running clusters. clusterVenvs map[string]*clusterEnv + + // dmsDeployments holds Deployment Metadata Service (DMS) records, keyed by + // deployment ID. Each record carries its versions and latest resource state. + dmsDeployments map[string]*dmsDeployment } func (s *FakeWorkspace) LockUnlock() func() { @@ -378,6 +382,7 @@ func NewFakeWorkspace(url, token string) *FakeWorkspace { postgresImplicitBranches: map[string]bool{}, postgresImplicitEndpoints: map[string]bool{}, clusterVenvs: map[string]*clusterEnv{}, + dmsDeployments: map[string]*dmsDeployment{}, Alerts: map[string]sql.AlertV2{}, Experiments: map[string]ml.GetExperimentResponse{}, ModelRegistryModels: map[string]ml.Model{}, diff --git a/libs/testserver/handlers.go b/libs/testserver/handlers.go index 1d534c47431..39fe6fbb057 100644 --- a/libs/testserver/handlers.go +++ b/libs/testserver/handlers.go @@ -266,6 +266,35 @@ func AddDefaultHandlers(server *Server) { return req.Workspace.JobsCreate(req) }) + // Deployment Metadata Service (DMS) endpoints. + server.Handle("POST", "/api/2.0/bundle/deployments", func(req Request) any { + return req.Workspace.CreateDeployment(req) + }) + server.Handle("GET", "/api/2.0/bundle/deployments/{deployment_id}", func(req Request) any { + return req.Workspace.GetDeployment(req.Vars["deployment_id"]) + }) + server.Handle("DELETE", "/api/2.0/bundle/deployments/{deployment_id}", func(req Request) any { + return req.Workspace.DeleteDeployment(req.Vars["deployment_id"]) + }) + server.Handle("GET", "/api/2.0/bundle/deployments/{deployment_id}/versions", func(req Request) any { + return req.Workspace.ListVersions(req.Vars["deployment_id"]) + }) + server.Handle("POST", "/api/2.0/bundle/deployments/{deployment_id}/versions", func(req Request) any { + return req.Workspace.CreateVersion(req, req.Vars["deployment_id"]) + }) + server.Handle("POST", "/api/2.0/bundle/deployments/{deployment_id}/versions/{version_id}/complete", func(req Request) any { + return req.Workspace.CompleteVersion(req, req.Vars["deployment_id"], req.Vars["version_id"]) + }) + server.Handle("POST", "/api/2.0/bundle/deployments/{deployment_id}/versions/{version_id}/heartbeat", func(req Request) any { + return req.Workspace.Heartbeat() + }) + server.Handle("POST", "/api/2.0/bundle/deployments/{deployment_id}/versions/{version_id}/operations", func(req Request) any { + return req.Workspace.CreateOperation(req, req.Vars["deployment_id"], req.Vars["version_id"]) + }) + server.Handle("GET", "/api/2.0/bundle/deployments/{deployment_id}/resources", func(req Request) any { + return req.Workspace.ListResources(req.Vars["deployment_id"]) + }) + server.Handle("POST", "/api/2.2/jobs/delete", func(req Request) any { var request jobs.DeleteJob if err := json.Unmarshal(req.Body, &request); err != nil { From da1ed9dfc00a9d6171b2fbeb60d9f17a3e900b26 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Fri, 24 Jul 2026 16:49:32 +0200 Subject: [PATCH 002/125] bundle: read DMS authority from last_successful_version_id MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The read overlay decided whether DMS owns a deployment's state by listing versions and scanning for a successful one. The deployment now exposes last_successful_version_id directly, so a single GetDeployment answers the same question — no version listing. The field is still stage:DEVELOPMENT in the proto and therefore stripped from the generated SDK, so this reads the deployment via a raw GET into a local struct as a temporary stub. Once the field is promoted to PRIVATE_PREVIEW and regenerated, the raw call collapses to client.GetDeployment(...). LastSuccessfulVersionId and the threaded config argument goes away (see the TODO in deploymentHasSuccessfulVersion). The testserver's GetDeployment now serves last_successful_version_id (tracked on version completion), and the now-unused ListVersions fake is removed. Co-authored-by: Isaac --- bundle/configsync/diff.go | 2 +- bundle/configsync/variables.go | 2 +- bundle/direct/bind.go | 12 +++--- bundle/direct/dstate/dms.go | 64 +++++++++++++++++++----------- bundle/direct/dstate/state.go | 9 ++++- bundle/direct/dstate/state_test.go | 30 +++++++------- cmd/bundle/generate/dashboard.go | 2 +- cmd/bundle/generate/genie_space.go | 2 +- cmd/bundle/utils/process.go | 7 +++- libs/testserver/bundle.go | 45 ++++++++++----------- libs/testserver/handlers.go | 3 -- 11 files changed, 100 insertions(+), 78 deletions(-) diff --git a/bundle/configsync/diff.go b/bundle/configsync/diff.go index ca5b2c9410b..17ed3b30d5e 100644 --- a/bundle/configsync/diff.go +++ b/bundle/configsync/diff.go @@ -149,7 +149,7 @@ func OpenDeploymentState(ctx context.Context, b *bundle.Bundle, engine engine.En deployBundle := &direct.DeploymentBundle{} _, statePath := b.StateFilenameConfigSnapshot(ctx) - if err := deployBundle.StateDB.Open(ctx, statePath, dstate.WithRecovery(true), dstate.WithWrite(false), nil); err != nil { + if err := deployBundle.StateDB.Open(ctx, statePath, dstate.WithRecovery(true), dstate.WithWrite(false), nil, nil); err != nil { return nil, fmt.Errorf("failed to open state: %w", err) } return deployBundle, nil diff --git a/bundle/configsync/variables.go b/bundle/configsync/variables.go index 433b607a037..be3e536f37f 100644 --- a/bundle/configsync/variables.go +++ b/bundle/configsync/variables.go @@ -147,7 +147,7 @@ func resourceIDLookup(ctx context.Context, b *bundle.Bundle) func(string) string } _, statePath := b.StateFilenameConfigSnapshot(ctx) db := &dstate.DeploymentState{} - if err := db.Open(ctx, statePath, dstate.WithRecovery(false), dstate.WithWrite(false), nil); err != nil { + if err := db.Open(ctx, statePath, dstate.WithRecovery(false), dstate.WithWrite(false), nil, nil); err != nil { log.Debugf(ctx, "variable restoration: failed to open state DB at %s: %v", statePath, err) return nil } diff --git a/bundle/direct/bind.go b/bundle/direct/bind.go index ec910b2734e..ccfbcf788ab 100644 --- a/bundle/direct/bind.go +++ b/bundle/direct/bind.go @@ -62,7 +62,7 @@ type BindResult struct { func (b *DeploymentBundle) Bind(ctx context.Context, client *databricks.WorkspaceClient, configRoot *config.Root, statePath, resourceKey, resourceID string) (*BindResult, error) { // Check if the resource is already managed (bound to a different ID) var checkStateDB dstate.DeploymentState - if err := checkStateDB.Open(ctx, statePath, dstate.WithRecovery(true), dstate.WithWrite(false), nil); err == nil { + if err := checkStateDB.Open(ctx, statePath, dstate.WithRecovery(true), dstate.WithWrite(false), nil, nil); err == nil { existingID := checkStateDB.GetResourceID(resourceKey) if _, err := checkStateDB.Finalize(ctx); err != nil { log.Warnf(ctx, "failed to finalize state: %v", err) @@ -86,7 +86,7 @@ func (b *DeploymentBundle) Bind(ctx context.Context, client *databricks.Workspac } // Open temp state - err := b.StateDB.Open(ctx, tmpStatePath, dstate.WithRecovery(false), dstate.WithWrite(true), nil) + err := b.StateDB.Open(ctx, tmpStatePath, dstate.WithRecovery(false), dstate.WithWrite(true), nil, nil) if err != nil { os.Remove(tmpStatePath) return nil, err @@ -109,7 +109,7 @@ func (b *DeploymentBundle) Bind(ctx context.Context, client *databricks.Workspac log.Infof(ctx, "Bound %s to id=%s (in temp state)", resourceKey, resourceID) // First plan + update: populate state with resolved config - err = b.StateDB.Open(ctx, tmpStatePath, dstate.WithRecovery(true), dstate.WithWrite(false), nil) + err = b.StateDB.Open(ctx, tmpStatePath, dstate.WithRecovery(true), dstate.WithWrite(false), nil, nil) if err != nil { os.Remove(tmpStatePath) return nil, err @@ -145,7 +145,7 @@ func (b *DeploymentBundle) Bind(ctx context.Context, client *databricks.Workspac } } - err = b.StateDB.Open(ctx, tmpStatePath, dstate.WithRecovery(true), dstate.WithWrite(true), nil) + err = b.StateDB.Open(ctx, tmpStatePath, dstate.WithRecovery(true), dstate.WithWrite(true), nil, nil) if err != nil { os.Remove(tmpStatePath) return nil, err @@ -165,7 +165,7 @@ func (b *DeploymentBundle) Bind(ctx context.Context, client *databricks.Workspac } // Second plan: this is the plan to present to the user (change between remote resource and config) - err = b.StateDB.Open(ctx, tmpStatePath, dstate.WithRecovery(true), dstate.WithWrite(false), nil) + err = b.StateDB.Open(ctx, tmpStatePath, dstate.WithRecovery(true), dstate.WithWrite(false), nil, nil) if err != nil { os.Remove(tmpStatePath) return nil, err @@ -215,7 +215,7 @@ func (result *BindResult) Cancel() { // Unbind removes a resource from direct engine state without deleting // the workspace resource. Also removes associated permissions/grants entries. func (b *DeploymentBundle) Unbind(ctx context.Context, statePath, resourceKey string) error { - err := b.StateDB.Open(ctx, statePath, dstate.WithRecovery(true), dstate.WithWrite(true), nil) + err := b.StateDB.Open(ctx, statePath, dstate.WithRecovery(true), dstate.WithWrite(true), nil, nil) if err != nil { return err } diff --git a/bundle/direct/dstate/dms.go b/bundle/direct/dstate/dms.go index 1d19d1fe214..659d7c9dd0b 100644 --- a/bundle/direct/dstate/dms.go +++ b/bundle/direct/dstate/dms.go @@ -5,8 +5,12 @@ import ( "encoding/json" "errors" "fmt" + "net/http" + "github.com/databricks/cli/libs/auth" "github.com/databricks/databricks-sdk-go/apierr" + "github.com/databricks/databricks-sdk-go/client" + sdkconfig "github.com/databricks/databricks-sdk-go/config" "github.com/databricks/databricks-sdk-go/service/bundledeployments" ) @@ -17,8 +21,11 @@ import ( // used when DMS has no successful version, or when the user opts out of // recording deployment history. The caller holds db.mu and has already // populated db.Data from the file, including the DeploymentID. -func (db *DeploymentState) overlayDMSState(ctx context.Context, client bundledeployments.BundleDeploymentsInterface) error { - authoritative, err := deploymentHasSuccessfulVersion(ctx, client, db.Data.DeploymentID) +// +// cfg is threaded in only for the temporary raw read in +// deploymentHasSuccessfulVersion; see the TODO there. +func (db *DeploymentState) overlayDMSState(ctx context.Context, client bundledeployments.BundleDeploymentsInterface, cfg *sdkconfig.Config) error { + authoritative, err := deploymentHasSuccessfulVersion(ctx, cfg, db.Data.DeploymentID) if err != nil { return err } @@ -46,29 +53,40 @@ func (db *DeploymentState) overlayDMSState(ctx context.Context, client bundledep // state: if the deployment was never recorded to DMS, or its initial DMS deploy // did not complete successfully, DMS state is absent or partial and Open keeps // the local file's resources instead. -func deploymentHasSuccessfulVersion(ctx context.Context, client bundledeployments.BundleDeploymentsInterface, deploymentID string) (bool, error) { - // Versions are listed newest-first and fetched page by page, and we stop at - // the first successful one, so a deployment with a long version history does - // not require reading the whole list (typically just the first page). - it := client.ListVersions(ctx, bundledeployments.ListVersionsRequest{ - Parent: "deployments/" + deploymentID, - }) - for it.HasNext(ctx) { - v, err := it.Next(ctx) - if err != nil { - // A deployment that was never recorded to DMS is not an error here: it - // just means DMS is not (yet) the source of truth. - if errors.Is(err, apierr.ErrNotFound) { - return false, nil - } - return false, fmt.Errorf("listing versions from deployment metadata service: %w", err) - } - if v.Status == bundledeployments.VersionStatusVersionStatusCompleted && - v.CompletionReason == bundledeployments.VersionCompleteVersionCompleteSuccess { - return true, nil +// +// The deployment carries last_successful_version_id, which the server advances +// only when a version completes successfully (unlike last_version_id, which +// also advances on failure). So a non-empty value is exactly the "DMS owns the +// state" signal, readable in a single GetDeployment. +// +// TODO(DMS): this reads the deployment via a raw GET into a local struct +// because last_successful_version_id is still stage:DEVELOPMENT in the proto +// and therefore stripped from the generated SDK. Once the field is promoted to +// PRIVATE_PREVIEW and regenerated, replace the raw call with +// client.GetDeployment(...).LastSuccessfulVersionId and drop the cfg argument +// (revert overlayDMSState/Open back to taking only the typed client). +func deploymentHasSuccessfulVersion(ctx context.Context, cfg *sdkconfig.Config, deploymentID string) (bool, error) { + apiClient, err := client.New(cfg) + if err != nil { + return false, fmt.Errorf("creating API client for deployment metadata service: %w", err) + } + + // Mirrors the SDK's GetDeployment path (/api/2.0/bundle/{name} with + // name=deployments/{id}); we unmarshal into a local struct so we can read + // last_successful_version_id, which the typed SDK response drops. + var dep struct { + LastSuccessfulVersionID string `json:"last_successful_version_id"` + } + err = apiClient.Do(ctx, http.MethodGet, "/api/2.0/bundle/deployments/"+deploymentID, auth.WorkspaceIDHeaders(cfg), nil, nil, &dep) + if err != nil { + // A deployment that was never recorded to DMS is not an error here: it + // just means DMS is not (yet) the source of truth. + if errors.Is(err, apierr.ErrNotFound) { + return false, nil } + return false, fmt.Errorf("reading deployment from deployment metadata service: %w", err) } - return false, nil + return dep.LastSuccessfulVersionID != "", nil } // fetchDeploymentResources lists every resource recorded for the deployment in diff --git a/bundle/direct/dstate/state.go b/bundle/direct/dstate/state.go index 64fc050bdc0..2c969667c9e 100644 --- a/bundle/direct/dstate/state.go +++ b/bundle/direct/dstate/state.go @@ -19,6 +19,7 @@ import ( "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/structs/structwalk" + sdkconfig "github.com/databricks/databricks-sdk-go/config" "github.com/databricks/databricks-sdk-go/service/bundledeployments" "github.com/google/uuid" ) @@ -253,7 +254,11 @@ type ( // (lineage, serial, and deployment ID) always comes from the file, since that // is what the write path increments and carries forward. A nil dmsClient keeps // the behavior file-only. -func (db *DeploymentState) Open(ctx context.Context, path string, withRecovery WithRecovery, withWrite WithWrite, dmsClient bundledeployments.BundleDeploymentsInterface) error { +// +// dmsCfg accompanies dmsClient (both come from the same workspace client) and +// is used only for a temporary raw read of last_successful_version_id; see the +// TODO in deploymentHasSuccessfulVersion. +func (db *DeploymentState) Open(ctx context.Context, path string, withRecovery WithRecovery, withWrite WithWrite, dmsClient bundledeployments.BundleDeploymentsInterface, dmsCfg *sdkconfig.Config) error { db.mu.Lock() defer db.mu.Unlock() @@ -302,7 +307,7 @@ func (db *DeploymentState) Open(ctx context.Context, path string, withRecovery W } if dmsClient != nil && db.Data.DeploymentID != "" { - if err := db.overlayDMSState(ctx, dmsClient); err != nil { + if err := db.overlayDMSState(ctx, dmsClient, dmsCfg); err != nil { return err } } diff --git a/bundle/direct/dstate/state_test.go b/bundle/direct/dstate/state_test.go index e95ad1b0224..16066bf81f8 100644 --- a/bundle/direct/dstate/state_test.go +++ b/bundle/direct/dstate/state_test.go @@ -20,14 +20,14 @@ func TestOpenSaveFinalizeRoundTrip(t *testing.T) { path := filepath.Join(t.TempDir(), "state.json") var db DeploymentState - require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil)) + require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil, nil)) require.NoError(t, db.SaveState("jobs.my_job", "123", map[string]string{"key": "val"}, nil)) mustFinalize(t, &db) // Re-open and verify persisted data. var db2 DeploymentState - require.NoError(t, db2.Open(t.Context(), path, WithRecovery(false), WithWrite(false), nil)) + require.NoError(t, db2.Open(t.Context(), path, WithRecovery(false), WithWrite(false), nil, nil)) assert.Equal(t, 1, db2.Data.Serial) assert.Equal(t, "123", db2.GetResourceID("jobs.my_job")) mustFinalize(t, &db2) @@ -37,7 +37,7 @@ func TestDeploymentIDPersistsAcrossOpen(t *testing.T) { path := filepath.Join(t.TempDir(), "state.json") var db DeploymentState - require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil)) + require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil, nil)) assert.Empty(t, db.GetDeploymentID()) // The deployment ID is set during deploy (after CreateDeployment) and @@ -47,7 +47,7 @@ func TestDeploymentIDPersistsAcrossOpen(t *testing.T) { mustFinalize(t, &db) var reopened DeploymentState - require.NoError(t, reopened.Open(t.Context(), path, WithRecovery(false), WithWrite(false), nil)) + require.NoError(t, reopened.Open(t.Context(), path, WithRecovery(false), WithWrite(false), nil, nil)) assert.Equal(t, "server-assigned-id", reopened.GetDeploymentID()) mustFinalize(t, &reopened) } @@ -56,7 +56,7 @@ func TestFinalizeWithNoEntriesDoesNotWriteStateFile(t *testing.T) { path := filepath.Join(t.TempDir(), "state.json") var db DeploymentState - require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil)) + require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil, nil)) mustFinalize(t, &db) _, err := os.Stat(path) @@ -112,10 +112,10 @@ func TestPanicOnDoubleOpen(t *testing.T) { path := filepath.Join(t.TempDir(), "state.json") var db DeploymentState - require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil)) + require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil, nil)) assert.Panics(t, func() { - _ = db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil) + _ = db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil, nil) }) mustFinalize(t, &db) } @@ -126,12 +126,12 @@ func TestHeaderOnlyWALRecoveryDoesNotAdvanceSerial(t *testing.T) { // Commit serial 1 with one resource. var db DeploymentState - require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil)) + require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil, nil)) require.NoError(t, db.SaveState("jobs.my_job", "123", map[string]string{}, nil)) mustFinalize(t, &db) var committed DeploymentState - require.NoError(t, committed.Open(t.Context(), path, WithRecovery(false), WithWrite(false), nil)) + require.NoError(t, committed.Open(t.Context(), path, WithRecovery(false), WithWrite(false), nil, nil)) lineage := committed.Data.Lineage require.Equal(t, 1, committed.Data.Serial) mustFinalize(t, &committed) @@ -147,7 +147,7 @@ func TestHeaderOnlyWALRecoveryDoesNotAdvanceSerial(t *testing.T) { require.NoError(t, os.WriteFile(walPath, append(headerLine, '\n'), 0o600)) var recovered DeploymentState - require.NoError(t, recovered.Open(t.Context(), path, WithRecovery(true), WithWrite(false), nil)) + require.NoError(t, recovered.Open(t.Context(), path, WithRecovery(true), WithWrite(false), nil, nil)) assert.Equal(t, 1, recovered.Data.Serial) assert.Equal(t, "123", recovered.GetResourceID("jobs.my_job")) assert.NoFileExists(t, walPath) @@ -190,17 +190,17 @@ func TestDeleteState(t *testing.T) { path := filepath.Join(t.TempDir(), "state.json") var db DeploymentState - require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil)) + require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil, nil)) require.NoError(t, db.SaveState("jobs.my_job", "123", map[string]string{}, nil)) mustFinalize(t, &db) var db2 DeploymentState - require.NoError(t, db2.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil)) + require.NoError(t, db2.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil, nil)) require.NoError(t, db2.DeleteState("jobs.my_job")) mustFinalize(t, &db2) var db3 DeploymentState - require.NoError(t, db3.Open(t.Context(), path, WithRecovery(false), WithWrite(false), nil)) + require.NoError(t, db3.Open(t.Context(), path, WithRecovery(false), WithWrite(false), nil, nil)) assert.Equal(t, 2, db3.Data.Serial) assert.Empty(t, db3.GetResourceID("jobs.my_job")) mustFinalize(t, &db3) @@ -212,7 +212,7 @@ func TestGetOrInitLineageReadableBeforeWriteAndPersisted(t *testing.T) { // Fresh state opened read-only, as the deploy does before planning: no // lineage yet. var db DeploymentState - require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(false), nil)) + require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(false), nil, nil)) require.Empty(t, db.Data.Lineage) // GetOrInitLineage initializes the lineage and makes it readable before any @@ -229,7 +229,7 @@ func TestGetOrInitLineageReadableBeforeWriteAndPersisted(t *testing.T) { // Re-open: the persisted lineage matches the one read before the write. var reopened DeploymentState - require.NoError(t, reopened.Open(t.Context(), path, WithRecovery(false), WithWrite(false), nil)) + require.NoError(t, reopened.Open(t.Context(), path, WithRecovery(false), WithWrite(false), nil, nil)) assert.Equal(t, lineage, reopened.Data.Lineage) mustFinalize(t, &reopened) } diff --git a/cmd/bundle/generate/dashboard.go b/cmd/bundle/generate/dashboard.go index 2b286bcad3d..4866f27c5b3 100644 --- a/cmd/bundle/generate/dashboard.go +++ b/cmd/bundle/generate/dashboard.go @@ -404,7 +404,7 @@ func (d *dashboard) runForResource(ctx context.Context, b *bundle.Bundle) { var state statemgmt.ExportedResourcesMap if stateDesc.Engine.IsDirect() { _, localPath := b.StateFilenameDirect(ctx) - if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WithRecovery(true), dstate.WithWrite(false), nil); err != nil { + if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WithRecovery(true), dstate.WithWrite(false), nil, nil); err != nil { logdiag.LogError(ctx, err) return } diff --git a/cmd/bundle/generate/genie_space.go b/cmd/bundle/generate/genie_space.go index 48ecc92a6cd..b5dbeed6c56 100644 --- a/cmd/bundle/generate/genie_space.go +++ b/cmd/bundle/generate/genie_space.go @@ -322,7 +322,7 @@ func (g *genieSpace) runForResource(ctx context.Context, b *bundle.Bundle) { var state statemgmt.ExportedResourcesMap if stateDesc.Engine.IsDirect() { _, localPath := b.StateFilenameDirect(ctx) - if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WithRecovery(true), dstate.WithWrite(false), nil); err != nil { + if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WithRecovery(true), dstate.WithWrite(false), nil, nil); err != nil { logdiag.LogError(ctx, err) return } diff --git a/cmd/bundle/utils/process.go b/cmd/bundle/utils/process.go index 2815f591b22..f556c2b3450 100644 --- a/cmd/bundle/utils/process.go +++ b/cmd/bundle/utils/process.go @@ -25,6 +25,7 @@ import ( "github.com/databricks/cli/libs/logdiag" "github.com/databricks/cli/libs/sync" "github.com/databricks/cli/libs/telemetry/protos" + sdkconfig "github.com/databricks/databricks-sdk-go/config" "github.com/databricks/databricks-sdk-go/service/bundledeployments" "github.com/spf13/cobra" ) @@ -217,11 +218,15 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle // service owns resource state, so hand Open its client to overlay DMS // state on top of the local identity (lineage/serial/deployment ID). // Reads open the state write-disabled, so no lineage is minted here. + // dmsCfg accompanies the client for a temporary raw read (see the TODO + // in dstate.deploymentHasSuccessfulVersion). var dmsClient bundledeployments.BundleDeploymentsInterface + var dmsCfg *sdkconfig.Config if b.Config.Experimental != nil && b.Config.Experimental.RecordDeploymentHistory { dmsClient = b.WorkspaceClient(ctx).BundleDeployments + dmsCfg = b.WorkspaceClient(ctx).Config } - if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WithRecovery(true), dstate.WithWrite(false), dmsClient); err != nil { + if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WithRecovery(true), dstate.WithWrite(false), dmsClient, dmsCfg); err != nil { logdiag.LogError(ctx, err) return b, stateDesc, root.ErrAlreadyPrinted } diff --git a/libs/testserver/bundle.go b/libs/testserver/bundle.go index 5f2c7e0cfd1..34003a507a5 100644 --- a/libs/testserver/bundle.go +++ b/libs/testserver/bundle.go @@ -20,6 +20,12 @@ type dmsDeployment struct { // resources is the latest resource state per resource key, updated as // operations are recorded. resources map[string]bundledeployments.Resource + // lastSuccessfulVersionID is the highest version that completed + // successfully. The server advances last_successful_version_id only on + // success (unlike last_version_id), and the read path treats a non-empty + // value as "DMS owns the state". Tracked separately because the SDK + // Deployment struct does not yet carry the field (still stage:DEVELOPMENT). + lastSuccessfulVersionID string } func (s *FakeWorkspace) CreateDeployment(req Request) Response { @@ -54,7 +60,18 @@ func (s *FakeWorkspace) GetDeployment(deploymentID string) Response { if !ok { return dmsNotFound("deployment " + deploymentID) } - return Response{Body: d.deployment} + + // The SDK Deployment struct does not yet carry last_successful_version_id + // (still stage:DEVELOPMENT, so stripped from generation), but the read path + // reads it off the raw JSON. Serve it as an extra field alongside the typed + // deployment so the overlay behaves as it will against the real server. + return Response{Body: struct { + bundledeployments.Deployment + LastSuccessfulVersionID string `json:"last_successful_version_id,omitempty"` + }{ + Deployment: d.deployment, + LastSuccessfulVersionID: d.lastSuccessfulVersionID, + }} } func (s *FakeWorkspace) DeleteDeployment(deploymentID string) Response { @@ -117,6 +134,9 @@ func (s *FakeWorkspace) CompleteVersion(req Request, deploymentID, versionID str v.Status = bundledeployments.VersionStatusVersionStatusCompleted v.CompletionReason = completeReq.CompletionReason + if completeReq.CompletionReason == bundledeployments.VersionCompleteVersionCompleteSuccess { + d.lastSuccessfulVersionID = versionID + } return Response{Body: *v} } @@ -160,29 +180,6 @@ func (s *FakeWorkspace) CreateOperation(req Request, deploymentID, versionID str return Response{Body: op} } -func (s *FakeWorkspace) ListVersions(deploymentID string) Response { - defer s.LockUnlock()() - - d, ok := s.dmsDeployments[deploymentID] - if !ok { - return dmsNotFound("deployment " + deploymentID) - } - - // The API returns versions newest-first (descending version_id). - ids := make([]int64, 0, len(d.versions)) - for id := range d.versions { - n, _ := strconv.ParseInt(id, 10, 64) - ids = append(ids, n) - } - slices.SortFunc(ids, func(a, b int64) int { return int(b - a) }) - - versions := make([]bundledeployments.Version, 0, len(ids)) - for _, id := range ids { - versions = append(versions, *d.versions[strconv.FormatInt(id, 10)]) - } - return Response{Body: bundledeployments.ListVersionsResponse{Versions: versions}} -} - func (s *FakeWorkspace) ListResources(deploymentID string) Response { defer s.LockUnlock()() diff --git a/libs/testserver/handlers.go b/libs/testserver/handlers.go index 39fe6fbb057..3cbfa16c7e1 100644 --- a/libs/testserver/handlers.go +++ b/libs/testserver/handlers.go @@ -276,9 +276,6 @@ func AddDefaultHandlers(server *Server) { server.Handle("DELETE", "/api/2.0/bundle/deployments/{deployment_id}", func(req Request) any { return req.Workspace.DeleteDeployment(req.Vars["deployment_id"]) }) - server.Handle("GET", "/api/2.0/bundle/deployments/{deployment_id}/versions", func(req Request) any { - return req.Workspace.ListVersions(req.Vars["deployment_id"]) - }) server.Handle("POST", "/api/2.0/bundle/deployments/{deployment_id}/versions", func(req Request) any { return req.Workspace.CreateVersion(req, req.Vars["deployment_id"]) }) From 011c0928c7962098840aa370c1e13508b80447a8 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 27 Jul 2026 00:41:17 +0000 Subject: [PATCH 003/125] bundle: fix DMS deployment recording bugs found in review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Five fixes to the DMS state recording added in #6052: 1. The fake DMS server dropped last_successful_version_id. GetDeployment serialized the response through a struct embedding bundledeployments.Deployment, whose promoted MarshalJSON silently discards sibling fields. The CLI reads a missing value as "DMS does not own the state", so the entire read/overlay path (overlayDMSState, fetchDeploymentResources, deploymentHasSuccessfulVersion) never ran in any test. Serialize through a map instead, and unit-test the shape. 2. A bundle with no resources leaked a deployment record per deploy. Such a deploy writes no WAL entries, and the state file was only persisted when the WAL carried entries, so the server-assigned deployment ID was dropped and the next deploy created a second deployment. Track a dirty header so Finalize persists an ID change on its own. A header-only WAL that changed nothing still skips the write, keeping the serial in step (acceptance/bundle/deploy/wal/header-only-wal). 3. Deploy after destroy failed permanently. A successful destroy deletes the deployment record but leaves its ID in local state, so the next deploy's GetDeployment 404'd and the error was fatal — unrecoverable on retry. Treat a missing deployment as "create a new one"; any other read error stays fatal. 4. The overlay dropped depends_on. DMS does not record dependency edges, so replacing local state with DMS resources lost them, affecting delete ordering, the apply graph, and --select expansion. Carry depends_on over from the local entry. Masked by (1) until now. 5. Recording bypassed secret redaction. dstate.SaveState redacts bundle:"sensitive" fields before writing state, but the operation recorder marshalled raw, so a secret would be sent to DMS in plaintext and read back into local state. Route through structwalk.RedactSensitiveFields. Latent today: no resource state type carries a sensitive field yet. Adds acceptance coverage for deploy-destroy-deploy and for a bundle with no resources, both of which now exercise the read path (visible as GET .../resources in the recorded requests). Co-authored-by: Isaac --- .../bundle/dms/no-resources/databricks.yml | 5 + .../bundle/dms/no-resources/out.test.toml | 3 + acceptance/bundle/dms/no-resources/output.txt | 78 ++++++++++++++++ acceptance/bundle/dms/no-resources/script | 8 ++ .../dms/redeploy-after-destroy/databricks.yml | 10 ++ .../dms/redeploy-after-destroy/out.test.toml | 3 + .../dms/redeploy-after-destroy/output.txt | 92 +++++++++++++++++++ .../bundle/dms/redeploy-after-destroy/script | 15 +++ bundle/direct/dstate/dms.go | 14 ++- bundle/direct/dstate/dms_test.go | 74 +++++++++++++++ bundle/direct/dstate/state.go | 42 +++++++-- bundle/direct/dstate/state_test.go | 42 +++++++++ bundle/direct/oprecorder.go | 9 +- bundle/direct/oprecorder_test.go | 22 +++++ libs/dms/recorder.go | 44 +++++---- libs/dms/recorder_test.go | 38 ++++++++ libs/testserver/bundle.go | 44 ++++++--- libs/testserver/bundle_test.go | 51 ++++++++++ 18 files changed, 552 insertions(+), 42 deletions(-) create mode 100644 acceptance/bundle/dms/no-resources/databricks.yml create mode 100644 acceptance/bundle/dms/no-resources/out.test.toml create mode 100644 acceptance/bundle/dms/no-resources/output.txt create mode 100644 acceptance/bundle/dms/no-resources/script create mode 100644 acceptance/bundle/dms/redeploy-after-destroy/databricks.yml create mode 100644 acceptance/bundle/dms/redeploy-after-destroy/out.test.toml create mode 100644 acceptance/bundle/dms/redeploy-after-destroy/output.txt create mode 100644 acceptance/bundle/dms/redeploy-after-destroy/script create mode 100644 bundle/direct/dstate/dms_test.go create mode 100644 libs/testserver/bundle_test.go diff --git a/acceptance/bundle/dms/no-resources/databricks.yml b/acceptance/bundle/dms/no-resources/databricks.yml new file mode 100644 index 00000000000..78fad3a292e --- /dev/null +++ b/acceptance/bundle/dms/no-resources/databricks.yml @@ -0,0 +1,5 @@ +bundle: + name: dms-no-resources + +experimental: + record_deployment_history: true diff --git a/acceptance/bundle/dms/no-resources/out.test.toml b/acceptance/bundle/dms/no-resources/out.test.toml new file mode 100644 index 00000000000..e90b6d5d1ba --- /dev/null +++ b/acceptance/bundle/dms/no-resources/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/dms/no-resources/output.txt b/acceptance/bundle/dms/no-resources/output.txt new file mode 100644 index 00000000000..e774fd7546c --- /dev/null +++ b/acceptance/bundle/dms/no-resources/output.txt @@ -0,0 +1,78 @@ + +=== First deploy of a bundle with no resources: the deployment is created and its ID is persisted, even though no resource state was written +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-no-resources/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> print_requests.py //api/2.0/bundle --sort --get +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments", + "body": { + "target_name": "default" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[UUID]/versions", + "q": { + "version_id": "1" + }, + "body": { + "cli_version": "[CLI_VERSION]", + "target_name": "default", + "version_type": "VERSION_TYPE_DEPLOY" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[UUID]/versions/1/complete", + "body": { + "completion_reason": "VERSION_COMPLETE_SUCCESS" + } +} + +>>> jq .deployment_id .databricks/bundle/default/resources.json +"[UUID]" + +=== Redeploy: the persisted ID is reused, so no second deployment is created +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-no-resources/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> print_requests.py //api/2.0/bundle --sort --get +{ + "method": "GET", + "path": "/api/2.0/bundle/deployments/[UUID]" +} +{ + "method": "GET", + "path": "/api/2.0/bundle/deployments/[UUID]" +} +{ + "method": "GET", + "path": "/api/2.0/bundle/deployments/[UUID]/resources" +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[UUID]/versions", + "q": { + "version_id": "2" + }, + "body": { + "cli_version": "[CLI_VERSION]", + "target_name": "default", + "version_type": "VERSION_TYPE_DEPLOY" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[UUID]/versions/2/complete", + "body": { + "completion_reason": "VERSION_COMPLETE_SUCCESS" + } +} diff --git a/acceptance/bundle/dms/no-resources/script b/acceptance/bundle/dms/no-resources/script new file mode 100644 index 00000000000..4bc97c5864d --- /dev/null +++ b/acceptance/bundle/dms/no-resources/script @@ -0,0 +1,8 @@ +title "First deploy of a bundle with no resources: the deployment is created and its ID is persisted, even though no resource state was written" +trace $CLI bundle deploy +trace print_requests.py //api/2.0/bundle --sort --get +trace jq .deployment_id .databricks/bundle/default/resources.json + +title "Redeploy: the persisted ID is reused, so no second deployment is created" +trace $CLI bundle deploy +trace print_requests.py //api/2.0/bundle --sort --get diff --git a/acceptance/bundle/dms/redeploy-after-destroy/databricks.yml b/acceptance/bundle/dms/redeploy-after-destroy/databricks.yml new file mode 100644 index 00000000000..8f79a2c0381 --- /dev/null +++ b/acceptance/bundle/dms/redeploy-after-destroy/databricks.yml @@ -0,0 +1,10 @@ +bundle: + name: dms-redeploy-after-destroy + +experimental: + record_deployment_history: true + +resources: + jobs: + foo: + name: foo diff --git a/acceptance/bundle/dms/redeploy-after-destroy/out.test.toml b/acceptance/bundle/dms/redeploy-after-destroy/out.test.toml new file mode 100644 index 00000000000..e90b6d5d1ba --- /dev/null +++ b/acceptance/bundle/dms/redeploy-after-destroy/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/dms/redeploy-after-destroy/output.txt b/acceptance/bundle/dms/redeploy-after-destroy/output.txt new file mode 100644 index 00000000000..5326653519a --- /dev/null +++ b/acceptance/bundle/dms/redeploy-after-destroy/output.txt @@ -0,0 +1,92 @@ + +=== Deploy, then destroy: the deployment record is deleted, but its ID stays behind in the local state file +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-redeploy-after-destroy/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.jobs.foo + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/dms-redeploy-after-destroy/default + +Deleting files... +Destroy complete! + +>>> jq .deployment_id .databricks/bundle/default/resources.json +"[DESTROYED_DEPLOYMENT_ID]" + +=== Deploy again: the stale deployment ID no longer resolves, so a new deployment is created instead of failing the deploy +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-redeploy-after-destroy/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> print_requests.py //api/2.0/bundle --sort --get +{ + "method": "GET", + "path": "/api/2.0/bundle/deployments/[DESTROYED_DEPLOYMENT_ID]" +} +{ + "method": "GET", + "path": "/api/2.0/bundle/deployments/[DESTROYED_DEPLOYMENT_ID]" +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments", + "body": { + "target_name": "default" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[UUID]/versions", + "q": { + "version_id": "1" + }, + "body": { + "cli_version": "[CLI_VERSION]", + "target_name": "default", + "version_type": "VERSION_TYPE_DEPLOY" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[UUID]/versions/1/complete", + "body": { + "completion_reason": "VERSION_COMPLETE_SUCCESS" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[UUID]/versions/1/operations", + "q": { + "resource_key": "jobs.foo" + }, + "body": { + "action_type": "OPERATION_ACTION_TYPE_CREATE", + "resource_id": "[NUMID]", + "resource_key": "jobs.foo", + "state": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-redeploy-after-destroy/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "foo", + "queue": { + "enabled": true + } + }, + "status": "OPERATION_STATUS_SUCCEEDED" + } +} + +=== The new deployment ID replaces the stale one in state +>>> jq .deployment_id .databricks/bundle/default/resources.json +"[UUID]" diff --git a/acceptance/bundle/dms/redeploy-after-destroy/script b/acceptance/bundle/dms/redeploy-after-destroy/script new file mode 100644 index 00000000000..628cd7bf494 --- /dev/null +++ b/acceptance/bundle/dms/redeploy-after-destroy/script @@ -0,0 +1,15 @@ +title "Deploy, then destroy: the deployment record is deleted, but its ID stays behind in the local state file" +trace $CLI bundle deploy +trace $CLI bundle destroy --auto-approve +print_requests.py //api/2.0/bundle --sort --get > /dev/null + +destroyed_id=$(jq -r .deployment_id .databricks/bundle/default/resources.json) +add_repl.py "$destroyed_id" DESTROYED_DEPLOYMENT_ID +trace jq .deployment_id .databricks/bundle/default/resources.json + +title "Deploy again: the stale deployment ID no longer resolves, so a new deployment is created instead of failing the deploy" +trace $CLI bundle deploy +trace print_requests.py //api/2.0/bundle --sort --get + +title "The new deployment ID replaces the stale one in state" +trace jq .deployment_id .databricks/bundle/default/resources.json diff --git a/bundle/direct/dstate/dms.go b/bundle/direct/dstate/dms.go index 659d7c9dd0b..e09cb859221 100644 --- a/bundle/direct/dstate/dms.go +++ b/bundle/direct/dstate/dms.go @@ -35,7 +35,7 @@ func (db *DeploymentState) overlayDMSState(ctx context.Context, client bundledep return nil } - resources, err := fetchDeploymentResources(ctx, client, db.Data.DeploymentID) + resources, err := fetchDeploymentResources(ctx, client, db.Data.DeploymentID, db.Data.State) if err != nil { return err } @@ -91,7 +91,12 @@ func deploymentHasSuccessfulVersion(ctx context.Context, cfg *sdkconfig.Config, // fetchDeploymentResources lists every resource recorded for the deployment in // DMS and maps them into state entries keyed by the fully-qualified resource key. -func fetchDeploymentResources(ctx context.Context, client bundledeployments.BundleDeploymentsInterface, deploymentID string) (map[string]ResourceEntry, error) { +// +// DMS does not record dependency edges, so depends_on is carried over from the +// local state entry for the same key. It is derived from the local config on +// every deploy and is only consumed for delete ordering, so falling back to an +// empty list when the local state has no entry is safe. +func fetchDeploymentResources(ctx context.Context, client bundledeployments.BundleDeploymentsInterface, deploymentID string, local map[string]ResourceEntry) (map[string]ResourceEntry, error) { it := client.ListResources(ctx, bundledeployments.ListResourcesRequest{ Parent: "deployments/" + deploymentID, }) @@ -114,8 +119,9 @@ func fetchDeploymentResources(ctx context.Context, client bundledeployments.Bund } out[key] = ResourceEntry{ - ID: res.ResourceId, - State: state, + ID: res.ResourceId, + State: state, + DependsOn: local[key].DependsOn, } } return out, nil diff --git a/bundle/direct/dstate/dms_test.go b/bundle/direct/dstate/dms_test.go new file mode 100644 index 00000000000..df1084b9de7 --- /dev/null +++ b/bundle/direct/dstate/dms_test.go @@ -0,0 +1,74 @@ +package dstate + +import ( + "context" + "encoding/json" + "testing" + + "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/databricks-sdk-go/listing" + "github.com/databricks/databricks-sdk-go/service/bundledeployments" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// fakeResourceLister serves a fixed set of resources from ListResources. It +// embeds the SDK interface so it satisfies it while only overriding the one +// method the read path uses. +type fakeResourceLister struct { + bundledeployments.BundleDeploymentsInterface + resources []bundledeployments.Resource +} + +func (f *fakeResourceLister) ListResources(ctx context.Context, req bundledeployments.ListResourcesRequest) listing.Iterator[bundledeployments.Resource] { + return listing.NewIterator( + &req, + func(ctx context.Context, r bundledeployments.ListResourcesRequest) (*bundledeployments.ListResourcesResponse, error) { + return &bundledeployments.ListResourcesResponse{Resources: f.resources}, nil + }, + func(resp *bundledeployments.ListResourcesResponse) []bundledeployments.Resource { + return resp.Resources + }, + func(resp *bundledeployments.ListResourcesResponse) *bundledeployments.ListResourcesRequest { + return nil + }, + ) +} + +func TestFetchDeploymentResourcesPreservesLocalDependsOn(t *testing.T) { + state := json.RawMessage(`{"name":"foo"}`) + f := &fakeResourceLister{resources: []bundledeployments.Resource{ + {ResourceKey: "jobs.foo", ResourceId: "123", State: &state}, + {ResourceKey: "pipelines.bar", ResourceId: "456"}, + }} + + dependsOn := []deployplan.DependsOnEntry{{Node: "resources.pipelines.bar", Label: "pipeline_id"}} + local := map[string]ResourceEntry{ + "resources.jobs.foo": {ID: "stale", DependsOn: dependsOn}, + } + + got, err := fetchDeploymentResources(t.Context(), f, "dep-1", local) + require.NoError(t, err) + + // DMS owns the ID and state, but it does not record dependency edges, so + // depends_on must survive from the local entry. Losing it breaks delete + // ordering and --select expansion. + assert.Equal(t, map[string]ResourceEntry{ + "resources.jobs.foo": {ID: "123", State: state, DependsOn: dependsOn}, + "resources.pipelines.bar": {ID: "456"}, + }, got) +} + +func TestFetchDeploymentResourcesWithNoLocalState(t *testing.T) { + f := &fakeResourceLister{resources: []bundledeployments.Resource{ + {ResourceKey: "jobs.foo", ResourceId: "123"}, + }} + + // A bundle whose local state was wiped has no entry to carry depends_on from; + // the resource is still recovered from DMS. + got, err := fetchDeploymentResources(t.Context(), f, "dep-1", nil) + require.NoError(t, err) + assert.Equal(t, map[string]ResourceEntry{ + "resources.jobs.foo": {ID: "123"}, + }, got) +} diff --git a/bundle/direct/dstate/state.go b/bundle/direct/dstate/state.go index 2c969667c9e..f554af3c9b6 100644 --- a/bundle/direct/dstate/state.go +++ b/bundle/direct/dstate/state.go @@ -74,6 +74,11 @@ type DeploymentState struct { // Maps resource key to ID. Unlike Data.State, this is up to date during writes (deploys). stateIDs map[string]string + + // headerDirty records that a header field changed in memory during this + // deployment (today only the DMS deployment ID), so the state file must be + // written on Finalize even when the WAL carried no resource entries. + headerDirty bool } type Header struct { @@ -231,10 +236,18 @@ func (db *DeploymentState) GetDeploymentID() string { // server-generated ID, and persisted to the state file by Finalize. Storing it // on db.Data (not the WAL header, which is written before the ID is known) // means the subsequent state write carries it forward. +// +// The header is marked dirty so Finalize persists it even when the deploy wrote +// no resource entries; otherwise a bundle with no resources would mint a fresh +// deployment record on every deploy, leaking one orphan per run. func (db *DeploymentState) SetDeploymentID(id string) { db.mu.Lock() defer db.mu.Unlock() + if db.Data.DeploymentID == id { + return + } db.Data.DeploymentID = id + db.headerDirty = true } type ( @@ -353,7 +366,7 @@ func (db *DeploymentState) OpenWithData(path string, data Database) { func (db *DeploymentState) replayWAL(ctx context.Context) error { walPath := db.Path + walSuffix - hasEntries, err := db.mergeWalIntoState(ctx) + persist, err := db.mergeWalIntoState(ctx) if err != nil { if errors.Is(err, errStaleWAL) { log.Debugf(ctx, "Deleting stale WAL file %s", walPath) @@ -362,7 +375,7 @@ func (db *DeploymentState) replayWAL(ctx context.Context) error { } return fmt.Errorf("WAL recovery failed: %w", err) } - if hasEntries { + if persist { if err := db.unlockedSave(); err != nil { return err } @@ -373,6 +386,9 @@ func (db *DeploymentState) replayWAL(ctx context.Context) error { return nil } +// mergeWalIntoState replays the WAL into db.Data and reports whether the caller +// must persist the state file: either the WAL carried resource entries, or a +// header field changed in memory during this deployment. func (db *DeploymentState) mergeWalIntoState(ctx context.Context) (bool, error) { if db.walFile != nil { panic("internal error: walFile must be closed") @@ -450,17 +466,23 @@ func (db *DeploymentState) mergeWalIntoState(ctx context.Context) (bool, error) hasEntries := lineNumber > 1 - // Only advance the serial when the WAL carried entries, because the caller - // (replayWAL) persists the new state file only in that case. A header-only - // WAL is a deploy that started but committed nothing; advancing the serial - // for it leaves the in-memory serial ahead of the persisted one, so the - // next deploy writes its WAL header at serial+2 and recovery rejects it as - // "ahead of expected". See acceptance/bundle/deploy/wal/header-only-wal. - if hasEntries { + // A header-only WAL still has to be persisted when a header field changed in + // memory during this deployment (the DMS deployment ID): dropping the write + // would lose the ID and make the next deploy create a second deployment. + persist := hasEntries || db.headerDirty + + // Only advance the serial when the state file is actually written, because + // the caller (replayWAL) persists it only in that case. A header-only WAL + // that changed nothing is a deploy that started but committed nothing; + // advancing the serial for it leaves the in-memory serial ahead of the + // persisted one, so the next deploy writes its WAL header at serial+2 and + // recovery rejects it as "ahead of expected". + // See acceptance/bundle/deploy/wal/header-only-wal. + if persist { db.Data.Serial = newSerial } - return hasEntries, nil + return persist, nil } // Finalize replays the WAL (if open for write), captures the resulting state, and resets. diff --git a/bundle/direct/dstate/state_test.go b/bundle/direct/dstate/state_test.go index 16066bf81f8..6530ca049d9 100644 --- a/bundle/direct/dstate/state_test.go +++ b/bundle/direct/dstate/state_test.go @@ -63,6 +63,48 @@ func TestFinalizeWithNoEntriesDoesNotWriteStateFile(t *testing.T) { assert.ErrorIs(t, err, os.ErrNotExist) } +func TestDeploymentIDPersistsWithNoResourceEntries(t *testing.T) { + path := filepath.Join(t.TempDir(), "state.json") + + // A bundle with no resources writes no WAL entries, but the deployment ID + // still has to be persisted: otherwise the next deploy sees no ID and creates + // a second deployment record, leaking one per deploy. + var db DeploymentState + require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil, nil)) + db.SetDeploymentID("server-assigned-id") + mustFinalize(t, &db) + + var reopened DeploymentState + require.NoError(t, reopened.Open(t.Context(), path, WithRecovery(false), WithWrite(false), nil, nil)) + assert.Equal(t, "server-assigned-id", reopened.GetDeploymentID()) + assert.Equal(t, 1, reopened.Data.Serial) + mustFinalize(t, &reopened) +} + +func TestSetDeploymentIDToSameValueDoesNotWriteStateFile(t *testing.T) { + path := filepath.Join(t.TempDir(), "state.json") + + var db DeploymentState + require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil, nil)) + db.SetDeploymentID("server-assigned-id") + require.NoError(t, db.SaveState("jobs.my_job", "123", map[string]string{}, nil)) + mustFinalize(t, &db) + + before, err := os.ReadFile(path) + require.NoError(t, err) + + // Re-setting the same ID is not a header change, so a deploy that commits + // nothing must not bump the serial (see mergeWalIntoState). + var reopened DeploymentState + require.NoError(t, reopened.Open(t.Context(), path, WithRecovery(false), WithWrite(true), nil, nil)) + reopened.SetDeploymentID("server-assigned-id") + mustFinalize(t, &reopened) + + after, err := os.ReadFile(path) + require.NoError(t, err) + assert.Equal(t, string(before), string(after)) +} + func TestExportStateFromDataJobRunJobID(t *testing.T) { data := Database{ State: map[string]ResourceEntry{ diff --git a/bundle/direct/oprecorder.go b/bundle/direct/oprecorder.go index 467f8ac648c..033aefb5f2f 100644 --- a/bundle/direct/oprecorder.go +++ b/bundle/direct/oprecorder.go @@ -7,6 +7,8 @@ import ( "strings" "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structwalk" "github.com/databricks/databricks-sdk-go/service/bundledeployments" ) @@ -67,8 +69,13 @@ func (r *operationRecorder) record(ctx context.Context, resourceKey string, acti // The DMS Operation.State field carries the serialized config so the backend // can serve it as resource state. It is intentionally left unset for delete, // where the resource no longer exists. + // + // Redact sensitive fields, matching what dstate.SaveState writes to the local + // state file: DMS state is read back as resource state, so recording secrets + // in plaintext would both leak them to the service and reintroduce them into + // a local state file via the read path. if state != nil { - raw, err := json.Marshal(state) + raw, err := structwalk.RedactSensitiveFields(state, dyn.SensitiveValueRedacted) if err != nil { return fmt.Errorf("serializing state: %w", err) } diff --git a/bundle/direct/oprecorder_test.go b/bundle/direct/oprecorder_test.go index 56d860de3e6..4c1bbcacda6 100644 --- a/bundle/direct/oprecorder_test.go +++ b/bundle/direct/oprecorder_test.go @@ -5,6 +5,7 @@ import ( "testing" "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/cli/libs/dyn" "github.com/databricks/databricks-sdk-go/service/bundledeployments" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -52,6 +53,27 @@ func TestOperationRecorderDeleteHasNoState(t *testing.T) { assert.Nil(t, f.requests[0].Operation.State) } +func TestOperationRecorderRedactsSensitiveFields(t *testing.T) { + f := &fakeOpClient{} + r := NewOperationRecorder(f, "dep-1", 2) + + state := struct { + Name string `json:"name"` + Token string `json:"token" bundle:"sensitive"` + }{Name: "foo", Token: "super-secret"} + + err := r.record(t.Context(), "resources.jobs.foo", deployplan.Create, "job-123", state) + require.NoError(t, err) + + require.Len(t, f.requests, 1) + require.NotNil(t, f.requests[0].Operation.State) + // Sensitive fields are redacted before leaving the CLI, matching what + // dstate.SaveState writes to the local state file. + assert.JSONEq(t, + `{"name":"foo","token":"`+dyn.SensitiveValueRedacted+`"}`, + string(*f.requests[0].Operation.State)) +} + func TestDeployActionToSDK(t *testing.T) { cases := []struct { action deployplan.ActionType diff --git a/libs/dms/recorder.go b/libs/dms/recorder.go index eed8485f2c2..0d9563dd052 100644 --- a/libs/dms/recorder.go +++ b/libs/dms/recorder.go @@ -149,10 +149,35 @@ func (r *Recorder) CompleteVersion(ctx context.Context, success bool) error { } // createDeploymentVersion ensures the deployment record exists, then creates a -// new version under it. On a first deploy (no stored deployment ID) it creates -// the deployment and lets the server assign the ID; otherwise it reads the -// existing deployment to compute the next version number. +// new version under it. When no deployment ID is stored, or the stored one no +// longer exists in DMS, it creates the deployment and lets the server assign the +// ID; otherwise it reads the existing deployment to compute the next version +// number. func (r *Recorder) createDeploymentVersion(ctx context.Context) (versionID string, err error) { + if r.deploymentID != "" { + // Existing deployment: read it to compute the next version number. + dep, getErr := r.svc.GetDeployment(ctx, bundledeployments.GetDeploymentRequest{ + Name: "deployments/" + r.deploymentID, + }) + switch { + case getErr == nil: + lastVersion, parseErr := strconv.ParseInt(dep.LastVersionId, 10, 64) + if parseErr != nil { + return "", fmt.Errorf("failed to parse last_version_id %q: %w", dep.LastVersionId, parseErr) + } + versionID = strconv.FormatInt(lastVersion+1, 10) + case errors.Is(getErr, apierr.ErrNotFound): + // The record the state points at is gone: a successful destroy deletes + // it (leaving the ID behind in the local state file), and it can also be + // deleted out of band. Recording must not dead-end on it, so fall back to + // creating a new deployment; the caller persists the new ID. + log.Debugf(ctx, "Deployment %s no longer exists in the deployment metadata service, creating a new one", r.deploymentID) + r.deploymentID = "" + default: + return "", fmt.Errorf("failed to get deployment: %w", getErr) + } + } + if r.deploymentID == "" { // First deploy: create the deployment with an empty ID so the server // assigns one, then start at version 1. @@ -170,19 +195,6 @@ func (r *Recorder) createDeploymentVersion(ctx context.Context) (versionID strin } r.deploymentID = id versionID = "1" - } else { - // Existing deployment: read it to compute the next version number. - dep, getErr := r.svc.GetDeployment(ctx, bundledeployments.GetDeploymentRequest{ - Name: "deployments/" + r.deploymentID, - }) - if getErr != nil { - return "", fmt.Errorf("failed to get deployment: %w", getErr) - } - lastVersion, parseErr := strconv.ParseInt(dep.LastVersionId, 10, 64) - if parseErr != nil { - return "", fmt.Errorf("failed to parse last_version_id %q: %w", dep.LastVersionId, parseErr) - } - versionID = strconv.FormatInt(lastVersion+1, 10) } // The server validates that versionID equals last_version_id + 1 and returns diff --git a/libs/dms/recorder_test.go b/libs/dms/recorder_test.go index 91848f74a70..9b60078635f 100644 --- a/libs/dms/recorder_test.go +++ b/libs/dms/recorder_test.go @@ -2,8 +2,11 @@ package dms import ( "context" + "errors" + "fmt" "testing" + "github.com/databricks/databricks-sdk-go/apierr" "github.com/databricks/databricks-sdk-go/service/bundledeployments" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -104,6 +107,41 @@ func TestRecorderSubsequentDeployReusesDeploymentAndIncrementsVersion(t *testing assert.Equal(t, "stored-id", r.DeploymentID()) } +func TestRecorderStaleDeploymentIDCreatesNewDeployment(t *testing.T) { + f := &fakeDMS{ + assignedID: "fresh-id", + getDeployment: func(id string) (*bundledeployments.Deployment, error) { + return nil, fmt.Errorf("deployment %s: %w", id, apierr.ErrNotFound) + }, + } + // A deploy after a destroy still has the destroyed deployment's ID in state, + // but the record is gone. Recording must recover rather than fail the deploy. + r := NewRecorder(f, "destroyed-id", "dev", VersionTypeDeploy) + + require.NoError(t, r.CreateVersion(t.Context())) + + require.Len(t, f.created, 1) + assert.Equal(t, "fresh-id", r.DeploymentID()) + require.Len(t, f.versions, 1) + assert.Equal(t, "1", f.versions[0].VersionId) + assert.Equal(t, "deployments/fresh-id", f.versions[0].Parent) +} + +func TestRecorderGetDeploymentErrorFailsDeploy(t *testing.T) { + f := &fakeDMS{ + getDeployment: func(id string) (*bundledeployments.Deployment, error) { + return nil, errors.New("boom") + }, + } + r := NewRecorder(f, "stored-id", "dev", VersionTypeDeploy) + + // Only a missing deployment is recovered from; any other read failure is fatal + // rather than silently forking a second deployment record. + err := r.CreateVersion(t.Context()) + assert.ErrorContains(t, err, "failed to get deployment") + assert.Empty(t, f.created) +} + func TestRecorderDestroyDeletesDeploymentOnSuccess(t *testing.T) { f := &fakeDMS{ getDeployment: func(id string) (*bundledeployments.Deployment, error) { diff --git a/libs/testserver/bundle.go b/libs/testserver/bundle.go index 34003a507a5..a1b0cba24a9 100644 --- a/libs/testserver/bundle.go +++ b/libs/testserver/bundle.go @@ -1,6 +1,7 @@ package testserver import ( + "bytes" "encoding/json" "slices" "strconv" @@ -61,17 +62,38 @@ func (s *FakeWorkspace) GetDeployment(deploymentID string) Response { return dmsNotFound("deployment " + deploymentID) } - // The SDK Deployment struct does not yet carry last_successful_version_id - // (still stage:DEVELOPMENT, so stripped from generation), but the read path - // reads it off the raw JSON. Serve it as an extra field alongside the typed - // deployment so the overlay behaves as it will against the real server. - return Response{Body: struct { - bundledeployments.Deployment - LastSuccessfulVersionID string `json:"last_successful_version_id,omitempty"` - }{ - Deployment: d.deployment, - LastSuccessfulVersionID: d.lastSuccessfulVersionID, - }} + body, err := deploymentBody(d) + if err != nil { + return Response{StatusCode: 500, Body: map[string]string{"message": err.Error()}} + } + return Response{Body: body} +} + +// deploymentBody renders a deployment the way the real server does: the typed +// fields plus last_successful_version_id, which the generated SDK struct does +// not carry yet (still stage:DEVELOPMENT) but the read path reads off the raw +// JSON. +// +// The extra field cannot be added by embedding Deployment in a wrapper struct: +// Deployment has its own MarshalJSON, which is promoted to the wrapper and +// silently drops any sibling field. +func deploymentBody(d *dmsDeployment) (map[string]any, error) { + raw, err := json.Marshal(d.deployment) + if err != nil { + return nil, err + } + + var body map[string]any + dec := json.NewDecoder(bytes.NewReader(raw)) + dec.UseNumber() + if err := dec.Decode(&body); err != nil { + return nil, err + } + + if d.lastSuccessfulVersionID != "" { + body["last_successful_version_id"] = d.lastSuccessfulVersionID + } + return body, nil } func (s *FakeWorkspace) DeleteDeployment(deploymentID string) Response { diff --git a/libs/testserver/bundle_test.go b/libs/testserver/bundle_test.go new file mode 100644 index 00000000000..4d28624ba3e --- /dev/null +++ b/libs/testserver/bundle_test.go @@ -0,0 +1,51 @@ +package testserver + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestDeploymentBodyKeepsTypedFieldsAndLastSuccessfulVersionID guards against +// serializing the deployment through a struct that embeds +// bundledeployments.Deployment: Deployment has its own MarshalJSON, which is +// promoted to the embedding struct and silently drops last_successful_version_id. +// The CLI read path treats a missing value as "DMS does not own the state", so +// losing the field here makes the whole overlay path untestable. +func TestDeploymentBodyKeepsTypedFieldsAndLastSuccessfulVersionID(t *testing.T) { + d := &dmsDeployment{lastSuccessfulVersionID: "2"} + d.deployment.Name = "deployments/abc" + d.deployment.LastVersionId = "3" + d.deployment.TargetName = "default" + + body, err := deploymentBody(d) + require.NoError(t, err) + + assert.Equal(t, "deployments/abc", body["name"]) + assert.Equal(t, "3", body["last_version_id"]) + assert.Equal(t, "default", body["target_name"]) + assert.Equal(t, "2", body["last_successful_version_id"]) + + // The response must round-trip as JSON the same way, since that is what the + // client actually reads. + raw, err := json.Marshal(body) + require.NoError(t, err) + assert.JSONEq(t, + `{"name":"deployments/abc","last_version_id":"3","target_name":"default","last_successful_version_id":"2"}`, + string(raw)) +} + +// TestDeploymentBodyOmitsUnsetLastSuccessfulVersionID checks that a deployment +// with no successful version does not advertise one: the read path must keep +// using the local state file in that case. +func TestDeploymentBodyOmitsUnsetLastSuccessfulVersionID(t *testing.T) { + d := &dmsDeployment{} + d.deployment.Name = "deployments/abc" + + body, err := deploymentBody(d) + require.NoError(t, err) + + assert.NotContains(t, body, "last_successful_version_id") +} From 38dc0823b4e97cf6caac784d8041f8a35339b51e Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 27 Jul 2026 13:09:47 +0000 Subject: [PATCH 004/125] bundle: upload DMS operations asynchronously Recording an operation with the deployment metadata service used to happen inline on the apply worker, so every resource paid a CreateOperation round trip before the worker moved on to the next one. Queue the operations instead and upload them from a small pool of background workers (operationQueue, in the new opqueue.go). The queue holds resource keys rather than operations, so an operation recorded for a resource that is still waiting replaces the queued one: DMS keeps one state per resource key, so the later operation supersedes the earlier one and a single request records both. This is best effort - only operations that no worker has picked up yet are coalesced. Uploads are not fire-and-forget. Apply drains the queue before returning and reports the first failure, because a version that completes successfully makes DMS authoritative for resource state; dropping an operation would leave DMS with an incomplete resource set and the next deploy would plan to create resources that already exist. At most one upload per resource key runs at a time, so the last operation recorded for a resource is also the last one the service sees. Co-authored-by: Isaac --- .../dms/multiple-resources/databricks.yml | 18 ++ .../dms/multiple-resources/out.test.toml | 3 + .../bundle/dms/multiple-resources/output.txt | 25 +++ .../bundle/dms/multiple-resources/script | 7 + bundle/direct/bundle_apply.go | 16 +- bundle/direct/opqueue.go | 192 ++++++++++++++++++ bundle/direct/opqueue_test.go | 192 ++++++++++++++++++ bundle/direct/oprecorder.go | 103 ++++++---- bundle/direct/oprecorder_test.go | 42 ++-- bundle/direct/pkg.go | 6 +- 10 files changed, 537 insertions(+), 67 deletions(-) create mode 100644 acceptance/bundle/dms/multiple-resources/databricks.yml create mode 100644 acceptance/bundle/dms/multiple-resources/out.test.toml create mode 100644 acceptance/bundle/dms/multiple-resources/output.txt create mode 100644 acceptance/bundle/dms/multiple-resources/script create mode 100644 bundle/direct/opqueue.go create mode 100644 bundle/direct/opqueue_test.go diff --git a/acceptance/bundle/dms/multiple-resources/databricks.yml b/acceptance/bundle/dms/multiple-resources/databricks.yml new file mode 100644 index 00000000000..30f2f433b81 --- /dev/null +++ b/acceptance/bundle/dms/multiple-resources/databricks.yml @@ -0,0 +1,18 @@ +bundle: + name: dms-multiple-resources + +experimental: + record_deployment_history: true + +resources: + jobs: + one: + name: one + two: + name: two + three: + name: three + four: + name: four + five: + name: five diff --git a/acceptance/bundle/dms/multiple-resources/out.test.toml b/acceptance/bundle/dms/multiple-resources/out.test.toml new file mode 100644 index 00000000000..e90b6d5d1ba --- /dev/null +++ b/acceptance/bundle/dms/multiple-resources/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/dms/multiple-resources/output.txt b/acceptance/bundle/dms/multiple-resources/output.txt new file mode 100644 index 00000000000..4bacec0c317 --- /dev/null +++ b/acceptance/bundle/dms/multiple-resources/output.txt @@ -0,0 +1,25 @@ + +=== Deploy several resources: operations are uploaded from background workers, so exactly one is recorded per resource no matter what order the uploads finish in. The serialized state is dropped from the output here; bundle/dms/record covers it. +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> print_requests.py //versions/1/operations --sort --del-body state --oneline +{"method": "POST", "path": "/api/2.0/bundle/deployments/[UUID]/versions/1/operations", "q": {"resource_key": "jobs.five"}, "body": {"action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_id": "[NUMID]", "resource_key": "jobs.five", "status": "OPERATION_STATUS_SUCCEEDED"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[UUID]/versions/1/operations", "q": {"resource_key": "jobs.four"}, "body": {"action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_id": "[NUMID]", "resource_key": "jobs.four", "status": "OPERATION_STATUS_SUCCEEDED"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[UUID]/versions/1/operations", "q": {"resource_key": "jobs.one"}, "body": {"action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_id": "[NUMID]", "resource_key": "jobs.one", "status": "OPERATION_STATUS_SUCCEEDED"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[UUID]/versions/1/operations", "q": {"resource_key": "jobs.three"}, "body": {"action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_id": "[NUMID]", "resource_key": "jobs.three", "status": "OPERATION_STATUS_SUCCEEDED"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[UUID]/versions/1/operations", "q": {"resource_key": "jobs.two"}, "body": {"action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_id": "[NUMID]", "resource_key": "jobs.two", "status": "OPERATION_STATUS_SUCCEEDED"}} + +=== Redeploy with no changes: nothing is applied, so no operations are recorded and only the version is opened and completed +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> print_requests.py //api/2.0/bundle --sort --oneline +{"method": "POST", "path": "/api/2.0/bundle/deployments/[UUID]/versions", "q": {"version_id": "2"}, "body": {"cli_version": "[CLI_VERSION]", "target_name": "default", "version_type": "VERSION_TYPE_DEPLOY"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[UUID]/versions/2/complete", "body": {"completion_reason": "VERSION_COMPLETE_SUCCESS"}} diff --git a/acceptance/bundle/dms/multiple-resources/script b/acceptance/bundle/dms/multiple-resources/script new file mode 100644 index 00000000000..e9b9339607e --- /dev/null +++ b/acceptance/bundle/dms/multiple-resources/script @@ -0,0 +1,7 @@ +title "Deploy several resources: operations are uploaded from background workers, so exactly one is recorded per resource no matter what order the uploads finish in. The serialized state is dropped from the output here; bundle/dms/record covers it." +trace $CLI bundle deploy +trace print_requests.py //versions/1/operations --sort --del-body state --oneline + +title "Redeploy with no changes: nothing is applied, so no operations are recorded and only the version is opened and completed" +trace $CLI bundle deploy +trace print_requests.py //api/2.0/bundle --sort --oneline diff --git a/bundle/direct/bundle_apply.go b/bundle/direct/bundle_apply.go index afef2367e5b..b26861128b7 100644 --- a/bundle/direct/bundle_apply.go +++ b/bundle/direct/bundle_apply.go @@ -34,6 +34,11 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa return } + // Operations are recorded with DMS from background workers so a resource's + // deploy is not held up by the CreateOperation round trip. The queue is + // drained below, once every apply worker has finished recording. + opQueue := newOperationQueue(ctx, b.OpRec) + g.Run(defaultParallelism, func(resourceKey string, failedDependency *string) bool { entry, err := plan.WriteLockEntry(resourceKey) if err != nil { @@ -89,7 +94,7 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa return false } // Record the delete with DMS. State is nil: the resource is gone. - if err := b.recordOperation(ctx, resourceKey, action, "", nil); err != nil { + if err := opQueue.record(ctx, resourceKey, action, "", nil); err != nil { logdiag.LogError(ctx, fmt.Errorf("%s: %w", errorPrefix, err)) return false } @@ -125,7 +130,7 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa // Record the operation with DMS. The resource ID and applied config // (sv.Value) come from the write just performed; GetResourceID reads // the ID assigned by Deploy. - if err := b.recordOperation(ctx, resourceKey, action, b.StateDB.GetResourceID(resourceKey), sv.Value); err != nil { + if err := opQueue.record(ctx, resourceKey, action, b.StateDB.GetResourceID(resourceKey), sv.Value); err != nil { logdiag.LogError(ctx, fmt.Errorf("%s: %w", errorPrefix, err)) return false } @@ -152,6 +157,13 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa return true }) + + // Wait for the queued operations before returning: the caller completes the + // DMS version right after, and a version must not be completed with uploads + // still in flight. + if err := opQueue.close(); err != nil { + logdiag.LogError(ctx, err) + } } func (b *DeploymentBundle) LookupReferencePostDeploy(ctx context.Context, path *structpath.PathNode) (any, error) { diff --git a/bundle/direct/opqueue.go b/bundle/direct/opqueue.go new file mode 100644 index 00000000000..0804ad2d5b3 --- /dev/null +++ b/bundle/direct/opqueue.go @@ -0,0 +1,192 @@ +package direct + +import ( + "context" + "fmt" + "sync" + + "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/cli/libs/log" +) + +const ( + // operationQueueSize bounds how many recorded operations wait for upload. + // Apply deploys at most defaultParallelism resources at a time, so a queue + // this deep means an apply worker practically never blocks on a free slot. + operationQueueSize = 10 + + // operationUploadWorkers is how many uploads run at a time. It is below + // operationQueueSize so a burst of operations is absorbed by the queue rather + // than by one request per resource. + operationUploadWorkers = 4 +) + +// operationQueue uploads recorded operations from background workers, so an apply +// worker does not wait for the CreateOperation round trip before moving on to the +// next resource. +// +// It guarantees at most one upload in flight per resource key: within a key the +// worker that owns it uploads sequentially, so the last operation recorded for a +// resource is also the last one the service sees. +// +// Uploads are not fire-and-forget: close drains the queue and returns the first +// failure, which fails the deploy. That matters because a successfully completed +// version makes DMS the source of truth for resource state (see +// dstate.overlayDMSState); silently dropping an operation would leave DMS with an +// incomplete resource set, and the next deploy would plan to create resources +// that already exist. +type operationQueue struct { + uploader operationUploader + + // queue carries resource keys, not the operations themselves: a worker looks + // the operation up in pending when it picks the key up, which is what lets + // record collapse repeated writes to the same resource. + queue chan string + wg sync.WaitGroup + + // mu guards the fields below. + mu sync.Mutex + + // pending is the latest operation recorded per resource key that no worker has + // picked up yet. + pending map[string]recordedOperation + + // inflight holds the resource keys a worker currently owns. A key that is + // in flight is not queued again: the owning worker re-checks pending after its + // upload and picks up anything recorded in the meantime. + inflight map[string]bool + + err error + closed bool +} + +// newOperationQueue starts the upload workers. It returns nil when uploader is +// nil (recording disabled), and every method is a no-op on a nil queue so callers +// do not have to branch. +// +// ctx is used for the uploads, so it must stay valid until close returns. +func newOperationQueue(ctx context.Context, uploader operationUploader) *operationQueue { + if uploader == nil { + return nil + } + + q := &operationQueue{ + uploader: uploader, + queue: make(chan string, operationQueueSize), + pending: make(map[string]recordedOperation), + inflight: make(map[string]bool), + } + + q.wg.Add(operationUploadWorkers) + for range operationUploadWorkers { + go q.work(ctx) + } + + return q +} + +// record serializes an operation and queues it for upload. It performs no API +// call, so upload failures surface from close rather than here; the error +// returned is only about turning the applied resource into a payload. +// +// When an operation for the same resource is already waiting it is replaced +// instead of queued again: DMS keeps one state per resource key, so the later +// operation supersedes the earlier one and a single upload records both. This is +// best effort - only operations that have not been picked up yet are collapsed. +func (q *operationQueue) record(ctx context.Context, resourceKey string, action deployplan.ActionType, resourceID string, state any) error { + if q == nil { + return nil + } + + op, err := newRecordedOperation(action, resourceID, state) + if err != nil { + return err + } + + q.mu.Lock() + _, waiting := q.pending[resourceKey] + owned := waiting || q.inflight[resourceKey] + q.pending[resourceKey] = op + q.mu.Unlock() + + if owned { + log.Debugf(ctx, "Coalescing queued deployment operation for %s", resourceKey) + return nil + } + + q.queue <- resourceKey + return nil +} + +// close drains the queue and returns the first upload error. All callers of +// record must have returned first: record on a closed queue panics. Calling close +// more than once is safe, so callers can defer it and still check the error at a +// specific point. +func (q *operationQueue) close() error { + if q == nil { + return nil + } + + q.mu.Lock() + closed := q.closed + q.closed = true + q.mu.Unlock() + + if !closed { + close(q.queue) + q.wg.Wait() + } + + q.mu.Lock() + defer q.mu.Unlock() + return q.err +} + +func (q *operationQueue) work(ctx context.Context) { + defer q.wg.Done() + + for resourceKey := range q.queue { + // Keep uploading this key until nothing new was recorded for it, instead of + // putting it back on the queue: a worker sending to the channel it consumes + // from can deadlock once the queue is full. + for { + op, ok := q.take(resourceKey) + if !ok { + break + } + + if err := q.uploader.upload(ctx, resourceKey, op); err != nil { + q.setErr(fmt.Errorf("recording operation for %s with the deployment metadata service: %w", resourceKey, err)) + } + } + } +} + +// take claims the operation waiting for resourceKey, marking the key in flight so +// record does not queue it a second time. It reports false, and releases the key, +// when nothing is waiting. +func (q *operationQueue) take(resourceKey string) (recordedOperation, bool) { + q.mu.Lock() + defer q.mu.Unlock() + + op, ok := q.pending[resourceKey] + if !ok { + delete(q.inflight, resourceKey) + return recordedOperation{}, false + } + + delete(q.pending, resourceKey) + q.inflight[resourceKey] = true + return op, true +} + +// setErr keeps the first upload error; later ones are dropped because one failure +// is enough to fail the deploy. +func (q *operationQueue) setErr(err error) { + q.mu.Lock() + defer q.mu.Unlock() + + if q.err == nil { + q.err = err + } +} diff --git a/bundle/direct/opqueue_test.go b/bundle/direct/opqueue_test.go new file mode 100644 index 00000000000..5b267c4d8fa --- /dev/null +++ b/bundle/direct/opqueue_test.go @@ -0,0 +1,192 @@ +package direct + +import ( + "context" + "errors" + "strconv" + "sync" + "testing" + + "github.com/databricks/cli/bundle/deployplan" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// fakeUploader records the uploads it receives and optionally blocks until +// release is closed, so a test can hold operations in the queue and observe +// coalescing. +type fakeUploader struct { + block chan struct{} + started chan string + err error + + mu sync.Mutex + uploads []string +} + +func (f *fakeUploader) upload(ctx context.Context, resourceKey string, op recordedOperation) error { + if f.started != nil { + f.started <- resourceKey + } + if f.block != nil { + <-f.block + } + + f.mu.Lock() + defer f.mu.Unlock() + f.uploads = append(f.uploads, resourceKey+"="+string(op.state)) + return f.err +} + +func (f *fakeUploader) recorded() []string { + f.mu.Lock() + defer f.mu.Unlock() + return append([]string(nil), f.uploads...) +} + +func recordState(t *testing.T, q *operationQueue, resourceKey, name string) { + t.Helper() + require.NoError(t, q.record(t.Context(), resourceKey, deployplan.Update, "id-1", map[string]string{"name": name})) +} + +func TestOperationQueueUploadsEachOperation(t *testing.T) { + f := &fakeUploader{} + q := newOperationQueue(t.Context(), f) + + for i := range 20 { + recordState(t, q, "resources.jobs.job"+strconv.Itoa(i), "n") + } + require.NoError(t, q.close()) + + assert.Len(t, f.recorded(), 20) +} + +func TestOperationQueueCoalescesQueuedOperationsForSameResource(t *testing.T) { + // Hold the first upload so later operations for the same resource pile up in + // the queue and are collapsed into one. + f := &fakeUploader{block: make(chan struct{}), started: make(chan string, 1)} + q := newOperationQueue(t.Context(), f) + + recordState(t, q, "resources.jobs.foo", "v1") + // Wait until a worker owns the key, so the operations below are queued behind + // an in-flight upload rather than racing it. + assert.Equal(t, "resources.jobs.foo", <-f.started) + + recordState(t, q, "resources.jobs.foo", "v2") + recordState(t, q, "resources.jobs.foo", "v3") + + close(f.block) + require.NoError(t, q.close()) + + // Two uploads, not three: v2 was superseded by v3 while both were queued, and + // the last recorded state is the one the service ends up with. + assert.Equal(t, []string{ + `resources.jobs.foo={"name":"v1"}`, + `resources.jobs.foo={"name":"v3"}`, + }, f.recorded()) +} + +func TestOperationQueueReturnsUploadError(t *testing.T) { + uploadErr := errors.New("boom") + f := &fakeUploader{err: uploadErr} + q := newOperationQueue(t.Context(), f) + + recordState(t, q, "resources.jobs.foo", "v1") + + err := q.close() + require.Error(t, err) + assert.ErrorIs(t, err, uploadErr) + assert.Contains(t, err.Error(), "resources.jobs.foo") +} + +func TestOperationQueueRecordRejectsUnsupportedAction(t *testing.T) { + f := &fakeUploader{} + q := newOperationQueue(t.Context(), f) + + // Serialization failures surface at record time, on the resource that caused + // them, rather than from the drain at the end of apply. + err := q.record(t.Context(), "resources.jobs.foo", deployplan.Skip, "id-1", nil) + require.Error(t, err) + + require.NoError(t, q.close()) + assert.Empty(t, f.recorded()) +} + +func TestOperationQueueCloseIsIdempotent(t *testing.T) { + f := &fakeUploader{err: errors.New("boom")} + q := newOperationQueue(t.Context(), f) + + recordState(t, q, "resources.jobs.foo", "v1") + + require.Error(t, q.close()) + // A second close reports the same error instead of panicking on the already + // closed channel, so callers can both defer close and check it explicitly. + require.Error(t, q.close()) +} + +// serialUploader fails if two uploads for the same resource key ever overlap. +type serialUploader struct { + mu sync.Mutex + live map[string]bool + last map[string]string + uneven bool +} + +func (s *serialUploader) upload(ctx context.Context, resourceKey string, op recordedOperation) error { + s.mu.Lock() + if s.live[resourceKey] { + s.uneven = true + } + s.live[resourceKey] = true + s.mu.Unlock() + + s.mu.Lock() + defer s.mu.Unlock() + s.live[resourceKey] = false + s.last[resourceKey] = string(op.state) + return nil +} + +func TestOperationQueueUploadsOneResourceAtATime(t *testing.T) { + // Concurrent apply workers repeatedly record overlapping resource keys, the + // case where a coalesced key can be handed to a second worker while the first + // is still uploading it. The service keeps one state per key, so overlapping + // uploads for a key could land out of order and leave a stale state behind. + const ( + workers = 10 + perWorker = 5 + distinctKeyMod = 12 + ) + + u := &serialUploader{live: map[string]bool{}, last: map[string]string{}} + q := newOperationQueue(t.Context(), u) + + var wg sync.WaitGroup + for w := range workers { + wg.Add(1) + go func() { + defer wg.Done() + for i := range perWorker { + key := "resources.jobs.job" + strconv.Itoa((w*perWorker+i)%distinctKeyMod) + recordState(t, q, key, strconv.Itoa(w)) + } + }() + } + wg.Wait() + require.NoError(t, q.close()) + + assert.False(t, u.uneven, "two uploads overlapped for the same resource key") + // Every distinct key was recorded, and close drained all of them. + assert.Len(t, u.last, distinctKeyMod) + assert.Empty(t, q.pending) + assert.Empty(t, q.inflight) +} + +func TestNilOperationQueueIsNoOp(t *testing.T) { + // Recording is disabled: newOperationQueue returns nil and every method is a + // no-op, so Apply does not have to branch. + q := newOperationQueue(t.Context(), nil) + require.Nil(t, q) + require.NoError(t, q.record(t.Context(), "resources.jobs.foo", deployplan.Create, "id-1", nil)) + require.NoError(t, q.close()) +} diff --git a/bundle/direct/oprecorder.go b/bundle/direct/oprecorder.go index 033aefb5f2f..cdd99966f15 100644 --- a/bundle/direct/oprecorder.go +++ b/bundle/direct/oprecorder.go @@ -12,25 +12,57 @@ import ( "github.com/databricks/databricks-sdk-go/service/bundledeployments" ) -// opRecorder records a resource operation with the deployment metadata service -// (DMS) after it has been applied to the workspace. state is the serialized -// local config after the operation and must be nil for delete operations. -type opRecorder interface { - record(ctx context.Context, resourceKey string, action deployplan.ActionType, resourceID string, state any) error +// recordedOperation is an applied resource operation, serialized and waiting to be +// uploaded to the deployment metadata service (DMS). +// +// The payload is built on the apply worker rather than in the uploader so the +// queue does not hold on to the live resource struct, and so a malformed state +// fails the resource that produced it instead of the drain at the end of apply. +type recordedOperation struct { + action bundledeployments.OperationActionType + resourceID string + + // state is the serialized local config after the operation. It is nil for a + // delete, where the resource no longer exists. + state json.RawMessage } -// recordOperation reports an applied resource operation to DMS. It is a no-op -// unless the bundle opted into recording deployment history (OpRec is set). -// state is the serialized local config after the operation and must be nil for -// delete operations. -func (b *DeploymentBundle) recordOperation(ctx context.Context, resourceKey string, action deployplan.ActionType, resourceID string, state any) error { - if b.OpRec == nil { - return nil +// newRecordedOperation serializes an applied operation for upload. state is the +// local config after the operation and must be nil for delete operations. +func newRecordedOperation(action deployplan.ActionType, resourceID string, state any) (recordedOperation, error) { + actionType, err := deployActionToSDK(action) + if err != nil { + return recordedOperation{}, err + } + + op := recordedOperation{action: actionType, resourceID: resourceID} + + // The DMS Operation.State field carries the serialized config so the backend + // can serve it as resource state. It is intentionally left unset for delete, + // where the resource no longer exists. + // + // Redact sensitive fields, matching what dstate.SaveState writes to the local + // state file: DMS state is read back as resource state, so recording secrets + // in plaintext would both leak them to the service and reintroduce them into + // a local state file via the read path. + if state != nil { + raw, err := structwalk.RedactSensitiveFields(state, dyn.SensitiveValueRedacted) + if err != nil { + return recordedOperation{}, fmt.Errorf("serializing state: %w", err) + } + op.state = raw } - return b.OpRec.record(ctx, resourceKey, action, resourceID, state) + + return op, nil +} + +// operationUploader records an applied resource operation with DMS. Uploads run +// on the operationQueue workers, off the apply path. +type operationUploader interface { + upload(ctx context.Context, resourceKey string, op recordedOperation) error } -// operationRecorder records operations via the DMS CreateOperation API. +// operationRecorder uploads operations via the DMS CreateOperation API. type operationRecorder struct { client bundledeployments.BundleDeploymentsInterface // parent is the version the operations are recorded under, formatted as @@ -38,55 +70,36 @@ type operationRecorder struct { parent string } -// NewOperationRecorder returns an opRecorder backed by the DMS CreateOperation -// API. deploymentID and version identify the deployment version assigned by DMS -// that the operations are recorded under. -func NewOperationRecorder(client bundledeployments.BundleDeploymentsInterface, deploymentID string, version int64) opRecorder { +// NewOperationRecorder returns an operationUploader backed by the DMS +// CreateOperation API. deploymentID and version identify the deployment version +// assigned by DMS that the operations are recorded under. +func NewOperationRecorder(client bundledeployments.BundleDeploymentsInterface, deploymentID string, version int64) operationUploader { return &operationRecorder{ client: client, parent: fmt.Sprintf("deployments/%s/versions/%d", deploymentID, version), } } -func (r *operationRecorder) record(ctx context.Context, resourceKey string, action deployplan.ActionType, resourceID string, state any) error { - actionType, err := deployActionToSDK(action) - if err != nil { - return err - } - +func (r *operationRecorder) upload(ctx context.Context, resourceKey string, op recordedOperation) error { // DMS resource keys are unprefixed (e.g. "jobs.foo"), while the CLI's state // keys carry a leading "resources." (e.g. "resources.jobs.foo"). Strip it on // the way out; the read path re-adds it (see dstate.fetchDeploymentResources). dmsKey := strings.TrimPrefix(resourceKey, "resources.") - op := bundledeployments.Operation{ - ActionType: actionType, - ResourceId: resourceID, + operation := bundledeployments.Operation{ + ActionType: op.action, + ResourceId: op.resourceID, ResourceKey: dmsKey, Status: bundledeployments.OperationStatusOperationStatusSucceeded, } - - // The DMS Operation.State field carries the serialized config so the backend - // can serve it as resource state. It is intentionally left unset for delete, - // where the resource no longer exists. - // - // Redact sensitive fields, matching what dstate.SaveState writes to the local - // state file: DMS state is read back as resource state, so recording secrets - // in plaintext would both leak them to the service and reintroduce them into - // a local state file via the read path. - if state != nil { - raw, err := structwalk.RedactSensitiveFields(state, dyn.SensitiveValueRedacted) - if err != nil { - return fmt.Errorf("serializing state: %w", err) - } - msg := json.RawMessage(raw) - op.State = &msg + if op.state != nil { + operation.State = &op.state } - _, err = r.client.CreateOperation(ctx, bundledeployments.CreateOperationRequest{ + _, err := r.client.CreateOperation(ctx, bundledeployments.CreateOperationRequest{ Parent: r.parent, ResourceKey: dmsKey, - Operation: op, + Operation: operation, }) return err } diff --git a/bundle/direct/oprecorder_test.go b/bundle/direct/oprecorder_test.go index 4c1bbcacda6..aaf3243ec60 100644 --- a/bundle/direct/oprecorder_test.go +++ b/bundle/direct/oprecorder_test.go @@ -2,6 +2,7 @@ package direct import ( "context" + "sync" "testing" "github.com/databricks/cli/bundle/deployplan" @@ -13,20 +14,32 @@ import ( type fakeOpClient struct { bundledeployments.BundleDeploymentsInterface + + mu sync.Mutex requests []bundledeployments.CreateOperationRequest } func (f *fakeOpClient) CreateOperation(ctx context.Context, req bundledeployments.CreateOperationRequest) (*bundledeployments.Operation, error) { + f.mu.Lock() + defer f.mu.Unlock() f.requests = append(f.requests, req) return &bundledeployments.Operation{}, nil } +// uploadOne records a single operation through the given uploader, mirroring what +// an operationQueue worker does. +func uploadOne(t *testing.T, u operationUploader, resourceKey string, action deployplan.ActionType, resourceID string, state any) { + t.Helper() + op, err := newRecordedOperation(action, resourceID, state) + require.NoError(t, err) + require.NoError(t, u.upload(t.Context(), resourceKey, op)) +} + func TestOperationRecorderStripsResourcePrefix(t *testing.T) { f := &fakeOpClient{} r := NewOperationRecorder(f, "dep-1", 2) - err := r.record(t.Context(), "resources.jobs.foo", deployplan.Create, "job-123", map[string]string{"name": "foo"}) - require.NoError(t, err) + uploadOne(t, r, "resources.jobs.foo", deployplan.Create, "job-123", map[string]string{"name": "foo"}) require.Len(t, f.requests, 1) req := f.requests[0] @@ -44,8 +57,7 @@ func TestOperationRecorderDeleteHasNoState(t *testing.T) { f := &fakeOpClient{} r := NewOperationRecorder(f, "dep-1", 3) - err := r.record(t.Context(), "resources.jobs.foo", deployplan.Delete, "", nil) - require.NoError(t, err) + uploadOne(t, r, "resources.jobs.foo", deployplan.Delete, "", nil) require.Len(t, f.requests, 1) assert.Equal(t, bundledeployments.OperationActionTypeOperationActionTypeDelete, f.requests[0].Operation.ActionType) @@ -53,25 +65,25 @@ func TestOperationRecorderDeleteHasNoState(t *testing.T) { assert.Nil(t, f.requests[0].Operation.State) } -func TestOperationRecorderRedactsSensitiveFields(t *testing.T) { - f := &fakeOpClient{} - r := NewOperationRecorder(f, "dep-1", 2) - +func TestNewRecordedOperationRedactsSensitiveFields(t *testing.T) { state := struct { Name string `json:"name"` Token string `json:"token" bundle:"sensitive"` }{Name: "foo", Token: "super-secret"} - err := r.record(t.Context(), "resources.jobs.foo", deployplan.Create, "job-123", state) + op, err := newRecordedOperation(deployplan.Create, "job-123", state) require.NoError(t, err) - require.Len(t, f.requests, 1) - require.NotNil(t, f.requests[0].Operation.State) // Sensitive fields are redacted before leaving the CLI, matching what // dstate.SaveState writes to the local state file. assert.JSONEq(t, `{"name":"foo","token":"`+dyn.SensitiveValueRedacted+`"}`, - string(*f.requests[0].Operation.State)) + string(op.state)) +} + +func TestNewRecordedOperationRejectsUnsupportedAction(t *testing.T) { + _, err := newRecordedOperation(deployplan.Skip, "job-123", nil) + assert.Error(t, err) } func TestDeployActionToSDK(t *testing.T) { @@ -98,9 +110,3 @@ func TestDeployActionToSDK(t *testing.T) { _, err = deployActionToSDK(deployplan.Undefined) assert.Error(t, err) } - -func TestRecordOperationNoOpWithoutRecorder(t *testing.T) { - b := &DeploymentBundle{} - // No OpRec set: recording is a no-op. - assert.NoError(t, b.recordOperation(t.Context(), "resources.jobs.foo", deployplan.Create, "id", struct{}{})) -} diff --git a/bundle/direct/pkg.go b/bundle/direct/pkg.go index f95b515f726..03864af5da2 100644 --- a/bundle/direct/pkg.go +++ b/bundle/direct/pkg.go @@ -45,10 +45,12 @@ type DeploymentBundle struct { RemoteStateCache sync.Map StateCache structvar.Cache - // OpRec records each applied resource operation with the deployment metadata + // OpRec uploads each applied resource operation to the deployment metadata // service (DMS). It is nil unless the bundle opts into recording deployment // history, in which case the phases package sets it after CreateVersion. - OpRec opRecorder + // Apply queues the operations and drains them before returning, so the + // uploads do not block the resources being deployed. + OpRec operationUploader } // SetRemoteState updates the remote state with type validation and marks as fresh. From 19467dc2425fd093f06e9d11773ca724ea3cb94c Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 27 Jul 2026 13:52:34 +0000 Subject: [PATCH 005/125] bundle: gate experimental.record_deployment_history behind an env var Recording deployment history is implemented end to end, but it cannot be exposed to users yet: enabling it makes the deployment metadata service the source of truth for resource state, and there is no upgrade path from an existing direct-engine state file to a DMS-owned one. Setting the flag is now an error. DATABRICKS_BUNDLE_ENABLE_RECORD_DEPLOYMENT_HISTORY lifts the error so the CLI's own acceptance tests and DMS development can exercise the feature until the direct state upgrade lands. Co-authored-by: Isaac --- .../bundle/dms/not-supported/databricks.yml | 10 ++++ .../bundle/dms/not-supported/out.test.toml | 3 + .../bundle/dms/not-supported/output.txt | 24 ++++++++ acceptance/bundle/dms/not-supported/script | 5 ++ acceptance/bundle/dms/not-supported/test.toml | 5 ++ acceptance/bundle/dms/test.toml | 6 ++ bundle/config/experimental.go | 4 ++ .../validate_record_deployment_history.go | 48 ++++++++++++++++ ...validate_record_deployment_history_test.go | 55 +++++++++++++++++++ bundle/env/record_deployment_history.go | 19 +++++++ bundle/internal/schema/annotations.yml | 2 + bundle/phases/initialize.go | 5 ++ bundle/schema/jsonschema.json | 2 +- 13 files changed, 187 insertions(+), 1 deletion(-) create mode 100644 acceptance/bundle/dms/not-supported/databricks.yml create mode 100644 acceptance/bundle/dms/not-supported/out.test.toml create mode 100644 acceptance/bundle/dms/not-supported/output.txt create mode 100644 acceptance/bundle/dms/not-supported/script create mode 100644 acceptance/bundle/dms/not-supported/test.toml create mode 100644 bundle/config/validate/validate_record_deployment_history.go create mode 100644 bundle/config/validate/validate_record_deployment_history_test.go create mode 100644 bundle/env/record_deployment_history.go diff --git a/acceptance/bundle/dms/not-supported/databricks.yml b/acceptance/bundle/dms/not-supported/databricks.yml new file mode 100644 index 00000000000..c6edca465b6 --- /dev/null +++ b/acceptance/bundle/dms/not-supported/databricks.yml @@ -0,0 +1,10 @@ +bundle: + name: dms-not-supported + +experimental: + record_deployment_history: true + +resources: + jobs: + one: + name: one diff --git a/acceptance/bundle/dms/not-supported/out.test.toml b/acceptance/bundle/dms/not-supported/out.test.toml new file mode 100644 index 00000000000..e90b6d5d1ba --- /dev/null +++ b/acceptance/bundle/dms/not-supported/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/dms/not-supported/output.txt b/acceptance/bundle/dms/not-supported/output.txt new file mode 100644 index 00000000000..ff5758d574d --- /dev/null +++ b/acceptance/bundle/dms/not-supported/output.txt @@ -0,0 +1,24 @@ + +=== record_deployment_history is rejected: DMS cannot own resource state until the direct engine has a state upgrade path +>>> musterr [CLI] bundle validate +Error: experimental.record_deployment_history is not supported yet + at experimental.record_deployment_history + in databricks.yml:5:30 + +Name: dms-not-supported +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/dms-not-supported/default + +Found 1 error + +=== The hidden opt-in lifts the error +>>> DATABRICKS_BUNDLE_ENABLE_RECORD_DEPLOYMENT_HISTORY=1 [CLI] bundle validate +Name: dms-not-supported +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/dms-not-supported/default + +Validation OK! diff --git a/acceptance/bundle/dms/not-supported/script b/acceptance/bundle/dms/not-supported/script new file mode 100644 index 00000000000..68c0a3c5f3e --- /dev/null +++ b/acceptance/bundle/dms/not-supported/script @@ -0,0 +1,5 @@ +title "record_deployment_history is rejected: DMS cannot own resource state until the direct engine has a state upgrade path" +trace musterr $CLI bundle validate + +title "The hidden opt-in lifts the error" +trace DATABRICKS_BUNDLE_ENABLE_RECORD_DEPLOYMENT_HISTORY=1 $CLI bundle validate diff --git a/acceptance/bundle/dms/not-supported/test.toml b/acceptance/bundle/dms/not-supported/test.toml new file mode 100644 index 00000000000..c6daad089b2 --- /dev/null +++ b/acceptance/bundle/dms/not-supported/test.toml @@ -0,0 +1,5 @@ +# Unset the opt-in inherited from the parent: this test asserts the error users see. +Env.DATABRICKS_BUNDLE_ENABLE_RECORD_DEPLOYMENT_HISTORY = "" + +# This test only checks validation output; no DMS request is made either way. +RecordRequests = false diff --git a/acceptance/bundle/dms/test.toml b/acceptance/bundle/dms/test.toml index 24ce9756629..8c12d014fea 100644 --- a/acceptance/bundle/dms/test.toml +++ b/acceptance/bundle/dms/test.toml @@ -11,3 +11,9 @@ RecordRequests = true Ignore = [ '.databricks', ] + +# experimental.record_deployment_history is rejected until the direct engine has a +# state upgrade path (see validate.ValidateRecordDeploymentHistory). These tests +# exercise the feature itself, so they opt in through the same escape hatch DMS +# development uses. bundle/dms/not-supported covers the rejection. +Env.DATABRICKS_BUNDLE_ENABLE_RECORD_DEPLOYMENT_HISTORY = "1" diff --git a/bundle/config/experimental.go b/bundle/config/experimental.go index 658f1cea819..56cf3486ed3 100644 --- a/bundle/config/experimental.go +++ b/bundle/config/experimental.go @@ -53,6 +53,10 @@ type Experimental struct { // RecordDeploymentHistory opts the bundle into the deployment metadata // service (DMS), which records deployment history and tracks what changed // across deployments. + // + // Setting this is currently an error: the direct engine needs a state upgrade + // path before DMS can own resource state. See + // validate.ValidateRecordDeploymentHistory. RecordDeploymentHistory bool `json:"record_deployment_history,omitempty"` } diff --git a/bundle/config/validate/validate_record_deployment_history.go b/bundle/config/validate/validate_record_deployment_history.go new file mode 100644 index 00000000000..10144d9c796 --- /dev/null +++ b/bundle/config/validate/validate_record_deployment_history.go @@ -0,0 +1,48 @@ +package validate + +import ( + "context" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/env" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dyn" +) + +const recordDeploymentHistoryPath = "experimental.record_deployment_history" + +func ValidateRecordDeploymentHistory() bundle.ReadOnlyMutator { + return &validateRecordDeploymentHistory{} +} + +type validateRecordDeploymentHistory struct{ bundle.RO } + +func (v *validateRecordDeploymentHistory) Name() string { + return "validate:validate_record_deployment_history" +} + +// Apply rejects experimental.record_deployment_history. +// +// Recording deployment history is implemented end to end, but it is not usable yet: +// enabling it makes the deployment metadata service the source of truth for resource +// state, and there is no upgrade path from an existing direct-engine state file to a +// DMS-owned one. A bundle that flips the flag on today would have its local state +// silently overlaid by an empty DMS resource set, and the next deploy would try to +// create resources that already exist. The direct state upgrade has to land before +// this flag can be exposed; until then it errors, and +// DATABRICKS_BUNDLE_ENABLE_RECORD_DEPLOYMENT_HISTORY lifts the error for the CLI's own +// tests and for DMS development. +func (v *validateRecordDeploymentHistory) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { + if b.Config.Experimental == nil || !b.Config.Experimental.RecordDeploymentHistory { + return nil + } + if env.EnableRecordDeploymentHistory(ctx) { + return nil + } + return diag.Diagnostics{{ + Severity: diag.Error, + Summary: recordDeploymentHistoryPath + " is not supported yet", + Paths: []dyn.Path{dyn.MustPathFromString(recordDeploymentHistoryPath)}, + Locations: b.Config.GetLocations(recordDeploymentHistoryPath), + }} +} diff --git a/bundle/config/validate/validate_record_deployment_history_test.go b/bundle/config/validate/validate_record_deployment_history_test.go new file mode 100644 index 00000000000..06f71d10afa --- /dev/null +++ b/bundle/config/validate/validate_record_deployment_history_test.go @@ -0,0 +1,55 @@ +package validate + +import ( + "testing" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + bundleenv "github.com/databricks/cli/bundle/env" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/env" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestValidateRecordDeploymentHistory(t *testing.T) { + tests := []struct { + name string + enabled bool + optIn string + wantError bool + }{ + {name: "flag unset", enabled: false, wantError: false}, + {name: "flag set", enabled: true, wantError: true}, + {name: "flag set with opt-in", enabled: true, optIn: "1", wantError: false}, + {name: "flag set with empty opt-in", enabled: true, optIn: "", wantError: true}, + {name: "flag unset with opt-in", enabled: false, optIn: "1", wantError: false}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + b := &bundle.Bundle{ + Config: config.Root{ + Experimental: &config.Experimental{RecordDeploymentHistory: tc.enabled}, + }, + } + + ctx := env.Set(t.Context(), bundleenv.EnableRecordDeploymentHistoryVariable, tc.optIn) + diags := ValidateRecordDeploymentHistory().Apply(ctx, b) + + if !tc.wantError { + assert.Empty(t, diags) + return + } + require.Len(t, diags, 1) + assert.Equal(t, diag.Error, diags[0].Severity) + assert.Equal(t, "experimental.record_deployment_history is not supported yet", diags[0].Summary) + assert.Equal(t, recordDeploymentHistoryPath, diags[0].Paths[0].String()) + }) + } +} + +func TestValidateRecordDeploymentHistoryNoExperimentalBlock(t *testing.T) { + b := &bundle.Bundle{Config: config.Root{}} + assert.Empty(t, ValidateRecordDeploymentHistory().Apply(t.Context(), b)) +} diff --git a/bundle/env/record_deployment_history.go b/bundle/env/record_deployment_history.go new file mode 100644 index 00000000000..e17fdeb9f8d --- /dev/null +++ b/bundle/env/record_deployment_history.go @@ -0,0 +1,19 @@ +package env + +import "context" + +// EnableRecordDeploymentHistoryVariable names the environment variable that lifts the +// error on experimental.record_deployment_history. It is deliberately undocumented: the +// feature is complete but cannot be exposed to users yet (see +// validate.ValidateRecordDeploymentHistory for why), and this variable exists so the +// CLI's own tests and the developers working on DMS can exercise the code path meanwhile. +const EnableRecordDeploymentHistoryVariable = "DATABRICKS_BUNDLE_ENABLE_RECORD_DEPLOYMENT_HISTORY" + +// EnableRecordDeploymentHistory reports whether the environment opts into +// experimental.record_deployment_history despite it being gated off. +func EnableRecordDeploymentHistory(ctx context.Context) bool { + value, ok := get(ctx, []string{ + EnableRecordDeploymentHistoryVariable, + }) + return ok && value != "" +} diff --git a/bundle/internal/schema/annotations.yml b/bundle/internal/schema/annotations.yml index 10832fe04a6..ed4420cbe15 100644 --- a/bundle/internal/schema/annotations.yml +++ b/bundle/internal/schema/annotations.yml @@ -178,6 +178,8 @@ experimental: "record_deployment_history": "description": |- Whether to record deployment history using the deployment metadata service (DMS), which tracks what changed across deployments. + + This setting is not supported yet and enabling it is an error. "scripts": "description": |- The commands to run. diff --git a/bundle/phases/initialize.go b/bundle/phases/initialize.go index bfa2af4124b..02e03837603 100644 --- a/bundle/phases/initialize.go +++ b/bundle/phases/initialize.go @@ -177,6 +177,11 @@ func Initialize(ctx context.Context, b *bundle.Bundle) { // They are set by the CLI to track the bundle deployment and must not be set by the user. validate.ValidateDeploymentFields(), + // Reads (typed): b.Config.Experimental.RecordDeploymentHistory + // Reads (env): DATABRICKS_BUNDLE_ENABLE_RECORD_DEPLOYMENT_HISTORY (non-empty value lifts the error) + // Rejects experimental.record_deployment_history until the direct state upgrade lands. + validate.ValidateRecordDeploymentHistory(), + // Reads (dynamic): * (strings) (searches for ${resources.*} references) // Warns (TF engine) or errors (direct engine) when a cross-resource reference // points to a Terraform-only field with no DABs equivalent. diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index 4c78bd7c384..1752e643769 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -3016,7 +3016,7 @@ "$ref": "#/$defs/bool" }, "record_deployment_history": { - "description": "Whether to record deployment history using the deployment metadata service (DMS), which tracks what changed across deployments.", + "description": "Whether to record deployment history using the deployment metadata service (DMS), which tracks what changed across deployments.\n\nThis setting is not supported yet and enabling it is an error.", "$ref": "#/$defs/bool" }, "scripts": { From 22ec66a4cc6b3a792b4943e7360bc787144edf47 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 27 Jul 2026 14:08:45 +0000 Subject: [PATCH 006/125] bundle: rename the record_deployment_history escape hatch to force_allow DATABRICKS_BUNDLE_ENABLE_RECORD_DEPLOYMENT_HISTORY read as if it were the switch that turns the feature on. It is not: the flag in databricks.yml does that, and this variable only permits the flag to be set while the feature is gated off. Name it DATABRICKS_BUNDLE_FORCE_ALLOW_RECORD_DEPLOYMENT_HISTORY. Co-authored-by: Isaac --- .../bundle/dms/not-supported/output.txt | 4 ++-- acceptance/bundle/dms/not-supported/script | 4 ++-- acceptance/bundle/dms/not-supported/test.toml | 5 +++-- acceptance/bundle/dms/test.toml | 6 +++--- .../validate_record_deployment_history.go | 6 +++--- ...validate_record_deployment_history_test.go | 16 ++++++++-------- .../force_allow_record_deployment_history.go | 19 +++++++++++++++++++ bundle/env/record_deployment_history.go | 19 ------------------- bundle/phases/initialize.go | 2 +- 9 files changed, 41 insertions(+), 40 deletions(-) create mode 100644 bundle/env/force_allow_record_deployment_history.go delete mode 100644 bundle/env/record_deployment_history.go diff --git a/acceptance/bundle/dms/not-supported/output.txt b/acceptance/bundle/dms/not-supported/output.txt index ff5758d574d..0237a81da27 100644 --- a/acceptance/bundle/dms/not-supported/output.txt +++ b/acceptance/bundle/dms/not-supported/output.txt @@ -13,8 +13,8 @@ Workspace: Found 1 error -=== The hidden opt-in lifts the error ->>> DATABRICKS_BUNDLE_ENABLE_RECORD_DEPLOYMENT_HISTORY=1 [CLI] bundle validate +=== The hidden force-allow variable permits it +>>> DATABRICKS_BUNDLE_FORCE_ALLOW_RECORD_DEPLOYMENT_HISTORY=1 [CLI] bundle validate Name: dms-not-supported Target: default Workspace: diff --git a/acceptance/bundle/dms/not-supported/script b/acceptance/bundle/dms/not-supported/script index 68c0a3c5f3e..3bf017b50a0 100644 --- a/acceptance/bundle/dms/not-supported/script +++ b/acceptance/bundle/dms/not-supported/script @@ -1,5 +1,5 @@ title "record_deployment_history is rejected: DMS cannot own resource state until the direct engine has a state upgrade path" trace musterr $CLI bundle validate -title "The hidden opt-in lifts the error" -trace DATABRICKS_BUNDLE_ENABLE_RECORD_DEPLOYMENT_HISTORY=1 $CLI bundle validate +title "The hidden force-allow variable permits it" +trace DATABRICKS_BUNDLE_FORCE_ALLOW_RECORD_DEPLOYMENT_HISTORY=1 $CLI bundle validate diff --git a/acceptance/bundle/dms/not-supported/test.toml b/acceptance/bundle/dms/not-supported/test.toml index c6daad089b2..4617ff88f20 100644 --- a/acceptance/bundle/dms/not-supported/test.toml +++ b/acceptance/bundle/dms/not-supported/test.toml @@ -1,5 +1,6 @@ -# Unset the opt-in inherited from the parent: this test asserts the error users see. -Env.DATABRICKS_BUNDLE_ENABLE_RECORD_DEPLOYMENT_HISTORY = "" +# Unset the force-allow variable inherited from the parent: this test asserts the +# error users see. +Env.DATABRICKS_BUNDLE_FORCE_ALLOW_RECORD_DEPLOYMENT_HISTORY = "" # This test only checks validation output; no DMS request is made either way. RecordRequests = false diff --git a/acceptance/bundle/dms/test.toml b/acceptance/bundle/dms/test.toml index 8c12d014fea..9942a441539 100644 --- a/acceptance/bundle/dms/test.toml +++ b/acceptance/bundle/dms/test.toml @@ -14,6 +14,6 @@ Ignore = [ # experimental.record_deployment_history is rejected until the direct engine has a # state upgrade path (see validate.ValidateRecordDeploymentHistory). These tests -# exercise the feature itself, so they opt in through the same escape hatch DMS -# development uses. bundle/dms/not-supported covers the rejection. -Env.DATABRICKS_BUNDLE_ENABLE_RECORD_DEPLOYMENT_HISTORY = "1" +# exercise the feature itself, so they force allow it the same way DMS development +# does. bundle/dms/not-supported covers the rejection. +Env.DATABRICKS_BUNDLE_FORCE_ALLOW_RECORD_DEPLOYMENT_HISTORY = "1" diff --git a/bundle/config/validate/validate_record_deployment_history.go b/bundle/config/validate/validate_record_deployment_history.go index 10144d9c796..4ebb681d595 100644 --- a/bundle/config/validate/validate_record_deployment_history.go +++ b/bundle/config/validate/validate_record_deployment_history.go @@ -30,13 +30,13 @@ func (v *validateRecordDeploymentHistory) Name() string { // silently overlaid by an empty DMS resource set, and the next deploy would try to // create resources that already exist. The direct state upgrade has to land before // this flag can be exposed; until then it errors, and -// DATABRICKS_BUNDLE_ENABLE_RECORD_DEPLOYMENT_HISTORY lifts the error for the CLI's own -// tests and for DMS development. +// DATABRICKS_BUNDLE_FORCE_ALLOW_RECORD_DEPLOYMENT_HISTORY force allows it for the CLI's +// own tests and for DMS development. func (v *validateRecordDeploymentHistory) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { if b.Config.Experimental == nil || !b.Config.Experimental.RecordDeploymentHistory { return nil } - if env.EnableRecordDeploymentHistory(ctx) { + if env.ForceAllowRecordDeploymentHistory(ctx) { return nil } return diag.Diagnostics{{ diff --git a/bundle/config/validate/validate_record_deployment_history_test.go b/bundle/config/validate/validate_record_deployment_history_test.go index 06f71d10afa..1bb172766f2 100644 --- a/bundle/config/validate/validate_record_deployment_history_test.go +++ b/bundle/config/validate/validate_record_deployment_history_test.go @@ -14,16 +14,16 @@ import ( func TestValidateRecordDeploymentHistory(t *testing.T) { tests := []struct { - name string - enabled bool - optIn string - wantError bool + name string + enabled bool + forceAllow string + wantError bool }{ {name: "flag unset", enabled: false, wantError: false}, {name: "flag set", enabled: true, wantError: true}, - {name: "flag set with opt-in", enabled: true, optIn: "1", wantError: false}, - {name: "flag set with empty opt-in", enabled: true, optIn: "", wantError: true}, - {name: "flag unset with opt-in", enabled: false, optIn: "1", wantError: false}, + {name: "flag set with force allow", enabled: true, forceAllow: "1", wantError: false}, + {name: "flag set with empty force allow", enabled: true, forceAllow: "", wantError: true}, + {name: "flag unset with force allow", enabled: false, forceAllow: "1", wantError: false}, } for _, tc := range tests { @@ -34,7 +34,7 @@ func TestValidateRecordDeploymentHistory(t *testing.T) { }, } - ctx := env.Set(t.Context(), bundleenv.EnableRecordDeploymentHistoryVariable, tc.optIn) + ctx := env.Set(t.Context(), bundleenv.ForceAllowRecordDeploymentHistoryVariable, tc.forceAllow) diags := ValidateRecordDeploymentHistory().Apply(ctx, b) if !tc.wantError { diff --git a/bundle/env/force_allow_record_deployment_history.go b/bundle/env/force_allow_record_deployment_history.go new file mode 100644 index 00000000000..297ccb6f6e3 --- /dev/null +++ b/bundle/env/force_allow_record_deployment_history.go @@ -0,0 +1,19 @@ +package env + +import "context" + +// ForceAllowRecordDeploymentHistoryVariable names the environment variable that force +// allows experimental.record_deployment_history. It is deliberately undocumented: the +// feature is complete but cannot be exposed to users yet (see +// validate.ValidateRecordDeploymentHistory for why), and this variable exists so the +// CLI's own tests and the developers working on DMS can exercise the code path meanwhile. +const ForceAllowRecordDeploymentHistoryVariable = "DATABRICKS_BUNDLE_FORCE_ALLOW_RECORD_DEPLOYMENT_HISTORY" + +// ForceAllowRecordDeploymentHistory reports whether the environment force allows +// experimental.record_deployment_history despite it being gated off. +func ForceAllowRecordDeploymentHistory(ctx context.Context) bool { + value, ok := get(ctx, []string{ + ForceAllowRecordDeploymentHistoryVariable, + }) + return ok && value != "" +} diff --git a/bundle/env/record_deployment_history.go b/bundle/env/record_deployment_history.go deleted file mode 100644 index e17fdeb9f8d..00000000000 --- a/bundle/env/record_deployment_history.go +++ /dev/null @@ -1,19 +0,0 @@ -package env - -import "context" - -// EnableRecordDeploymentHistoryVariable names the environment variable that lifts the -// error on experimental.record_deployment_history. It is deliberately undocumented: the -// feature is complete but cannot be exposed to users yet (see -// validate.ValidateRecordDeploymentHistory for why), and this variable exists so the -// CLI's own tests and the developers working on DMS can exercise the code path meanwhile. -const EnableRecordDeploymentHistoryVariable = "DATABRICKS_BUNDLE_ENABLE_RECORD_DEPLOYMENT_HISTORY" - -// EnableRecordDeploymentHistory reports whether the environment opts into -// experimental.record_deployment_history despite it being gated off. -func EnableRecordDeploymentHistory(ctx context.Context) bool { - value, ok := get(ctx, []string{ - EnableRecordDeploymentHistoryVariable, - }) - return ok && value != "" -} diff --git a/bundle/phases/initialize.go b/bundle/phases/initialize.go index 02e03837603..60ea68fab82 100644 --- a/bundle/phases/initialize.go +++ b/bundle/phases/initialize.go @@ -178,7 +178,7 @@ func Initialize(ctx context.Context, b *bundle.Bundle) { validate.ValidateDeploymentFields(), // Reads (typed): b.Config.Experimental.RecordDeploymentHistory - // Reads (env): DATABRICKS_BUNDLE_ENABLE_RECORD_DEPLOYMENT_HISTORY (non-empty value lifts the error) + // Reads (env): DATABRICKS_BUNDLE_FORCE_ALLOW_RECORD_DEPLOYMENT_HISTORY (non-empty value force allows it) // Rejects experimental.record_deployment_history until the direct state upgrade lands. validate.ValidateRecordDeploymentHistory(), From d7441e43355eae51c355c07f10b52f84fdf00984 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 27 Jul 2026 14:45:13 +0000 Subject: [PATCH 007/125] bundle: only record net-new deployments in DMS Replace the blanket gate on experimental.record_deployment_history with a narrower check in dstate.DeploymentState.Open: recording is refused only when the state file already tracks deployed resources that DMS does not know about. Once DMS holds a successful version it is authoritative for resource state even when its resource set is empty, so adopting a state file written by a CLI that predates DMS would make already-owned resources look absent and create them a second time. A state file that DMS already owns is fine, and so is one with no resources (e.g. left behind by a destroy), which is what makes the error's destroy-and-redeploy advice work. The check keys off len(State) rather than the file existing because destroy leaves resources.json in place with an empty resource set. This drops validate.ValidateRecordDeploymentHistory and the DATABRICKS_BUNDLE_FORCE_ALLOW_RECORD_DEPLOYMENT_HISTORY escape hatch, which are no longer needed. Upgrading an existing state in place (state v3 with a feature flag plus per-resource tombstones so older clients refuse the state) is left as a TODO. Co-authored-by: Isaac --- .../databricks.yml | 4 +- .../out.test.toml | 0 .../bundle/dms/existing-state/output.txt | 45 +++++++++++++++ acceptance/bundle/dms/existing-state/script | 17 ++++++ .../bundle/dms/not-supported/output.txt | 24 -------- acceptance/bundle/dms/not-supported/script | 5 -- acceptance/bundle/dms/not-supported/test.toml | 6 -- acceptance/bundle/dms/test.toml | 6 -- bundle/config/experimental.go | 6 +- .../validate_record_deployment_history.go | 48 ---------------- ...validate_record_deployment_history_test.go | 55 ------------------- bundle/direct/dstate/state.go | 30 +++++++++- .../force_allow_record_deployment_history.go | 19 ------- bundle/internal/schema/annotations.yml | 2 +- bundle/phases/initialize.go | 5 -- bundle/schema/jsonschema.json | 2 +- 16 files changed, 96 insertions(+), 178 deletions(-) rename acceptance/bundle/dms/{not-supported => existing-state}/databricks.yml (52%) rename acceptance/bundle/dms/{not-supported => existing-state}/out.test.toml (100%) create mode 100644 acceptance/bundle/dms/existing-state/output.txt create mode 100644 acceptance/bundle/dms/existing-state/script delete mode 100644 acceptance/bundle/dms/not-supported/output.txt delete mode 100644 acceptance/bundle/dms/not-supported/script delete mode 100644 acceptance/bundle/dms/not-supported/test.toml delete mode 100644 bundle/config/validate/validate_record_deployment_history.go delete mode 100644 bundle/config/validate/validate_record_deployment_history_test.go delete mode 100644 bundle/env/force_allow_record_deployment_history.go diff --git a/acceptance/bundle/dms/not-supported/databricks.yml b/acceptance/bundle/dms/existing-state/databricks.yml similarity index 52% rename from acceptance/bundle/dms/not-supported/databricks.yml rename to acceptance/bundle/dms/existing-state/databricks.yml index c6edca465b6..cfd64979342 100644 --- a/acceptance/bundle/dms/not-supported/databricks.yml +++ b/acceptance/bundle/dms/existing-state/databricks.yml @@ -1,8 +1,8 @@ bundle: - name: dms-not-supported + name: dms-existing-state experimental: - record_deployment_history: true + record_deployment_history: false resources: jobs: diff --git a/acceptance/bundle/dms/not-supported/out.test.toml b/acceptance/bundle/dms/existing-state/out.test.toml similarity index 100% rename from acceptance/bundle/dms/not-supported/out.test.toml rename to acceptance/bundle/dms/existing-state/out.test.toml diff --git a/acceptance/bundle/dms/existing-state/output.txt b/acceptance/bundle/dms/existing-state/output.txt new file mode 100644 index 00000000000..d0cae5fb0ec --- /dev/null +++ b/acceptance/bundle/dms/existing-state/output.txt @@ -0,0 +1,45 @@ + +=== Deploy without recording: the bundle gets ordinary direct-engine state, unknown to DMS +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-existing-state/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> print_requests.py //api/2.0/bundle --sort --oneline + +=== Turning recording on afterwards is an error: those resources were never recorded, so treating DMS as authoritative would deploy them a second time +>>> update_file.py databricks.yml record_deployment_history: false record_deployment_history: true + +>>> musterr [CLI] bundle deploy +Error: cannot record deployment history for a bundle that already has deployed resources tracked in [TEST_TMP_DIR]/.databricks/bundle/default/resources.json: only new deployments can be recorded. Remove experimental.record_deployment_history, or destroy the bundle and deploy it again + + +=== No deployment was created in DMS +>>> print_requests.py //api/2.0/bundle --sort --oneline + +=== Destroy clears the tracked resources, so recording can be enabled afterwards +>>> update_file.py databricks.yml record_deployment_history: true record_deployment_history: false + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.jobs.one + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/dms-existing-state/default + +Deleting files... +Destroy complete! + +>>> update_file.py databricks.yml record_deployment_history: false record_deployment_history: true + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-existing-state/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> print_requests.py //api/2.0/bundle --sort --oneline +{"method": "POST", "path": "/api/2.0/bundle/deployments", "body": {"target_name": "default"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[UUID]/versions", "q": {"version_id": "1"}, "body": {"cli_version": "[CLI_VERSION]", "target_name": "default", "version_type": "VERSION_TYPE_DEPLOY"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[UUID]/versions/1/complete", "body": {"completion_reason": "VERSION_COMPLETE_SUCCESS"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[UUID]/versions/1/operations", "q": {"resource_key": "jobs.one"}, "body": {"action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_id": "[NUMID]", "resource_key": "jobs.one", "state": {"deployment": {"kind": "BUNDLE", "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-existing-state/default/state/metadata.json"}, "edit_mode": "UI_LOCKED", "format": "MULTI_TASK", "max_concurrent_runs": 1, "name": "one", "queue": {"enabled": true}}, "status": "OPERATION_STATUS_SUCCEEDED"}} diff --git a/acceptance/bundle/dms/existing-state/script b/acceptance/bundle/dms/existing-state/script new file mode 100644 index 00000000000..7bc296465d5 --- /dev/null +++ b/acceptance/bundle/dms/existing-state/script @@ -0,0 +1,17 @@ +title "Deploy without recording: the bundle gets ordinary direct-engine state, unknown to DMS" +trace $CLI bundle deploy +trace print_requests.py //api/2.0/bundle --sort --oneline + +title "Turning recording on afterwards is an error: those resources were never recorded, so treating DMS as authoritative would deploy them a second time" +trace update_file.py databricks.yml "record_deployment_history: false" "record_deployment_history: true" +trace musterr $CLI bundle deploy + +title "No deployment was created in DMS" +trace print_requests.py //api/2.0/bundle --sort --oneline + +title "Destroy clears the tracked resources, so recording can be enabled afterwards" +trace update_file.py databricks.yml "record_deployment_history: true" "record_deployment_history: false" +trace $CLI bundle destroy --auto-approve +trace update_file.py databricks.yml "record_deployment_history: false" "record_deployment_history: true" +trace $CLI bundle deploy +trace print_requests.py //api/2.0/bundle --sort --oneline diff --git a/acceptance/bundle/dms/not-supported/output.txt b/acceptance/bundle/dms/not-supported/output.txt deleted file mode 100644 index 0237a81da27..00000000000 --- a/acceptance/bundle/dms/not-supported/output.txt +++ /dev/null @@ -1,24 +0,0 @@ - -=== record_deployment_history is rejected: DMS cannot own resource state until the direct engine has a state upgrade path ->>> musterr [CLI] bundle validate -Error: experimental.record_deployment_history is not supported yet - at experimental.record_deployment_history - in databricks.yml:5:30 - -Name: dms-not-supported -Target: default -Workspace: - User: [USERNAME] - Path: /Workspace/Users/[USERNAME]/.bundle/dms-not-supported/default - -Found 1 error - -=== The hidden force-allow variable permits it ->>> DATABRICKS_BUNDLE_FORCE_ALLOW_RECORD_DEPLOYMENT_HISTORY=1 [CLI] bundle validate -Name: dms-not-supported -Target: default -Workspace: - User: [USERNAME] - Path: /Workspace/Users/[USERNAME]/.bundle/dms-not-supported/default - -Validation OK! diff --git a/acceptance/bundle/dms/not-supported/script b/acceptance/bundle/dms/not-supported/script deleted file mode 100644 index 3bf017b50a0..00000000000 --- a/acceptance/bundle/dms/not-supported/script +++ /dev/null @@ -1,5 +0,0 @@ -title "record_deployment_history is rejected: DMS cannot own resource state until the direct engine has a state upgrade path" -trace musterr $CLI bundle validate - -title "The hidden force-allow variable permits it" -trace DATABRICKS_BUNDLE_FORCE_ALLOW_RECORD_DEPLOYMENT_HISTORY=1 $CLI bundle validate diff --git a/acceptance/bundle/dms/not-supported/test.toml b/acceptance/bundle/dms/not-supported/test.toml deleted file mode 100644 index 4617ff88f20..00000000000 --- a/acceptance/bundle/dms/not-supported/test.toml +++ /dev/null @@ -1,6 +0,0 @@ -# Unset the force-allow variable inherited from the parent: this test asserts the -# error users see. -Env.DATABRICKS_BUNDLE_FORCE_ALLOW_RECORD_DEPLOYMENT_HISTORY = "" - -# This test only checks validation output; no DMS request is made either way. -RecordRequests = false diff --git a/acceptance/bundle/dms/test.toml b/acceptance/bundle/dms/test.toml index 9942a441539..24ce9756629 100644 --- a/acceptance/bundle/dms/test.toml +++ b/acceptance/bundle/dms/test.toml @@ -11,9 +11,3 @@ RecordRequests = true Ignore = [ '.databricks', ] - -# experimental.record_deployment_history is rejected until the direct engine has a -# state upgrade path (see validate.ValidateRecordDeploymentHistory). These tests -# exercise the feature itself, so they force allow it the same way DMS development -# does. bundle/dms/not-supported covers the rejection. -Env.DATABRICKS_BUNDLE_FORCE_ALLOW_RECORD_DEPLOYMENT_HISTORY = "1" diff --git a/bundle/config/experimental.go b/bundle/config/experimental.go index 56cf3486ed3..c3f1465d880 100644 --- a/bundle/config/experimental.go +++ b/bundle/config/experimental.go @@ -54,9 +54,9 @@ type Experimental struct { // service (DMS), which records deployment history and tracks what changed // across deployments. // - // Setting this is currently an error: the direct engine needs a state upgrade - // path before DMS can own resource state. See - // validate.ValidateRecordDeploymentHistory. + // Only supported for a bundle with no deployed resources yet: DMS becomes the + // source of truth for resource state, and resources tracked in an existing + // state file cannot be handed over to it yet. See dstate.DeploymentState.Open. RecordDeploymentHistory bool `json:"record_deployment_history,omitempty"` } diff --git a/bundle/config/validate/validate_record_deployment_history.go b/bundle/config/validate/validate_record_deployment_history.go deleted file mode 100644 index 4ebb681d595..00000000000 --- a/bundle/config/validate/validate_record_deployment_history.go +++ /dev/null @@ -1,48 +0,0 @@ -package validate - -import ( - "context" - - "github.com/databricks/cli/bundle" - "github.com/databricks/cli/bundle/env" - "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/dyn" -) - -const recordDeploymentHistoryPath = "experimental.record_deployment_history" - -func ValidateRecordDeploymentHistory() bundle.ReadOnlyMutator { - return &validateRecordDeploymentHistory{} -} - -type validateRecordDeploymentHistory struct{ bundle.RO } - -func (v *validateRecordDeploymentHistory) Name() string { - return "validate:validate_record_deployment_history" -} - -// Apply rejects experimental.record_deployment_history. -// -// Recording deployment history is implemented end to end, but it is not usable yet: -// enabling it makes the deployment metadata service the source of truth for resource -// state, and there is no upgrade path from an existing direct-engine state file to a -// DMS-owned one. A bundle that flips the flag on today would have its local state -// silently overlaid by an empty DMS resource set, and the next deploy would try to -// create resources that already exist. The direct state upgrade has to land before -// this flag can be exposed; until then it errors, and -// DATABRICKS_BUNDLE_FORCE_ALLOW_RECORD_DEPLOYMENT_HISTORY force allows it for the CLI's -// own tests and for DMS development. -func (v *validateRecordDeploymentHistory) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { - if b.Config.Experimental == nil || !b.Config.Experimental.RecordDeploymentHistory { - return nil - } - if env.ForceAllowRecordDeploymentHistory(ctx) { - return nil - } - return diag.Diagnostics{{ - Severity: diag.Error, - Summary: recordDeploymentHistoryPath + " is not supported yet", - Paths: []dyn.Path{dyn.MustPathFromString(recordDeploymentHistoryPath)}, - Locations: b.Config.GetLocations(recordDeploymentHistoryPath), - }} -} diff --git a/bundle/config/validate/validate_record_deployment_history_test.go b/bundle/config/validate/validate_record_deployment_history_test.go deleted file mode 100644 index 1bb172766f2..00000000000 --- a/bundle/config/validate/validate_record_deployment_history_test.go +++ /dev/null @@ -1,55 +0,0 @@ -package validate - -import ( - "testing" - - "github.com/databricks/cli/bundle" - "github.com/databricks/cli/bundle/config" - bundleenv "github.com/databricks/cli/bundle/env" - "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/env" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestValidateRecordDeploymentHistory(t *testing.T) { - tests := []struct { - name string - enabled bool - forceAllow string - wantError bool - }{ - {name: "flag unset", enabled: false, wantError: false}, - {name: "flag set", enabled: true, wantError: true}, - {name: "flag set with force allow", enabled: true, forceAllow: "1", wantError: false}, - {name: "flag set with empty force allow", enabled: true, forceAllow: "", wantError: true}, - {name: "flag unset with force allow", enabled: false, forceAllow: "1", wantError: false}, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - b := &bundle.Bundle{ - Config: config.Root{ - Experimental: &config.Experimental{RecordDeploymentHistory: tc.enabled}, - }, - } - - ctx := env.Set(t.Context(), bundleenv.ForceAllowRecordDeploymentHistoryVariable, tc.forceAllow) - diags := ValidateRecordDeploymentHistory().Apply(ctx, b) - - if !tc.wantError { - assert.Empty(t, diags) - return - } - require.Len(t, diags, 1) - assert.Equal(t, diag.Error, diags[0].Severity) - assert.Equal(t, "experimental.record_deployment_history is not supported yet", diags[0].Summary) - assert.Equal(t, recordDeploymentHistoryPath, diags[0].Paths[0].String()) - }) - } -} - -func TestValidateRecordDeploymentHistoryNoExperimentalBlock(t *testing.T) { - b := &bundle.Bundle{Config: config.Root{}} - assert.Empty(t, ValidateRecordDeploymentHistory().Apply(t.Context(), b)) -} diff --git a/bundle/direct/dstate/state.go b/bundle/direct/dstate/state.go index f554af3c9b6..e5eb44df24a 100644 --- a/bundle/direct/dstate/state.go +++ b/bundle/direct/dstate/state.go @@ -319,9 +319,33 @@ func (db *DeploymentState) Open(ctx context.Context, path string, withRecovery W return fmt.Errorf("migrating state %s: %w", path, err) } - if dmsClient != nil && db.Data.DeploymentID != "" { - if err := db.overlayDMSState(ctx, dmsClient, dmsCfg); err != nil { - return err + if dmsClient != nil { + // Only deployments that start out empty are recorded in DMS. Resources + // tracked in a state file that DMS does not know about are not in DMS and + // never will be: the first recorded deploy would create a deployment whose + // resource set covers only what that deploy touched, and DMS would then be + // authoritative for everything (see overlayDMSState). Resources this bundle + // already owns would look absent and be created a second time. + // + // A state file that DMS already owns (it carries a deployment ID) is fine — + // that is a bundle that opted in while it was still empty. So is a state file + // with no resources, e.g. one left behind by a destroy. + // + // TODO(DMS): lift this restriction by upgrading an existing state in place. + // That means writing the state at featureStateVersion (3) with a feature flag + // recording that DMS owns it, plus a tombstone entry per resource so a CLI + // that predates DMS refuses the state instead of silently deploying against a + // resource set it cannot see. The feature-flag scaffolding for this already + // exists (see featureStateVersion and Header.Features); once it is written, + // this check goes away and record_deployment_history becomes usable on + // existing bundles. + if db.Data.DeploymentID == "" && len(db.Data.State) > 0 { + return fmt.Errorf("cannot record deployment history for a bundle that already has deployed resources tracked in %s: only new deployments can be recorded. Remove experimental.record_deployment_history, or destroy the bundle and deploy it again", path) + } + if db.Data.DeploymentID != "" { + if err := db.overlayDMSState(ctx, dmsClient, dmsCfg); err != nil { + return err + } } } diff --git a/bundle/env/force_allow_record_deployment_history.go b/bundle/env/force_allow_record_deployment_history.go deleted file mode 100644 index 297ccb6f6e3..00000000000 --- a/bundle/env/force_allow_record_deployment_history.go +++ /dev/null @@ -1,19 +0,0 @@ -package env - -import "context" - -// ForceAllowRecordDeploymentHistoryVariable names the environment variable that force -// allows experimental.record_deployment_history. It is deliberately undocumented: the -// feature is complete but cannot be exposed to users yet (see -// validate.ValidateRecordDeploymentHistory for why), and this variable exists so the -// CLI's own tests and the developers working on DMS can exercise the code path meanwhile. -const ForceAllowRecordDeploymentHistoryVariable = "DATABRICKS_BUNDLE_FORCE_ALLOW_RECORD_DEPLOYMENT_HISTORY" - -// ForceAllowRecordDeploymentHistory reports whether the environment force allows -// experimental.record_deployment_history despite it being gated off. -func ForceAllowRecordDeploymentHistory(ctx context.Context) bool { - value, ok := get(ctx, []string{ - ForceAllowRecordDeploymentHistoryVariable, - }) - return ok && value != "" -} diff --git a/bundle/internal/schema/annotations.yml b/bundle/internal/schema/annotations.yml index ed4420cbe15..100d33356fd 100644 --- a/bundle/internal/schema/annotations.yml +++ b/bundle/internal/schema/annotations.yml @@ -179,7 +179,7 @@ experimental: "description": |- Whether to record deployment history using the deployment metadata service (DMS), which tracks what changed across deployments. - This setting is not supported yet and enabling it is an error. + Only supported for a bundle with no deployed resources yet. "scripts": "description": |- The commands to run. diff --git a/bundle/phases/initialize.go b/bundle/phases/initialize.go index 60ea68fab82..bfa2af4124b 100644 --- a/bundle/phases/initialize.go +++ b/bundle/phases/initialize.go @@ -177,11 +177,6 @@ func Initialize(ctx context.Context, b *bundle.Bundle) { // They are set by the CLI to track the bundle deployment and must not be set by the user. validate.ValidateDeploymentFields(), - // Reads (typed): b.Config.Experimental.RecordDeploymentHistory - // Reads (env): DATABRICKS_BUNDLE_FORCE_ALLOW_RECORD_DEPLOYMENT_HISTORY (non-empty value force allows it) - // Rejects experimental.record_deployment_history until the direct state upgrade lands. - validate.ValidateRecordDeploymentHistory(), - // Reads (dynamic): * (strings) (searches for ${resources.*} references) // Warns (TF engine) or errors (direct engine) when a cross-resource reference // points to a Terraform-only field with no DABs equivalent. diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index 1752e643769..c52e96d5dc9 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -3016,7 +3016,7 @@ "$ref": "#/$defs/bool" }, "record_deployment_history": { - "description": "Whether to record deployment history using the deployment metadata service (DMS), which tracks what changed across deployments.\n\nThis setting is not supported yet and enabling it is an error.", + "description": "Whether to record deployment history using the deployment metadata service (DMS), which tracks what changed across deployments.\n\nOnly supported for a bundle with no deployed resources yet.", "$ref": "#/$defs/bool" }, "scripts": { From 026a5213943453ad2a35f47420cefa7503e8f34a Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 27 Jul 2026 16:04:51 +0000 Subject: [PATCH 008/125] bundle: fix lint errors in the operation queue test The concurrent-producer test tripped three linters: - modernize/revive want wg.Go instead of wg.Add + go func + defer wg.Done. - testifylint's go-require flags recordState, which calls require inside the spawned goroutines. testify assertions may only run on the goroutine running the test function. Record inline and send each error to a buffered channel that the test goroutine drains after wg.Wait, so the assertions stay on the test goroutine. Co-authored-by: Isaac --- bundle/direct/opqueue_test.go | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/bundle/direct/opqueue_test.go b/bundle/direct/opqueue_test.go index 5b267c4d8fa..1b818eb3ba1 100644 --- a/bundle/direct/opqueue_test.go +++ b/bundle/direct/opqueue_test.go @@ -158,21 +158,27 @@ func TestOperationQueueUploadsOneResourceAtATime(t *testing.T) { distinctKeyMod = 12 ) + ctx := t.Context() u := &serialUploader{live: map[string]bool{}, last: map[string]string{}} - q := newOperationQueue(t.Context(), u) + q := newOperationQueue(ctx, u) + // Collect record errors instead of asserting inside the goroutines: testify + // assertions may only run on the goroutine running the test function. + errs := make(chan error, workers*perWorker) var wg sync.WaitGroup for w := range workers { - wg.Add(1) - go func() { - defer wg.Done() + wg.Go(func() { for i := range perWorker { key := "resources.jobs.job" + strconv.Itoa((w*perWorker+i)%distinctKeyMod) - recordState(t, q, key, strconv.Itoa(w)) + errs <- q.record(ctx, key, deployplan.Update, "id-1", map[string]string{"name": strconv.Itoa(w)}) } - }() + }) } wg.Wait() + close(errs) + for err := range errs { + require.NoError(t, err) + } require.NoError(t, q.close()) assert.False(t, u.uneven, "two uploads overlapped for the same resource key") From 481259bd95019be6463ef77c2a34134767d1b428 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 27 Jul 2026 16:24:50 +0000 Subject: [PATCH 009/125] bundle: keep the create action when coalescing DMS operations The operation queue collapses repeated writes to the same resource key by replacing the queued operation wholesale, so a create followed by an update was uploaded as an update. That tells DMS the resource already existed before this deploy, when in fact this deploy created it. Merge the actions instead: the state uploaded is still the later one, but a queued create or recreate wins over a subsequent update. A delete still wins over anything queued before it, since the resource is gone. This is not reachable from Apply today - each resource is recorded once per deploy, because there is one record call per graph node and dagrun visits each node exactly once - so this is about the queue being correct for any caller that records a resource more than once. Co-authored-by: Isaac --- bundle/direct/opqueue.go | 11 ++++++--- bundle/direct/opqueue_test.go | 41 ++++++++++++++++++++++++++++++++ bundle/direct/oprecorder.go | 18 ++++++++++++++ bundle/direct/oprecorder_test.go | 30 +++++++++++++++++++++++ 4 files changed, 97 insertions(+), 3 deletions(-) diff --git a/bundle/direct/opqueue.go b/bundle/direct/opqueue.go index 0804ad2d5b3..72a2fa9c39e 100644 --- a/bundle/direct/opqueue.go +++ b/bundle/direct/opqueue.go @@ -91,8 +91,10 @@ func newOperationQueue(ctx context.Context, uploader operationUploader) *operati // // When an operation for the same resource is already waiting it is replaced // instead of queued again: DMS keeps one state per resource key, so the later -// operation supersedes the earlier one and a single upload records both. This is -// best effort - only operations that have not been picked up yet are collapsed. +// operation supersedes the earlier one and a single upload records both. The +// merged operation keeps the action of a queued create (see mergeAction), so +// collapsing a create and a later update still records a create. This is best +// effort - only operations that have not been picked up yet are collapsed. func (q *operationQueue) record(ctx context.Context, resourceKey string, action deployplan.ActionType, resourceID string, state any) error { if q == nil { return nil @@ -104,8 +106,11 @@ func (q *operationQueue) record(ctx context.Context, resourceKey string, action } q.mu.Lock() - _, waiting := q.pending[resourceKey] + queued, waiting := q.pending[resourceKey] owned := waiting || q.inflight[resourceKey] + if waiting { + op.action = mergeAction(queued.action, op.action) + } q.pending[resourceKey] = op q.mu.Unlock() diff --git a/bundle/direct/opqueue_test.go b/bundle/direct/opqueue_test.go index 1b818eb3ba1..a8e05141fc7 100644 --- a/bundle/direct/opqueue_test.go +++ b/bundle/direct/opqueue_test.go @@ -8,6 +8,7 @@ import ( "testing" "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/databricks-sdk-go/service/bundledeployments" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -22,6 +23,7 @@ type fakeUploader struct { mu sync.Mutex uploads []string + actions map[string]bundledeployments.OperationActionType } func (f *fakeUploader) upload(ctx context.Context, resourceKey string, op recordedOperation) error { @@ -35,6 +37,10 @@ func (f *fakeUploader) upload(ctx context.Context, resourceKey string, op record f.mu.Lock() defer f.mu.Unlock() f.uploads = append(f.uploads, resourceKey+"="+string(op.state)) + if f.actions == nil { + f.actions = map[string]bundledeployments.OperationActionType{} + } + f.actions[resourceKey] = op.action return f.err } @@ -44,6 +50,12 @@ func (f *fakeUploader) recorded() []string { return append([]string(nil), f.uploads...) } +func (f *fakeUploader) actionFor(resourceKey string) bundledeployments.OperationActionType { + f.mu.Lock() + defer f.mu.Unlock() + return f.actions[resourceKey] +} + func recordState(t *testing.T, q *operationQueue, resourceKey, name string) { t.Helper() require.NoError(t, q.record(t.Context(), resourceKey, deployplan.Update, "id-1", map[string]string{"name": name})) @@ -86,6 +98,35 @@ func TestOperationQueueCoalescesQueuedOperationsForSameResource(t *testing.T) { }, f.recorded()) } +func TestOperationQueueCoalescingKeepsCreateAction(t *testing.T) { + // Hold the first upload so the create below stays queued and the update + // coalesces into it. + f := &fakeUploader{block: make(chan struct{}), started: make(chan string, 1)} + q := newOperationQueue(t.Context(), f) + + recordState(t, q, "resources.jobs.hold", "v1") + assert.Equal(t, "resources.jobs.hold", <-f.started) + + // Occupy the remaining workers so nothing drains the key under test. + for i := range operationUploadWorkers - 1 { + recordState(t, q, "resources.jobs.hold"+strconv.Itoa(i), "v1") + assert.Equal(t, "resources.jobs.hold"+strconv.Itoa(i), <-f.started) + } + + require.NoError(t, q.record(t.Context(), "resources.jobs.foo", deployplan.Create, "id-1", map[string]string{"name": "created"})) + require.NoError(t, q.record(t.Context(), "resources.jobs.foo", deployplan.Update, "id-1", map[string]string{"name": "updated"})) + + close(f.block) + require.NoError(t, q.close()) + + // The state is the later one, but the action stays CREATE: recording an update + // would tell DMS the resource already existed before this deploy. + assert.Contains(t, f.recorded(), `resources.jobs.foo={"name":"updated"}`) + assert.Equal(t, + bundledeployments.OperationActionTypeOperationActionTypeCreate, + f.actionFor("resources.jobs.foo")) +} + func TestOperationQueueReturnsUploadError(t *testing.T) { uploadErr := errors.New("boom") f := &fakeUploader{err: uploadErr} diff --git a/bundle/direct/oprecorder.go b/bundle/direct/oprecorder.go index cdd99966f15..63d0c84a621 100644 --- a/bundle/direct/oprecorder.go +++ b/bundle/direct/oprecorder.go @@ -56,6 +56,24 @@ func newRecordedOperation(action deployplan.ActionType, resourceID string, state return op, nil } +// mergeAction returns the action to record when a later operation coalesces into +// one still queued for the same resource (see operationQueue.record). The state +// uploaded is the later one, but the action must not be downgraded: Create and +// Recreate tell DMS the resource ID is new, and a subsequent Update only refines +// the state of that same new resource. Recording the pair as an Update would +// claim the resource already existed. A Delete is the exception - the resource is +// gone, so nothing earlier is worth reporting. +func mergeAction(queued, next bundledeployments.OperationActionType) bundledeployments.OperationActionType { + if next == bundledeployments.OperationActionTypeOperationActionTypeDelete { + return next + } + if queued == bundledeployments.OperationActionTypeOperationActionTypeCreate || + queued == bundledeployments.OperationActionTypeOperationActionTypeRecreate { + return queued + } + return next +} + // operationUploader records an applied resource operation with DMS. Uploads run // on the operationQueue workers, off the apply path. type operationUploader interface { diff --git a/bundle/direct/oprecorder_test.go b/bundle/direct/oprecorder_test.go index aaf3243ec60..70afb788cd3 100644 --- a/bundle/direct/oprecorder_test.go +++ b/bundle/direct/oprecorder_test.go @@ -86,6 +86,36 @@ func TestNewRecordedOperationRejectsUnsupportedAction(t *testing.T) { assert.Error(t, err) } +func TestMergeAction(t *testing.T) { + const ( + create = bundledeployments.OperationActionTypeOperationActionTypeCreate + recreate = bundledeployments.OperationActionTypeOperationActionTypeRecreate + update = bundledeployments.OperationActionTypeOperationActionTypeUpdate + resize = bundledeployments.OperationActionTypeOperationActionTypeResize + del = bundledeployments.OperationActionTypeOperationActionTypeDelete + ) + + cases := []struct { + queued, next, want bundledeployments.OperationActionType + }{ + // A queued create is not downgraded: the resource is still new. + {create, update, create}, + {create, resize, create}, + {recreate, update, recreate}, + {create, create, create}, + // A delete wins: the resource is gone, so the earlier action is moot. + {create, del, del}, + {update, del, del}, + // Neither side is a create, so the later action stands. + {update, resize, resize}, + {resize, update, update}, + {del, create, create}, + } + for _, c := range cases { + assert.Equal(t, c.want, mergeAction(c.queued, c.next), "queued %s, next %s", c.queued, c.next) + } +} + func TestDeployActionToSDK(t *testing.T) { cases := []struct { action deployplan.ActionType From 309a1a4fe5098bd122103ba880b862ac5d65f864 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 27 Jul 2026 22:32:16 +0000 Subject: [PATCH 010/125] bundle: drop the dms package comment Co-authored-by: Isaac --- libs/dms/recorder.go | 9 --------- 1 file changed, 9 deletions(-) diff --git a/libs/dms/recorder.go b/libs/dms/recorder.go index 0d9563dd052..1a113491cf9 100644 --- a/libs/dms/recorder.go +++ b/libs/dms/recorder.go @@ -1,12 +1,3 @@ -// Package dms records bundle deployments as versions with the Deployment -// Metadata Service (DMS). -// -// It is intentionally independent of the deployment lock: a Recorder does not -// acquire or hold any lock. Callers are responsible for serializing concurrent -// deployments (today via the workspace-filesystem lock). The server-side -// version counter — CreateVersion only succeeds when the requested version is -// last_version_id + 1 — provides the concurrency control for the records -// themselves. package dms import ( From 211aa94a2b5e00107b2bc6231192f1d9c3c7df88 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 27 Jul 2026 23:33:49 +0000 Subject: [PATCH 011/125] bundle: resolve the DMS deployment ID from the workspace The deployment ID was persisted in the local state file header, which made the CLI the source of truth for a value the service mints. DMS registers each deployment as a workspace node named resources.deployment.json under initial_parent_path, and the node's ID *is* the deployment ID, so it can be resolved from the workspace instead. ResolveDeploymentID does a get-status on /resources.deployment.json and returns the node ID, or empty when the node is absent. Deploy, destroy, and the read path all resolve the ID that way and pass it down; the read path then constructs state from GetDeployment + ListResources as before. Consequences: - Header.DeploymentID, GetDeploymentID, and SetDeploymentID are gone, along with the headerDirty machinery that only existed to persist the ID on a resource-less deploy. - Open takes a *DMSSource instead of a (client, config) pair, since the resolved ID now has to be threaded in too. - CreateDeployment sets initial_parent_path, which the service requires and the CLI never set. - createDeploymentVersion no longer recovers from a 404 on GetDeployment by creating a second deployment. A destroy trashes the node, so a resolved ID whose record is missing means the two are out of sync, and creating another deployment would collide on the same node path. The testserver models the real derivation: CreateDeployment creates the workspace node and uses its object ID as the deployment ID, so the acceptance tests exercise get-status resolution end to end. dms/record now wipes the local cache before redeploying and records zero operations, which is the read path reconstructing state entirely from DMS. Co-authored-by: Isaac --- .../bundle/dms/existing-state/output.txt | 8 +- .../bundle/dms/multiple-resources/output.txt | 14 +-- acceptance/bundle/dms/no-resources/output.txt | 28 +++--- acceptance/bundle/dms/no-resources/script | 6 +- acceptance/bundle/dms/record/output.txt | 33 ++++--- acceptance/bundle/dms/record/script | 7 +- .../dms/redeploy-after-destroy/output.txt | 31 +++---- .../bundle/dms/redeploy-after-destroy/script | 12 +-- acceptance/bundle/dms/test.toml | 5 +- bundle/configsync/diff.go | 2 +- bundle/configsync/variables.go | 2 +- bundle/direct/bind.go | 12 +-- bundle/direct/dstate/dms.go | 16 ++-- bundle/direct/dstate/state.go | 91 ++++++------------- bundle/direct/dstate/state_test.go | 87 +++--------------- bundle/phases/deploy.go | 12 ++- bundle/phases/destroy.go | 6 +- bundle/phases/dms.go | 29 +++--- cmd/bundle/generate/dashboard.go | 2 +- cmd/bundle/generate/genie_space.go | 2 +- cmd/bundle/utils/process.go | 29 +++--- libs/dms/recorder.go | 73 ++++++++------- libs/dms/recorder_test.go | 83 ++++++++--------- libs/dms/resolve.go | 45 +++++++++ libs/dms/resolve_test.go | 67 ++++++++++++++ libs/testserver/bundle.go | 47 ++++++++-- 26 files changed, 406 insertions(+), 343 deletions(-) create mode 100644 libs/dms/resolve.go create mode 100644 libs/dms/resolve_test.go diff --git a/acceptance/bundle/dms/existing-state/output.txt b/acceptance/bundle/dms/existing-state/output.txt index d0cae5fb0ec..6aeff83c04b 100644 --- a/acceptance/bundle/dms/existing-state/output.txt +++ b/acceptance/bundle/dms/existing-state/output.txt @@ -39,7 +39,7 @@ Updating deployment state... Deployment complete! >>> print_requests.py //api/2.0/bundle --sort --oneline -{"method": "POST", "path": "/api/2.0/bundle/deployments", "body": {"target_name": "default"}} -{"method": "POST", "path": "/api/2.0/bundle/deployments/[UUID]/versions", "q": {"version_id": "1"}, "body": {"cli_version": "[CLI_VERSION]", "target_name": "default", "version_type": "VERSION_TYPE_DEPLOY"}} -{"method": "POST", "path": "/api/2.0/bundle/deployments/[UUID]/versions/1/complete", "body": {"completion_reason": "VERSION_COMPLETE_SUCCESS"}} -{"method": "POST", "path": "/api/2.0/bundle/deployments/[UUID]/versions/1/operations", "q": {"resource_key": "jobs.one"}, "body": {"action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_id": "[NUMID]", "resource_key": "jobs.one", "state": {"deployment": {"kind": "BUNDLE", "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-existing-state/default/state/metadata.json"}, "edit_mode": "UI_LOCKED", "format": "MULTI_TASK", "max_concurrent_runs": 1, "name": "one", "queue": {"enabled": true}}, "status": "OPERATION_STATUS_SUCCEEDED"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments", "body": {"initial_parent_path": "/Workspace/Users/[USERNAME]/.bundle/dms-existing-state/default/state", "target_name": "default"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": {"version_id": "1"}, "body": {"cli_version": "[CLI_VERSION]", "target_name": "default", "version_type": "VERSION_TYPE_DEPLOY"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/complete", "body": {"completion_reason": "VERSION_COMPLETE_SUCCESS"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations", "q": {"resource_key": "jobs.one"}, "body": {"action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_id": "[NUMID]", "resource_key": "jobs.one", "state": {"deployment": {"kind": "BUNDLE", "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-existing-state/default/state/metadata.json"}, "edit_mode": "UI_LOCKED", "format": "MULTI_TASK", "max_concurrent_runs": 1, "name": "one", "queue": {"enabled": true}}, "status": "OPERATION_STATUS_SUCCEEDED"}} diff --git a/acceptance/bundle/dms/multiple-resources/output.txt b/acceptance/bundle/dms/multiple-resources/output.txt index 4bacec0c317..75086ceb5f7 100644 --- a/acceptance/bundle/dms/multiple-resources/output.txt +++ b/acceptance/bundle/dms/multiple-resources/output.txt @@ -7,11 +7,11 @@ Updating deployment state... Deployment complete! >>> print_requests.py //versions/1/operations --sort --del-body state --oneline -{"method": "POST", "path": "/api/2.0/bundle/deployments/[UUID]/versions/1/operations", "q": {"resource_key": "jobs.five"}, "body": {"action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_id": "[NUMID]", "resource_key": "jobs.five", "status": "OPERATION_STATUS_SUCCEEDED"}} -{"method": "POST", "path": "/api/2.0/bundle/deployments/[UUID]/versions/1/operations", "q": {"resource_key": "jobs.four"}, "body": {"action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_id": "[NUMID]", "resource_key": "jobs.four", "status": "OPERATION_STATUS_SUCCEEDED"}} -{"method": "POST", "path": "/api/2.0/bundle/deployments/[UUID]/versions/1/operations", "q": {"resource_key": "jobs.one"}, "body": {"action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_id": "[NUMID]", "resource_key": "jobs.one", "status": "OPERATION_STATUS_SUCCEEDED"}} -{"method": "POST", "path": "/api/2.0/bundle/deployments/[UUID]/versions/1/operations", "q": {"resource_key": "jobs.three"}, "body": {"action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_id": "[NUMID]", "resource_key": "jobs.three", "status": "OPERATION_STATUS_SUCCEEDED"}} -{"method": "POST", "path": "/api/2.0/bundle/deployments/[UUID]/versions/1/operations", "q": {"resource_key": "jobs.two"}, "body": {"action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_id": "[NUMID]", "resource_key": "jobs.two", "status": "OPERATION_STATUS_SUCCEEDED"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations", "q": {"resource_key": "jobs.five"}, "body": {"action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_id": "[NUMID]", "resource_key": "jobs.five", "status": "OPERATION_STATUS_SUCCEEDED"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations", "q": {"resource_key": "jobs.four"}, "body": {"action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_id": "[NUMID]", "resource_key": "jobs.four", "status": "OPERATION_STATUS_SUCCEEDED"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations", "q": {"resource_key": "jobs.one"}, "body": {"action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_id": "[NUMID]", "resource_key": "jobs.one", "status": "OPERATION_STATUS_SUCCEEDED"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations", "q": {"resource_key": "jobs.three"}, "body": {"action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_id": "[NUMID]", "resource_key": "jobs.three", "status": "OPERATION_STATUS_SUCCEEDED"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations", "q": {"resource_key": "jobs.two"}, "body": {"action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_id": "[NUMID]", "resource_key": "jobs.two", "status": "OPERATION_STATUS_SUCCEEDED"}} === Redeploy with no changes: nothing is applied, so no operations are recorded and only the version is opened and completed >>> [CLI] bundle deploy @@ -21,5 +21,5 @@ Updating deployment state... Deployment complete! >>> print_requests.py //api/2.0/bundle --sort --oneline -{"method": "POST", "path": "/api/2.0/bundle/deployments/[UUID]/versions", "q": {"version_id": "2"}, "body": {"cli_version": "[CLI_VERSION]", "target_name": "default", "version_type": "VERSION_TYPE_DEPLOY"}} -{"method": "POST", "path": "/api/2.0/bundle/deployments/[UUID]/versions/2/complete", "body": {"completion_reason": "VERSION_COMPLETE_SUCCESS"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": {"version_id": "2"}, "body": {"cli_version": "[CLI_VERSION]", "target_name": "default", "version_type": "VERSION_TYPE_DEPLOY"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/complete", "body": {"completion_reason": "VERSION_COMPLETE_SUCCESS"}} diff --git a/acceptance/bundle/dms/no-resources/output.txt b/acceptance/bundle/dms/no-resources/output.txt index e774fd7546c..73bd3adfa11 100644 --- a/acceptance/bundle/dms/no-resources/output.txt +++ b/acceptance/bundle/dms/no-resources/output.txt @@ -1,9 +1,8 @@ -=== First deploy of a bundle with no resources: the deployment is created and its ID is persisted, even though no resource state was written +=== First deploy of a bundle with no resources: the deployment is created, and its workspace node identifies it even though no resource state was written >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-no-resources/default/files... Deploying resources... -Updating deployment state... Deployment complete! >>> print_requests.py //api/2.0/bundle --sort --get @@ -11,12 +10,13 @@ Deployment complete! "method": "POST", "path": "/api/2.0/bundle/deployments", "body": { + "initial_parent_path": "/Workspace/Users/[USERNAME]/.bundle/dms-no-resources/default/state", "target_name": "default" } } { "method": "POST", - "path": "/api/2.0/bundle/deployments/[UUID]/versions", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": { "version_id": "1" }, @@ -28,38 +28,40 @@ Deployment complete! } { "method": "POST", - "path": "/api/2.0/bundle/deployments/[UUID]/versions/1/complete", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/complete", "body": { "completion_reason": "VERSION_COMPLETE_SUCCESS" } } ->>> jq .deployment_id .databricks/bundle/default/resources.json -"[UUID]" +>>> [CLI] workspace get-status /Workspace/Users/[USERNAME]/.bundle/dms-no-resources/default/state/resources.deployment.json +{ + "object_type": "FILE", + "path": "/Workspace/Users/[USERNAME]/.bundle/dms-no-resources/default/state/resources.deployment.json" +} -=== Redeploy: the persisted ID is reused, so no second deployment is created +=== Redeploy: the deployment is resolved from that node, so no second deployment is created >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-no-resources/default/files... Deploying resources... -Updating deployment state... Deployment complete! >>> print_requests.py //api/2.0/bundle --sort --get { "method": "GET", - "path": "/api/2.0/bundle/deployments/[UUID]" + "path": "/api/2.0/bundle/deployments/[NUMID]" } { "method": "GET", - "path": "/api/2.0/bundle/deployments/[UUID]" + "path": "/api/2.0/bundle/deployments/[NUMID]" } { "method": "GET", - "path": "/api/2.0/bundle/deployments/[UUID]/resources" + "path": "/api/2.0/bundle/deployments/[NUMID]/resources" } { "method": "POST", - "path": "/api/2.0/bundle/deployments/[UUID]/versions", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": { "version_id": "2" }, @@ -71,7 +73,7 @@ Deployment complete! } { "method": "POST", - "path": "/api/2.0/bundle/deployments/[UUID]/versions/2/complete", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/complete", "body": { "completion_reason": "VERSION_COMPLETE_SUCCESS" } diff --git a/acceptance/bundle/dms/no-resources/script b/acceptance/bundle/dms/no-resources/script index 4bc97c5864d..847935af5d1 100644 --- a/acceptance/bundle/dms/no-resources/script +++ b/acceptance/bundle/dms/no-resources/script @@ -1,8 +1,8 @@ -title "First deploy of a bundle with no resources: the deployment is created and its ID is persisted, even though no resource state was written" +title "First deploy of a bundle with no resources: the deployment is created, and its workspace node identifies it even though no resource state was written" trace $CLI bundle deploy trace print_requests.py //api/2.0/bundle --sort --get -trace jq .deployment_id .databricks/bundle/default/resources.json +trace $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-no-resources/default/state/resources.deployment.json" | jq '{object_type,path}' -title "Redeploy: the persisted ID is reused, so no second deployment is created" +title "Redeploy: the deployment is resolved from that node, so no second deployment is created" trace $CLI bundle deploy trace print_requests.py //api/2.0/bundle --sort --get diff --git a/acceptance/bundle/dms/record/output.txt b/acceptance/bundle/dms/record/output.txt index 5c0317f38cc..ed10f06e699 100644 --- a/acceptance/bundle/dms/record/output.txt +++ b/acceptance/bundle/dms/record/output.txt @@ -11,12 +11,13 @@ Deployment complete! "method": "POST", "path": "/api/2.0/bundle/deployments", "body": { + "initial_parent_path": "/Workspace/Users/[USERNAME]/.bundle/dms-record/default/state", "target_name": "default" } } { "method": "POST", - "path": "/api/2.0/bundle/deployments/[UUID]/versions", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": { "version_id": "1" }, @@ -28,14 +29,14 @@ Deployment complete! } { "method": "POST", - "path": "/api/2.0/bundle/deployments/[UUID]/versions/1/complete", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/complete", "body": { "completion_reason": "VERSION_COMPLETE_SUCCESS" } } { "method": "POST", - "path": "/api/2.0/bundle/deployments/[UUID]/versions/1/operations", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations", "q": { "resource_key": "jobs.foo" }, @@ -60,11 +61,17 @@ Deployment complete! } } -=== The server-assigned deployment ID is persisted in the local state file ->>> jq .deployment_id .databricks/bundle/default/resources.json -"[UUID]" +=== The deployment ID is the ID of the workspace node the service registered under initial_parent_path; the CLI stores nothing locally +>>> [CLI] workspace get-status /Workspace/Users/[USERNAME]/.bundle/dms-record/default/state/resources.deployment.json +{ + "object_type": "FILE", + "path": "/Workspace/Users/[USERNAME]/.bundle/dms-record/default/state/resources.deployment.json" +} + +>>> jq has("deployment_id") .databricks/bundle/default/resources.json +false -=== Redeploy after deleting the local cache: the deployment ID is recovered from remote state, the same deployment is reused, and the version increments (no new CreateDeployment) +=== Redeploy after deleting the local cache: the deployment ID is resolved from that node, the same deployment is reused, and the version increments (no new CreateDeployment) >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-record/default/files... Deploying resources... @@ -74,7 +81,7 @@ Deployment complete! >>> print_requests.py //api/2.0/bundle --sort { "method": "POST", - "path": "/api/2.0/bundle/deployments/[UUID]/versions", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": { "version_id": "2" }, @@ -86,7 +93,7 @@ Deployment complete! } { "method": "POST", - "path": "/api/2.0/bundle/deployments/[UUID]/versions/2/complete", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/complete", "body": { "completion_reason": "VERSION_COMPLETE_SUCCESS" } @@ -105,11 +112,11 @@ Destroy complete! >>> print_requests.py //api/2.0/bundle --sort { "method": "DELETE", - "path": "/api/2.0/bundle/deployments/[UUID]" + "path": "/api/2.0/bundle/deployments/[NUMID]" } { "method": "POST", - "path": "/api/2.0/bundle/deployments/[UUID]/versions", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": { "version_id": "3" }, @@ -121,14 +128,14 @@ Destroy complete! } { "method": "POST", - "path": "/api/2.0/bundle/deployments/[UUID]/versions/3/complete", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/3/complete", "body": { "completion_reason": "VERSION_COMPLETE_SUCCESS" } } { "method": "POST", - "path": "/api/2.0/bundle/deployments/[UUID]/versions/3/operations", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/3/operations", "q": { "resource_key": "jobs.foo" }, diff --git a/acceptance/bundle/dms/record/script b/acceptance/bundle/dms/record/script index ab59d38afb4..63eaa323b1b 100644 --- a/acceptance/bundle/dms/record/script +++ b/acceptance/bundle/dms/record/script @@ -2,10 +2,11 @@ title "Deploy: the server assigns the deployment ID, and a version + create oper trace $CLI bundle deploy trace print_requests.py //api/2.0/bundle --sort -title "The server-assigned deployment ID is persisted in the local state file" -trace jq .deployment_id .databricks/bundle/default/resources.json +title "The deployment ID is the ID of the workspace node the service registered under initial_parent_path; the CLI stores nothing locally" +trace $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-record/default/state/resources.deployment.json" | jq '{object_type,path}' +trace jq 'has("deployment_id")' .databricks/bundle/default/resources.json -title "Redeploy after deleting the local cache: the deployment ID is recovered from remote state, the same deployment is reused, and the version increments (no new CreateDeployment)" +title "Redeploy after deleting the local cache: the deployment ID is resolved from that node, the same deployment is reused, and the version increments (no new CreateDeployment)" rm -rf .databricks trace $CLI bundle deploy trace print_requests.py //api/2.0/bundle --sort diff --git a/acceptance/bundle/dms/redeploy-after-destroy/output.txt b/acceptance/bundle/dms/redeploy-after-destroy/output.txt index 5326653519a..ed7b6ede989 100644 --- a/acceptance/bundle/dms/redeploy-after-destroy/output.txt +++ b/acceptance/bundle/dms/redeploy-after-destroy/output.txt @@ -1,5 +1,5 @@ -=== Deploy, then destroy: the deployment record is deleted, but its ID stays behind in the local state file +=== Deploy, then destroy: the deployment record and its workspace node are both deleted, so nothing points at the destroyed deployment >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-redeploy-after-destroy/default/files... Deploying resources... @@ -15,35 +15,34 @@ All files and directories at the following location will be deleted: /Workspace/ Deleting files... Destroy complete! ->>> jq .deployment_id .databricks/bundle/default/resources.json -"[DESTROYED_DEPLOYMENT_ID]" +>>> musterr [CLI] workspace get-status /Workspace/Users/[USERNAME]/.bundle/dms-redeploy-after-destroy/default/state/resources.deployment.json +Error: Path (/Workspace/Users/[USERNAME]/.bundle/dms-redeploy-after-destroy/default/state/resources.deployment.json) doesn't exist. -=== Deploy again: the stale deployment ID no longer resolves, so a new deployment is created instead of failing the deploy +=== Deploy again: with no node to resolve, a fresh deployment is created at version 1, with its own workspace node >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-redeploy-after-destroy/default/files... Deploying resources... Updating deployment state... Deployment complete! ->>> print_requests.py //api/2.0/bundle --sort --get -{ - "method": "GET", - "path": "/api/2.0/bundle/deployments/[DESTROYED_DEPLOYMENT_ID]" -} +>>> [CLI] workspace get-status /Workspace/Users/[USERNAME]/.bundle/dms-redeploy-after-destroy/default/state/resources.deployment.json { - "method": "GET", - "path": "/api/2.0/bundle/deployments/[DESTROYED_DEPLOYMENT_ID]" + "object_type": "FILE", + "path": "/Workspace/Users/[USERNAME]/.bundle/dms-redeploy-after-destroy/default/state/resources.deployment.json" } + +>>> print_requests.py //api/2.0/bundle --sort --get { "method": "POST", "path": "/api/2.0/bundle/deployments", "body": { + "initial_parent_path": "/Workspace/Users/[USERNAME]/.bundle/dms-redeploy-after-destroy/default/state", "target_name": "default" } } { "method": "POST", - "path": "/api/2.0/bundle/deployments/[UUID]/versions", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": { "version_id": "1" }, @@ -55,14 +54,14 @@ Deployment complete! } { "method": "POST", - "path": "/api/2.0/bundle/deployments/[UUID]/versions/1/complete", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/complete", "body": { "completion_reason": "VERSION_COMPLETE_SUCCESS" } } { "method": "POST", - "path": "/api/2.0/bundle/deployments/[UUID]/versions/1/operations", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations", "q": { "resource_key": "jobs.foo" }, @@ -86,7 +85,3 @@ Deployment complete! "status": "OPERATION_STATUS_SUCCEEDED" } } - -=== The new deployment ID replaces the stale one in state ->>> jq .deployment_id .databricks/bundle/default/resources.json -"[UUID]" diff --git a/acceptance/bundle/dms/redeploy-after-destroy/script b/acceptance/bundle/dms/redeploy-after-destroy/script index 628cd7bf494..04c639019c9 100644 --- a/acceptance/bundle/dms/redeploy-after-destroy/script +++ b/acceptance/bundle/dms/redeploy-after-destroy/script @@ -1,15 +1,11 @@ -title "Deploy, then destroy: the deployment record is deleted, but its ID stays behind in the local state file" +title "Deploy, then destroy: the deployment record and its workspace node are both deleted, so nothing points at the destroyed deployment" trace $CLI bundle deploy trace $CLI bundle destroy --auto-approve print_requests.py //api/2.0/bundle --sort --get > /dev/null -destroyed_id=$(jq -r .deployment_id .databricks/bundle/default/resources.json) -add_repl.py "$destroyed_id" DESTROYED_DEPLOYMENT_ID -trace jq .deployment_id .databricks/bundle/default/resources.json +trace musterr $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-redeploy-after-destroy/default/state/resources.deployment.json" -title "Deploy again: the stale deployment ID no longer resolves, so a new deployment is created instead of failing the deploy" +title "Deploy again: with no node to resolve, a fresh deployment is created at version 1, with its own workspace node" trace $CLI bundle deploy +trace $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-redeploy-after-destroy/default/state/resources.deployment.json" | jq '{object_type,path}' trace print_requests.py //api/2.0/bundle --sort --get - -title "The new deployment ID replaces the stale one in state" -trace jq .deployment_id .databricks/bundle/default/resources.json diff --git a/acceptance/bundle/dms/test.toml b/acceptance/bundle/dms/test.toml index 24ce9756629..1e36331a16f 100644 --- a/acceptance/bundle/dms/test.toml +++ b/acceptance/bundle/dms/test.toml @@ -1,9 +1,8 @@ Local = true Cloud = false -# Deployment Metadata Service (DMS) recording is only meaningful in the direct -# engine, where the deployment ID is stored in and read from the direct-engine -# state. +# Deployment Metadata Service (DMS) recording is only supported by the direct +# engine; it is a no-op on terraform. EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] RecordRequests = true diff --git a/bundle/configsync/diff.go b/bundle/configsync/diff.go index 17ed3b30d5e..ca5b2c9410b 100644 --- a/bundle/configsync/diff.go +++ b/bundle/configsync/diff.go @@ -149,7 +149,7 @@ func OpenDeploymentState(ctx context.Context, b *bundle.Bundle, engine engine.En deployBundle := &direct.DeploymentBundle{} _, statePath := b.StateFilenameConfigSnapshot(ctx) - if err := deployBundle.StateDB.Open(ctx, statePath, dstate.WithRecovery(true), dstate.WithWrite(false), nil, nil); err != nil { + if err := deployBundle.StateDB.Open(ctx, statePath, dstate.WithRecovery(true), dstate.WithWrite(false), nil); err != nil { return nil, fmt.Errorf("failed to open state: %w", err) } return deployBundle, nil diff --git a/bundle/configsync/variables.go b/bundle/configsync/variables.go index be3e536f37f..433b607a037 100644 --- a/bundle/configsync/variables.go +++ b/bundle/configsync/variables.go @@ -147,7 +147,7 @@ func resourceIDLookup(ctx context.Context, b *bundle.Bundle) func(string) string } _, statePath := b.StateFilenameConfigSnapshot(ctx) db := &dstate.DeploymentState{} - if err := db.Open(ctx, statePath, dstate.WithRecovery(false), dstate.WithWrite(false), nil, nil); err != nil { + if err := db.Open(ctx, statePath, dstate.WithRecovery(false), dstate.WithWrite(false), nil); err != nil { log.Debugf(ctx, "variable restoration: failed to open state DB at %s: %v", statePath, err) return nil } diff --git a/bundle/direct/bind.go b/bundle/direct/bind.go index ccfbcf788ab..ec910b2734e 100644 --- a/bundle/direct/bind.go +++ b/bundle/direct/bind.go @@ -62,7 +62,7 @@ type BindResult struct { func (b *DeploymentBundle) Bind(ctx context.Context, client *databricks.WorkspaceClient, configRoot *config.Root, statePath, resourceKey, resourceID string) (*BindResult, error) { // Check if the resource is already managed (bound to a different ID) var checkStateDB dstate.DeploymentState - if err := checkStateDB.Open(ctx, statePath, dstate.WithRecovery(true), dstate.WithWrite(false), nil, nil); err == nil { + if err := checkStateDB.Open(ctx, statePath, dstate.WithRecovery(true), dstate.WithWrite(false), nil); err == nil { existingID := checkStateDB.GetResourceID(resourceKey) if _, err := checkStateDB.Finalize(ctx); err != nil { log.Warnf(ctx, "failed to finalize state: %v", err) @@ -86,7 +86,7 @@ func (b *DeploymentBundle) Bind(ctx context.Context, client *databricks.Workspac } // Open temp state - err := b.StateDB.Open(ctx, tmpStatePath, dstate.WithRecovery(false), dstate.WithWrite(true), nil, nil) + err := b.StateDB.Open(ctx, tmpStatePath, dstate.WithRecovery(false), dstate.WithWrite(true), nil) if err != nil { os.Remove(tmpStatePath) return nil, err @@ -109,7 +109,7 @@ func (b *DeploymentBundle) Bind(ctx context.Context, client *databricks.Workspac log.Infof(ctx, "Bound %s to id=%s (in temp state)", resourceKey, resourceID) // First plan + update: populate state with resolved config - err = b.StateDB.Open(ctx, tmpStatePath, dstate.WithRecovery(true), dstate.WithWrite(false), nil, nil) + err = b.StateDB.Open(ctx, tmpStatePath, dstate.WithRecovery(true), dstate.WithWrite(false), nil) if err != nil { os.Remove(tmpStatePath) return nil, err @@ -145,7 +145,7 @@ func (b *DeploymentBundle) Bind(ctx context.Context, client *databricks.Workspac } } - err = b.StateDB.Open(ctx, tmpStatePath, dstate.WithRecovery(true), dstate.WithWrite(true), nil, nil) + err = b.StateDB.Open(ctx, tmpStatePath, dstate.WithRecovery(true), dstate.WithWrite(true), nil) if err != nil { os.Remove(tmpStatePath) return nil, err @@ -165,7 +165,7 @@ func (b *DeploymentBundle) Bind(ctx context.Context, client *databricks.Workspac } // Second plan: this is the plan to present to the user (change between remote resource and config) - err = b.StateDB.Open(ctx, tmpStatePath, dstate.WithRecovery(true), dstate.WithWrite(false), nil, nil) + err = b.StateDB.Open(ctx, tmpStatePath, dstate.WithRecovery(true), dstate.WithWrite(false), nil) if err != nil { os.Remove(tmpStatePath) return nil, err @@ -215,7 +215,7 @@ func (result *BindResult) Cancel() { // Unbind removes a resource from direct engine state without deleting // the workspace resource. Also removes associated permissions/grants entries. func (b *DeploymentBundle) Unbind(ctx context.Context, statePath, resourceKey string) error { - err := b.StateDB.Open(ctx, statePath, dstate.WithRecovery(true), dstate.WithWrite(true), nil, nil) + err := b.StateDB.Open(ctx, statePath, dstate.WithRecovery(true), dstate.WithWrite(true), nil) if err != nil { return err } diff --git a/bundle/direct/dstate/dms.go b/bundle/direct/dstate/dms.go index e09cb859221..0941d87d35c 100644 --- a/bundle/direct/dstate/dms.go +++ b/bundle/direct/dstate/dms.go @@ -19,13 +19,10 @@ import ( // deployment. Once DMS is authoritative its resource set is trusted even when // empty (a successful deploy with no resources); the file's resources are only // used when DMS has no successful version, or when the user opts out of -// recording deployment history. The caller holds db.mu and has already -// populated db.Data from the file, including the DeploymentID. -// -// cfg is threaded in only for the temporary raw read in -// deploymentHasSuccessfulVersion; see the TODO there. -func (db *DeploymentState) overlayDMSState(ctx context.Context, client bundledeployments.BundleDeploymentsInterface, cfg *sdkconfig.Config) error { - authoritative, err := deploymentHasSuccessfulVersion(ctx, cfg, db.Data.DeploymentID) +// recording deployment history. The caller holds db.mu, has already populated +// db.Data from the file, and has resolved src.DeploymentID. +func (db *DeploymentState) overlayDMSState(ctx context.Context, src *DMSSource) error { + authoritative, err := deploymentHasSuccessfulVersion(ctx, src.Config, src.DeploymentID) if err != nil { return err } @@ -35,7 +32,7 @@ func (db *DeploymentState) overlayDMSState(ctx context.Context, client bundledep return nil } - resources, err := fetchDeploymentResources(ctx, client, db.Data.DeploymentID, db.Data.State) + resources, err := fetchDeploymentResources(ctx, src.Client, src.DeploymentID, db.Data.State) if err != nil { return err } @@ -63,8 +60,7 @@ func (db *DeploymentState) overlayDMSState(ctx context.Context, client bundledep // because last_successful_version_id is still stage:DEVELOPMENT in the proto // and therefore stripped from the generated SDK. Once the field is promoted to // PRIVATE_PREVIEW and regenerated, replace the raw call with -// client.GetDeployment(...).LastSuccessfulVersionId and drop the cfg argument -// (revert overlayDMSState/Open back to taking only the typed client). +// client.GetDeployment(...).LastSuccessfulVersionId and drop DMSSource.Config. func deploymentHasSuccessfulVersion(ctx context.Context, cfg *sdkconfig.Config, deploymentID string) (bool, error) { apiClient, err := client.New(cfg) if err != nil { diff --git a/bundle/direct/dstate/state.go b/bundle/direct/dstate/state.go index e5eb44df24a..c27de3ca44d 100644 --- a/bundle/direct/dstate/state.go +++ b/bundle/direct/dstate/state.go @@ -74,11 +74,6 @@ type DeploymentState struct { // Maps resource key to ID. Unlike Data.State, this is up to date during writes (deploys). stateIDs map[string]string - - // headerDirty records that a header field changed in memory during this - // deployment (today only the DMS deployment ID), so the state file must be - // written on Finalize even when the WAL carried no resource entries. - headerDirty bool } type Header struct { @@ -87,13 +82,6 @@ type Header struct { Lineage string `json:"lineage"` Serial int `json:"serial"` - // DeploymentID is the ID the deployment metadata service (DMS) assigned to - // this deployment. Unlike Lineage (a locally generated identifier for the - // state file), it is minted server-side by CreateDeployment and stored here so - // later deploys can find the same DMS deployment record and read its state. - // Empty/omitted until the bundle first records to DMS. - DeploymentID string `json:"deployment_id,omitempty"` - // Features maps each feature flag this state depends on to a (currently empty) // value. This CLI writes no features; it only reads the field to detect a state // that depends on features it lacks and refuse it (see migrateState). It is a @@ -223,33 +211,6 @@ func (db *DeploymentState) GetOrInitLineage() string { return db.Data.Lineage } -// GetDeploymentID returns the DMS deployment ID recorded in the state, or an -// empty string if this bundle has not yet recorded a deployment to DMS. -func (db *DeploymentState) GetDeploymentID() string { - db.mu.Lock() - defer db.mu.Unlock() - return db.Data.DeploymentID -} - -// SetDeploymentID stores the DMS-assigned deployment ID in the in-memory state -// header. It is set during deploy, after CreateDeployment returns the -// server-generated ID, and persisted to the state file by Finalize. Storing it -// on db.Data (not the WAL header, which is written before the ID is known) -// means the subsequent state write carries it forward. -// -// The header is marked dirty so Finalize persists it even when the deploy wrote -// no resource entries; otherwise a bundle with no resources would mint a fresh -// deployment record on every deploy, leaking one orphan per run. -func (db *DeploymentState) SetDeploymentID(id string) { - db.mu.Lock() - defer db.mu.Unlock() - if db.Data.DeploymentID == id { - return - } - db.Data.DeploymentID = id - db.headerDirty = true -} - type ( // If true, then Open reads the WAL and merges it in the state. If false, and WAL is present, Open returns an error. WithRecovery bool @@ -259,19 +220,32 @@ type ( WithWrite bool ) +// DMSSource tells Open to read resource state from the deployment metadata +// service instead of the state file. A nil *DMSSource keeps Open file-only. +type DMSSource struct { + // Client is the DMS client used to list the deployment's resources. + Client bundledeployments.BundleDeploymentsInterface + + // Config accompanies Client (both come from the same workspace client) and is + // used only for a temporary raw read of last_successful_version_id; see the + // TODO in deploymentHasSuccessfulVersion. + Config *sdkconfig.Config + + // DeploymentID identifies the deployment in DMS, resolved from the + // deployment's workspace node (see dms.ResolveDeploymentID). It is empty for a + // bundle that has not recorded a deployment yet. + DeploymentID string +} + // Open reads the deployment state from disk (and recovers the WAL when -// withRecovery is set). When dmsClient is non-nil, the deployment metadata +// withRecovery is set). When dmsSource is non-nil, the deployment metadata // service is the source of truth for resource state: if DMS holds a // successfully completed version for this deployment, the resources read from // the file are replaced with the ones recorded in DMS. The local identity -// (lineage, serial, and deployment ID) always comes from the file, since that -// is what the write path increments and carries forward. A nil dmsClient keeps -// the behavior file-only. -// -// dmsCfg accompanies dmsClient (both come from the same workspace client) and -// is used only for a temporary raw read of last_successful_version_id; see the -// TODO in deploymentHasSuccessfulVersion. -func (db *DeploymentState) Open(ctx context.Context, path string, withRecovery WithRecovery, withWrite WithWrite, dmsClient bundledeployments.BundleDeploymentsInterface, dmsCfg *sdkconfig.Config) error { +// (lineage and serial) always comes from the file, since that is what the write +// path increments and carries forward. A nil dmsSource keeps the behavior +// file-only. +func (db *DeploymentState) Open(ctx context.Context, path string, withRecovery WithRecovery, withWrite WithWrite, dmsSource *DMSSource) error { db.mu.Lock() defer db.mu.Unlock() @@ -319,7 +293,7 @@ func (db *DeploymentState) Open(ctx context.Context, path string, withRecovery W return fmt.Errorf("migrating state %s: %w", path, err) } - if dmsClient != nil { + if dmsSource != nil { // Only deployments that start out empty are recorded in DMS. Resources // tracked in a state file that DMS does not know about are not in DMS and // never will be: the first recorded deploy would create a deployment whose @@ -327,9 +301,9 @@ func (db *DeploymentState) Open(ctx context.Context, path string, withRecovery W // authoritative for everything (see overlayDMSState). Resources this bundle // already owns would look absent and be created a second time. // - // A state file that DMS already owns (it carries a deployment ID) is fine — - // that is a bundle that opted in while it was still empty. So is a state file - // with no resources, e.g. one left behind by a destroy. + // A deployment DMS already owns (deploymentID is non-empty) is fine — that is + // a bundle that opted in while it was still empty. So is a state file with no + // resources, e.g. one left behind by a destroy. // // TODO(DMS): lift this restriction by upgrading an existing state in place. // That means writing the state at featureStateVersion (3) with a feature flag @@ -339,11 +313,11 @@ func (db *DeploymentState) Open(ctx context.Context, path string, withRecovery W // exists (see featureStateVersion and Header.Features); once it is written, // this check goes away and record_deployment_history becomes usable on // existing bundles. - if db.Data.DeploymentID == "" && len(db.Data.State) > 0 { + if dmsSource.DeploymentID == "" && len(db.Data.State) > 0 { return fmt.Errorf("cannot record deployment history for a bundle that already has deployed resources tracked in %s: only new deployments can be recorded. Remove experimental.record_deployment_history, or destroy the bundle and deploy it again", path) } - if db.Data.DeploymentID != "" { - if err := db.overlayDMSState(ctx, dmsClient, dmsCfg); err != nil { + if dmsSource.DeploymentID != "" { + if err := db.overlayDMSState(ctx, dmsSource); err != nil { return err } } @@ -488,12 +462,7 @@ func (db *DeploymentState) mergeWalIntoState(ctx context.Context) (bool, error) } } - hasEntries := lineNumber > 1 - - // A header-only WAL still has to be persisted when a header field changed in - // memory during this deployment (the DMS deployment ID): dropping the write - // would lose the ID and make the next deploy create a second deployment. - persist := hasEntries || db.headerDirty + persist := lineNumber > 1 // Only advance the serial when the state file is actually written, because // the caller (replayWAL) persists it only in that case. A header-only WAL diff --git a/bundle/direct/dstate/state_test.go b/bundle/direct/dstate/state_test.go index 6530ca049d9..a9c90530514 100644 --- a/bundle/direct/dstate/state_test.go +++ b/bundle/direct/dstate/state_test.go @@ -20,91 +20,30 @@ func TestOpenSaveFinalizeRoundTrip(t *testing.T) { path := filepath.Join(t.TempDir(), "state.json") var db DeploymentState - require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil, nil)) + require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil)) require.NoError(t, db.SaveState("jobs.my_job", "123", map[string]string{"key": "val"}, nil)) mustFinalize(t, &db) // Re-open and verify persisted data. var db2 DeploymentState - require.NoError(t, db2.Open(t.Context(), path, WithRecovery(false), WithWrite(false), nil, nil)) + require.NoError(t, db2.Open(t.Context(), path, WithRecovery(false), WithWrite(false), nil)) assert.Equal(t, 1, db2.Data.Serial) assert.Equal(t, "123", db2.GetResourceID("jobs.my_job")) mustFinalize(t, &db2) } -func TestDeploymentIDPersistsAcrossOpen(t *testing.T) { - path := filepath.Join(t.TempDir(), "state.json") - - var db DeploymentState - require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil, nil)) - assert.Empty(t, db.GetDeploymentID()) - - // The deployment ID is set during deploy (after CreateDeployment) and - // persisted by Finalize even though it is not part of the WAL header. - db.SetDeploymentID("server-assigned-id") - require.NoError(t, db.SaveState("jobs.my_job", "123", map[string]string{}, nil)) - mustFinalize(t, &db) - - var reopened DeploymentState - require.NoError(t, reopened.Open(t.Context(), path, WithRecovery(false), WithWrite(false), nil, nil)) - assert.Equal(t, "server-assigned-id", reopened.GetDeploymentID()) - mustFinalize(t, &reopened) -} - func TestFinalizeWithNoEntriesDoesNotWriteStateFile(t *testing.T) { path := filepath.Join(t.TempDir(), "state.json") var db DeploymentState - require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil, nil)) + require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil)) mustFinalize(t, &db) _, err := os.Stat(path) assert.ErrorIs(t, err, os.ErrNotExist) } -func TestDeploymentIDPersistsWithNoResourceEntries(t *testing.T) { - path := filepath.Join(t.TempDir(), "state.json") - - // A bundle with no resources writes no WAL entries, but the deployment ID - // still has to be persisted: otherwise the next deploy sees no ID and creates - // a second deployment record, leaking one per deploy. - var db DeploymentState - require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil, nil)) - db.SetDeploymentID("server-assigned-id") - mustFinalize(t, &db) - - var reopened DeploymentState - require.NoError(t, reopened.Open(t.Context(), path, WithRecovery(false), WithWrite(false), nil, nil)) - assert.Equal(t, "server-assigned-id", reopened.GetDeploymentID()) - assert.Equal(t, 1, reopened.Data.Serial) - mustFinalize(t, &reopened) -} - -func TestSetDeploymentIDToSameValueDoesNotWriteStateFile(t *testing.T) { - path := filepath.Join(t.TempDir(), "state.json") - - var db DeploymentState - require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil, nil)) - db.SetDeploymentID("server-assigned-id") - require.NoError(t, db.SaveState("jobs.my_job", "123", map[string]string{}, nil)) - mustFinalize(t, &db) - - before, err := os.ReadFile(path) - require.NoError(t, err) - - // Re-setting the same ID is not a header change, so a deploy that commits - // nothing must not bump the serial (see mergeWalIntoState). - var reopened DeploymentState - require.NoError(t, reopened.Open(t.Context(), path, WithRecovery(false), WithWrite(true), nil, nil)) - reopened.SetDeploymentID("server-assigned-id") - mustFinalize(t, &reopened) - - after, err := os.ReadFile(path) - require.NoError(t, err) - assert.Equal(t, string(before), string(after)) -} - func TestExportStateFromDataJobRunJobID(t *testing.T) { data := Database{ State: map[string]ResourceEntry{ @@ -154,10 +93,10 @@ func TestPanicOnDoubleOpen(t *testing.T) { path := filepath.Join(t.TempDir(), "state.json") var db DeploymentState - require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil, nil)) + require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil)) assert.Panics(t, func() { - _ = db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil, nil) + _ = db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil) }) mustFinalize(t, &db) } @@ -168,12 +107,12 @@ func TestHeaderOnlyWALRecoveryDoesNotAdvanceSerial(t *testing.T) { // Commit serial 1 with one resource. var db DeploymentState - require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil, nil)) + require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil)) require.NoError(t, db.SaveState("jobs.my_job", "123", map[string]string{}, nil)) mustFinalize(t, &db) var committed DeploymentState - require.NoError(t, committed.Open(t.Context(), path, WithRecovery(false), WithWrite(false), nil, nil)) + require.NoError(t, committed.Open(t.Context(), path, WithRecovery(false), WithWrite(false), nil)) lineage := committed.Data.Lineage require.Equal(t, 1, committed.Data.Serial) mustFinalize(t, &committed) @@ -189,7 +128,7 @@ func TestHeaderOnlyWALRecoveryDoesNotAdvanceSerial(t *testing.T) { require.NoError(t, os.WriteFile(walPath, append(headerLine, '\n'), 0o600)) var recovered DeploymentState - require.NoError(t, recovered.Open(t.Context(), path, WithRecovery(true), WithWrite(false), nil, nil)) + require.NoError(t, recovered.Open(t.Context(), path, WithRecovery(true), WithWrite(false), nil)) assert.Equal(t, 1, recovered.Data.Serial) assert.Equal(t, "123", recovered.GetResourceID("jobs.my_job")) assert.NoFileExists(t, walPath) @@ -232,17 +171,17 @@ func TestDeleteState(t *testing.T) { path := filepath.Join(t.TempDir(), "state.json") var db DeploymentState - require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil, nil)) + require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil)) require.NoError(t, db.SaveState("jobs.my_job", "123", map[string]string{}, nil)) mustFinalize(t, &db) var db2 DeploymentState - require.NoError(t, db2.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil, nil)) + require.NoError(t, db2.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil)) require.NoError(t, db2.DeleteState("jobs.my_job")) mustFinalize(t, &db2) var db3 DeploymentState - require.NoError(t, db3.Open(t.Context(), path, WithRecovery(false), WithWrite(false), nil, nil)) + require.NoError(t, db3.Open(t.Context(), path, WithRecovery(false), WithWrite(false), nil)) assert.Equal(t, 2, db3.Data.Serial) assert.Empty(t, db3.GetResourceID("jobs.my_job")) mustFinalize(t, &db3) @@ -254,7 +193,7 @@ func TestGetOrInitLineageReadableBeforeWriteAndPersisted(t *testing.T) { // Fresh state opened read-only, as the deploy does before planning: no // lineage yet. var db DeploymentState - require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(false), nil, nil)) + require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(false), nil)) require.Empty(t, db.Data.Lineage) // GetOrInitLineage initializes the lineage and makes it readable before any @@ -271,7 +210,7 @@ func TestGetOrInitLineageReadableBeforeWriteAndPersisted(t *testing.T) { // Re-open: the persisted lineage matches the one read before the write. var reopened DeploymentState - require.NoError(t, reopened.Open(t.Context(), path, WithRecovery(false), WithWrite(false), nil, nil)) + require.NoError(t, reopened.Open(t.Context(), path, WithRecovery(false), WithWrite(false), nil)) assert.Equal(t, lineage, reopened.Data.Lineage) mustFinalize(t, &reopened) } diff --git a/bundle/phases/deploy.go b/bundle/phases/deploy.go index 792c016f963..3d70a218b15 100644 --- a/bundle/phases/deploy.go +++ b/bundle/phases/deploy.go @@ -169,7 +169,11 @@ func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHand // nothing; the deferred CompleteVersion is a no-op until CreateVersion runs. // CompleteVersion is deferred before lock.Release so it runs while the lock // is still held (defers run last-in-first-out). - recorder := newDeploymentRecorder(ctx, b, stateEngine, dms.VersionTypeDeploy) + recorder, err := newDeploymentRecorder(ctx, b, stateEngine, dms.VersionTypeDeploy) + if err != nil { + logdiag.LogError(ctx, err) + return + } defer func() { if err := recorder.CompleteVersion(ctx, !logdiag.HasError(ctx)); err != nil { logdiag.LogError(ctx, err) @@ -276,11 +280,9 @@ func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHand return } if recorder != nil { - // On a first deploy the server assigned the deployment ID; persist it in - // state (Finalize writes it to disk) so later deploys reuse the record. // Record operations under the version just created so DMS holds the - // deployed resource state. - b.DeploymentBundle.StateDB.SetDeploymentID(recorder.DeploymentID()) + // deployed resource state. On a first deploy the deployment ID was only + // assigned by CreateVersion above, so this must come after it. b.DeploymentBundle.OpRec = direct.NewOperationRecorder( b.WorkspaceClient(ctx).BundleDeployments, recorder.DeploymentID(), diff --git a/bundle/phases/destroy.go b/bundle/phases/destroy.go index 244f593476f..2925e80bca8 100644 --- a/bundle/phases/destroy.go +++ b/bundle/phases/destroy.go @@ -137,7 +137,11 @@ func Destroy(ctx context.Context, b *bundle.Bundle, engine engine.EngineType) { // created until the destroy is approved (below), so a cancelled destroy // records nothing; the deferred CompleteVersion is a no-op until then. It is // deferred before lock.Release so it runs while the lock is still held. - recorder := newDeploymentRecorder(ctx, b, engine, dms.VersionTypeDestroy) + recorder, err := newDeploymentRecorder(ctx, b, engine, dms.VersionTypeDestroy) + if err != nil { + logdiag.LogError(ctx, err) + return + } defer func() { if err := recorder.CompleteVersion(ctx, !logdiag.HasError(ctx)); err != nil { logdiag.LogError(ctx, err) diff --git a/bundle/phases/dms.go b/bundle/phases/dms.go index 667ef8627aa..02254237595 100644 --- a/bundle/phases/dms.go +++ b/bundle/phases/dms.go @@ -14,24 +14,31 @@ import ( // // Recording is enabled only when experimental.record_deployment_history is set // AND the engine is direct: DMS resource state is tracked per direct-engine -// deployment, and only the direct engine opens the state DB where the -// deployment ID is stored. Returning nil for terraform leaves those deployments -// untouched. +// deployment. Returning nil for terraform leaves those deployments untouched. // -// The deployment ID passed to the recorder is the one persisted in state from a -// previous deploy; it is empty on a bundle's first recorded deploy, in which -// case the recorder creates the deployment and the server assigns the ID. -func newDeploymentRecorder(ctx context.Context, b *bundle.Bundle, eng engine.EngineType, versionType dms.VersionType) *dms.Recorder { +// The deployment ID is resolved from the workspace rather than from local state +// (see dms.ResolveDeploymentID). The lookup happens here, after the deployment +// lock has been acquired, so it observes any deployment a concurrent deploy +// created. It is empty on a bundle's first recorded deploy, in which case the +// recorder creates the deployment and the server assigns the ID. +func newDeploymentRecorder(ctx context.Context, b *bundle.Bundle, eng engine.EngineType, versionType dms.VersionType) (*dms.Recorder, error) { if b.Config.Experimental == nil || !b.Config.Experimental.RecordDeploymentHistory { - return nil + return nil, nil } if !eng.IsDirect() { - return nil + return nil, nil + } + + statePath := b.Config.Workspace.StatePath + deploymentID, err := dms.ResolveDeploymentID(ctx, b.WorkspaceClient(ctx), statePath) + if err != nil { + return nil, err } return dms.NewRecorder( b.WorkspaceClient(ctx).BundleDeployments, - b.DeploymentBundle.StateDB.GetDeploymentID(), + deploymentID, + statePath, b.Config.Bundle.Target, versionType, - ) + ), nil } diff --git a/cmd/bundle/generate/dashboard.go b/cmd/bundle/generate/dashboard.go index 4866f27c5b3..2b286bcad3d 100644 --- a/cmd/bundle/generate/dashboard.go +++ b/cmd/bundle/generate/dashboard.go @@ -404,7 +404,7 @@ func (d *dashboard) runForResource(ctx context.Context, b *bundle.Bundle) { var state statemgmt.ExportedResourcesMap if stateDesc.Engine.IsDirect() { _, localPath := b.StateFilenameDirect(ctx) - if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WithRecovery(true), dstate.WithWrite(false), nil, nil); err != nil { + if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WithRecovery(true), dstate.WithWrite(false), nil); err != nil { logdiag.LogError(ctx, err) return } diff --git a/cmd/bundle/generate/genie_space.go b/cmd/bundle/generate/genie_space.go index b5dbeed6c56..48ecc92a6cd 100644 --- a/cmd/bundle/generate/genie_space.go +++ b/cmd/bundle/generate/genie_space.go @@ -322,7 +322,7 @@ func (g *genieSpace) runForResource(ctx context.Context, b *bundle.Bundle) { var state statemgmt.ExportedResourcesMap if stateDesc.Engine.IsDirect() { _, localPath := b.StateFilenameDirect(ctx) - if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WithRecovery(true), dstate.WithWrite(false), nil, nil); err != nil { + if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WithRecovery(true), dstate.WithWrite(false), nil); err != nil { logdiag.LogError(ctx, err) return } diff --git a/cmd/bundle/utils/process.go b/cmd/bundle/utils/process.go index f556c2b3450..e9840188db4 100644 --- a/cmd/bundle/utils/process.go +++ b/cmd/bundle/utils/process.go @@ -20,13 +20,12 @@ import ( "github.com/databricks/cli/cmd/root" "github.com/databricks/cli/internal/build" "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dms" "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/logdiag" "github.com/databricks/cli/libs/sync" "github.com/databricks/cli/libs/telemetry/protos" - sdkconfig "github.com/databricks/databricks-sdk-go/config" - "github.com/databricks/databricks-sdk-go/service/bundledeployments" "github.com/spf13/cobra" ) @@ -215,18 +214,24 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle _, localPath := b.StateFilenameDirect(ctx) // When the bundle records deployment history, the deployment metadata - // service owns resource state, so hand Open its client to overlay DMS - // state on top of the local identity (lineage/serial/deployment ID). - // Reads open the state write-disabled, so no lineage is minted here. - // dmsCfg accompanies the client for a temporary raw read (see the TODO - // in dstate.deploymentHasSuccessfulVersion). - var dmsClient bundledeployments.BundleDeploymentsInterface - var dmsCfg *sdkconfig.Config + // service owns resource state, so hand Open a DMS source to overlay that + // state on top of the local identity (lineage/serial). Reads open the + // state write-disabled, so no lineage is minted here. + var dmsSource *dstate.DMSSource if b.Config.Experimental != nil && b.Config.Experimental.RecordDeploymentHistory { - dmsClient = b.WorkspaceClient(ctx).BundleDeployments - dmsCfg = b.WorkspaceClient(ctx).Config + w := b.WorkspaceClient(ctx) + deploymentID, err := dms.ResolveDeploymentID(ctx, w, b.Config.Workspace.StatePath) + if err != nil { + logdiag.LogError(ctx, err) + return b, stateDesc, root.ErrAlreadyPrinted + } + dmsSource = &dstate.DMSSource{ + Client: w.BundleDeployments, + Config: w.Config, + DeploymentID: deploymentID, + } } - if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WithRecovery(true), dstate.WithWrite(false), dmsClient, dmsCfg); err != nil { + if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WithRecovery(true), dstate.WithWrite(false), dmsSource); err != nil { logdiag.LogError(ctx, err) return b, stateDesc, root.ErrAlreadyPrinted } diff --git a/libs/dms/recorder.go b/libs/dms/recorder.go index 1a113491cf9..6fa9a1a24be 100644 --- a/libs/dms/recorder.go +++ b/libs/dms/recorder.go @@ -30,13 +30,15 @@ const ( // Recorder records a single deploy/destroy as a version with DMS. // // The deployment ID is assigned by the server on the first deploy: NewRecorder -// is given the ID persisted in state (empty on a bundle's first-ever recorded -// deploy), and CreateVersion creates the deployment record when that ID is -// empty and exposes the server-assigned ID via DeploymentID so the caller can -// persist it. Later deploys pass the stored ID back in and reuse the record. +// is given the ID resolved from the workspace (empty on a bundle's first-ever +// recorded deploy, see ResolveDeploymentID), and CreateVersion creates the +// deployment record when that ID is empty. Later deploys resolve the same ID +// from the deployment's workspace node and reuse the record; a destroy deletes +// the record and its node, so the next deploy starts over from empty. type Recorder struct { svc bundledeployments.BundleDeploymentsInterface deploymentID string + statePath string targetName string versionType VersionType @@ -46,12 +48,15 @@ type Recorder struct { } // NewRecorder returns a Recorder for the given deployment. deploymentID is the -// DMS deployment ID persisted in state, or empty if this bundle has not yet -// recorded a deployment (the server assigns one during CreateVersion). -func NewRecorder(svc bundledeployments.BundleDeploymentsInterface, deploymentID, targetName string, versionType VersionType) *Recorder { +// ID resolved from the deployment's workspace node, or empty if this bundle has +// not yet recorded a deployment (the server assigns one during CreateVersion). +// statePath is the bundle's remote state directory, under which DMS registers +// the deployment node. +func NewRecorder(svc bundledeployments.BundleDeploymentsInterface, deploymentID, statePath, targetName string, versionType VersionType) *Recorder { return &Recorder{ svc: svc, deploymentID: deploymentID, + statePath: statePath, targetName: targetName, versionType: versionType, } @@ -59,8 +64,7 @@ func NewRecorder(svc bundledeployments.BundleDeploymentsInterface, deploymentID, // DeploymentID returns the DMS deployment ID this recorder is bound to. It is // empty until CreateVersion has created the deployment record (on a first -// deploy) and non-empty afterwards, so callers persist it once CreateVersion -// succeeds. +// deploy) and non-empty afterwards, so callers can parent operations under it. func (r *Recorder) DeploymentID() string { if r == nil { return "" @@ -140,41 +144,40 @@ func (r *Recorder) CompleteVersion(ctx context.Context, success bool) error { } // createDeploymentVersion ensures the deployment record exists, then creates a -// new version under it. When no deployment ID is stored, or the stored one no -// longer exists in DMS, it creates the deployment and lets the server assign the -// ID; otherwise it reads the existing deployment to compute the next version -// number. +// new version under it. With no deployment ID it creates the deployment and lets +// the server assign the ID; otherwise it reads the existing deployment to +// compute the next version number. func (r *Recorder) createDeploymentVersion(ctx context.Context) (versionID string, err error) { if r.deploymentID != "" { - // Existing deployment: read it to compute the next version number. + // Existing deployment: read it to compute the next version number. A 404 is + // not recovered from by creating a second deployment: the ID was just + // resolved from the deployment's workspace node, which the service trashes + // when it deletes the record, so a missing record here means the two are out + // of sync and creating another one would collide on the same node path. dep, getErr := r.svc.GetDeployment(ctx, bundledeployments.GetDeploymentRequest{ Name: "deployments/" + r.deploymentID, }) - switch { - case getErr == nil: - lastVersion, parseErr := strconv.ParseInt(dep.LastVersionId, 10, 64) - if parseErr != nil { - return "", fmt.Errorf("failed to parse last_version_id %q: %w", dep.LastVersionId, parseErr) - } - versionID = strconv.FormatInt(lastVersion+1, 10) - case errors.Is(getErr, apierr.ErrNotFound): - // The record the state points at is gone: a successful destroy deletes - // it (leaving the ID behind in the local state file), and it can also be - // deleted out of band. Recording must not dead-end on it, so fall back to - // creating a new deployment; the caller persists the new ID. - log.Debugf(ctx, "Deployment %s no longer exists in the deployment metadata service, creating a new one", r.deploymentID) - r.deploymentID = "" - default: + if getErr != nil { return "", fmt.Errorf("failed to get deployment: %w", getErr) } - } - - if r.deploymentID == "" { - // First deploy: create the deployment with an empty ID so the server - // assigns one, then start at version 1. + lastVersion, parseErr := strconv.ParseInt(dep.LastVersionId, 10, 64) + if parseErr != nil { + return "", fmt.Errorf("failed to parse last_version_id %q: %w", dep.LastVersionId, parseErr) + } + versionID = strconv.FormatInt(lastVersion+1, 10) + } else { + // First deploy: create the deployment so the server assigns an ID, then + // start at version 1. + // + // initial_parent_path is required: the service creates the deployment's + // BUNDLE_DEPLOYMENT node under it, and that node's ID becomes the + // deployment ID that ResolveDeploymentID reads back on later deploys. The + // folder must already exist, which it does by this point - the deployment + // lock lives in the same directory. dep, createErr := r.svc.CreateDeployment(ctx, bundledeployments.CreateDeploymentRequest{ Deployment: bundledeployments.Deployment{ - TargetName: r.targetName, + InitialParentPath: r.statePath, + TargetName: r.targetName, }, }) if createErr != nil { diff --git a/libs/dms/recorder_test.go b/libs/dms/recorder_test.go index 9b60078635f..6c2f334c946 100644 --- a/libs/dms/recorder_test.go +++ b/libs/dms/recorder_test.go @@ -12,6 +12,10 @@ import ( "github.com/stretchr/testify/require" ) +// testStatePath is the bundle state directory the recorder registers the +// deployment node under; several tests assert it round-trips to the service. +const testStatePath = "/Workspace/Users/me/.bundle/proj/dev/state" + // fakeDMS records the calls the recorder makes and lets a test script the // server-side responses. It embeds the SDK interface so it satisfies it while // only overriding the methods the recorder uses. @@ -33,11 +37,9 @@ type fakeDMS struct { func (f *fakeDMS) CreateDeployment(ctx context.Context, req bundledeployments.CreateDeploymentRequest) (*bundledeployments.Deployment, error) { f.created = append(f.created, req) - id := req.DeploymentId - if id == "" { - id = f.assignedID - } - return &bundledeployments.Deployment{Name: "deployments/" + id}, nil + // The server always assigns the ID; it is the ID of the workspace node it + // creates under initial_parent_path. + return &bundledeployments.Deployment{Name: "deployments/" + f.assignedID}, nil } func (f *fakeDMS) GetDeployment(ctx context.Context, req bundledeployments.GetDeploymentRequest) (*bundledeployments.Deployment, error) { @@ -66,16 +68,18 @@ func (f *fakeDMS) Heartbeat(ctx context.Context, req bundledeployments.Heartbeat func TestRecorderFirstDeployCreatesDeploymentWithServerAssignedID(t *testing.T) { f := &fakeDMS{assignedID: "server-generated-id"} - // A first deploy has no stored deployment ID. - r := NewRecorder(f, "", "dev", VersionTypeDeploy) + // A first deploy resolves no deployment ID from the workspace. + r := NewRecorder(f, "", testStatePath, "dev", VersionTypeDeploy) require.NoError(t, r.CreateVersion(t.Context())) - // The deployment was created with an empty ID so the server assigns one, and - // the recorder exposes the assigned ID for the caller to persist. + // The server assigned the ID, and the recorder exposes it for the rest of the + // deploy (it parents the operations recorded under this version). require.Len(t, f.created, 1) - assert.Empty(t, f.created[0].DeploymentId) assert.Equal(t, "server-generated-id", r.DeploymentID()) + // initial_parent_path is required: the service creates the deployment node + // under it, and that node is what ResolveDeploymentID looks up later. + assert.Equal(t, testStatePath, f.created[0].Deployment.InitialParentPath) // The first version is 1, parented under the assigned deployment. require.Len(t, f.versions, 1) @@ -96,7 +100,7 @@ func TestRecorderSubsequentDeployReusesDeploymentAndIncrementsVersion(t *testing }, } // A subsequent deploy passes the stored deployment ID. - r := NewRecorder(f, "stored-id", "dev", VersionTypeDeploy) + r := NewRecorder(f, "stored-id", testStatePath, "dev", VersionTypeDeploy) require.NoError(t, r.CreateVersion(t.Context())) @@ -107,39 +111,28 @@ func TestRecorderSubsequentDeployReusesDeploymentAndIncrementsVersion(t *testing assert.Equal(t, "stored-id", r.DeploymentID()) } -func TestRecorderStaleDeploymentIDCreatesNewDeployment(t *testing.T) { - f := &fakeDMS{ - assignedID: "fresh-id", - getDeployment: func(id string) (*bundledeployments.Deployment, error) { - return nil, fmt.Errorf("deployment %s: %w", id, apierr.ErrNotFound) - }, - } - // A deploy after a destroy still has the destroyed deployment's ID in state, - // but the record is gone. Recording must recover rather than fail the deploy. - r := NewRecorder(f, "destroyed-id", "dev", VersionTypeDeploy) - - require.NoError(t, r.CreateVersion(t.Context())) - - require.Len(t, f.created, 1) - assert.Equal(t, "fresh-id", r.DeploymentID()) - require.Len(t, f.versions, 1) - assert.Equal(t, "1", f.versions[0].VersionId) - assert.Equal(t, "deployments/fresh-id", f.versions[0].Parent) -} - func TestRecorderGetDeploymentErrorFailsDeploy(t *testing.T) { - f := &fakeDMS{ - getDeployment: func(id string) (*bundledeployments.Deployment, error) { - return nil, errors.New("boom") - }, + cases := map[string]error{ + // A resolved ID whose record is missing means the record and the workspace + // node it was resolved from are out of sync. Creating a second deployment + // would collide on the same node path, so fail instead. + "not found": fmt.Errorf("deployment: %w", apierr.ErrNotFound), + "other": errors.New("boom"), + } + for name, getErr := range cases { + t.Run(name, func(t *testing.T) { + f := &fakeDMS{ + getDeployment: func(id string) (*bundledeployments.Deployment, error) { + return nil, getErr + }, + } + r := NewRecorder(f, "stored-id", testStatePath, "dev", VersionTypeDeploy) + + err := r.CreateVersion(t.Context()) + assert.ErrorContains(t, err, "failed to get deployment") + assert.Empty(t, f.created) + }) } - r := NewRecorder(f, "stored-id", "dev", VersionTypeDeploy) - - // Only a missing deployment is recovered from; any other read failure is fatal - // rather than silently forking a second deployment record. - err := r.CreateVersion(t.Context()) - assert.ErrorContains(t, err, "failed to get deployment") - assert.Empty(t, f.created) } func TestRecorderDestroyDeletesDeploymentOnSuccess(t *testing.T) { @@ -148,7 +141,7 @@ func TestRecorderDestroyDeletesDeploymentOnSuccess(t *testing.T) { return &bundledeployments.Deployment{Name: "deployments/" + id, LastVersionId: "2"}, nil }, } - r := NewRecorder(f, "stored-id", "dev", VersionTypeDestroy) + r := NewRecorder(f, "stored-id", testStatePath, "dev", VersionTypeDestroy) require.NoError(t, r.CreateVersion(t.Context())) assert.Equal(t, bundledeployments.VersionTypeVersionTypeDestroy, f.versions[0].Version.VersionType) @@ -164,7 +157,7 @@ func TestRecorderFailedDestroyKeepsDeployment(t *testing.T) { return &bundledeployments.Deployment{Name: "deployments/" + id, LastVersionId: "2"}, nil }, } - r := NewRecorder(f, "stored-id", "dev", VersionTypeDestroy) + r := NewRecorder(f, "stored-id", testStatePath, "dev", VersionTypeDestroy) require.NoError(t, r.CreateVersion(t.Context())) require.NoError(t, r.CompleteVersion(t.Context(), false)) @@ -184,7 +177,7 @@ func TestNilRecorderIsNoOp(t *testing.T) { func TestRecorderCompleteVersionNoOpWithoutCreateVersion(t *testing.T) { f := &fakeDMS{} - r := NewRecorder(f, "stored-id", "dev", VersionTypeDeploy) + r := NewRecorder(f, "stored-id", testStatePath, "dev", VersionTypeDeploy) // CompleteVersion before CreateVersion is a no-op (nothing was claimed). require.NoError(t, r.CompleteVersion(t.Context(), true)) assert.Empty(t, f.completed) diff --git a/libs/dms/resolve.go b/libs/dms/resolve.go new file mode 100644 index 00000000000..0d26448e558 --- /dev/null +++ b/libs/dms/resolve.go @@ -0,0 +1,45 @@ +package dms + +import ( + "context" + "errors" + "fmt" + "path" + "strconv" + + "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/apierr" +) + +// DeploymentNodeName is the workspace node DMS creates for a deployment. The +// name is fixed for every deployment: the node *is* the bundle's state file. +// It must match DeploymentWhsClient.DEPLOYMENT_NODE_NAME on the service side. +const DeploymentNodeName = "resources.deployment.json" + +// ResolveDeploymentID returns the DMS deployment ID for the bundle whose state +// lives under statePath, or an empty string when the bundle has no deployment +// recorded yet. +// +// The ID is not stored anywhere by the CLI. DMS registers each deployment as a +// BUNDLE_DEPLOYMENT node at statePath/resources.deployment.json, and the +// workspace-assigned node ID *is* the deployment ID (see DeploymentHandler: +// deploymentId = Long.toString(createdNode.getId())). So a get-status on that +// path is the lookup, which keeps the workspace the single source of truth: a +// deployment that was destroyed or deleted out of band reports absent here +// rather than leaving a dangling ID behind in the local state file. +func ResolveDeploymentID(ctx context.Context, w *databricks.WorkspaceClient, statePath string) (string, error) { + nodePath := path.Join(statePath, DeploymentNodeName) + + obj, err := w.Workspace.GetStatusByPath(ctx, nodePath) + if err != nil { + if errors.Is(err, apierr.ErrNotFound) || errors.Is(err, apierr.ErrResourceDoesNotExist) { + return "", nil + } + return "", fmt.Errorf("looking up deployment at %s: %w", nodePath, err) + } + + if obj.ObjectId == 0 { + return "", fmt.Errorf("deployment at %s has no object ID", nodePath) + } + return strconv.FormatInt(obj.ObjectId, 10), nil +} diff --git a/libs/dms/resolve_test.go b/libs/dms/resolve_test.go new file mode 100644 index 00000000000..24d5303983d --- /dev/null +++ b/libs/dms/resolve_test.go @@ -0,0 +1,67 @@ +package dms + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/config" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// newTestClient returns a workspace client pointed at a server that serves a +// single get-status response. +func newTestClient(t *testing.T, statusCode int, body string) *databricks.WorkspaceClient { + t.Helper() + + var gotPath string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotPath = r.URL.Query().Get("path") + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(statusCode) + _, _ = w.Write([]byte(body)) + })) + t.Cleanup(srv.Close) + t.Cleanup(func() { + assert.Equal(t, "/Workspace/state/"+DeploymentNodeName, gotPath) + }) + + w, err := databricks.NewWorkspaceClient(&databricks.Config{ + Host: srv.URL, + Token: "token", + Credentials: config.PatCredentials{}, + }) + require.NoError(t, err) + return w +} + +func TestResolveDeploymentIDReturnsNodeID(t *testing.T) { + w := newTestClient(t, http.StatusOK, `{"object_type":"FILE","object_id":123456789,"path":"/Workspace/state/`+DeploymentNodeName+`"}`) + + // The workspace node ID is the deployment ID, so no local state is consulted. + id, err := ResolveDeploymentID(t.Context(), w, "/Workspace/state") + require.NoError(t, err) + assert.Equal(t, "123456789", id) +} + +func TestResolveDeploymentIDAbsentWhenNodeMissing(t *testing.T) { + w := newTestClient(t, http.StatusNotFound, `{"error_code":"RESOURCE_DOES_NOT_EXIST","message":"Path (/Workspace/state/`+DeploymentNodeName+`) doesn't exist."}`) + + // A bundle that never recorded a deployment, or whose deployment was + // destroyed (the service trashes the node), has no ID rather than an error. + id, err := ResolveDeploymentID(t.Context(), w, "/Workspace/state") + require.NoError(t, err) + assert.Empty(t, id) +} + +func TestResolveDeploymentIDPropagatesOtherErrors(t *testing.T) { + w := newTestClient(t, http.StatusForbidden, `{"error_code":"PERMISSION_DENIED","message":"nope"}`) + + // Anything other than a missing node is fatal: silently treating it as absent + // would create a second deployment for a bundle that already has one. + _, err := ResolveDeploymentID(t.Context(), w, "/Workspace/state") + require.Error(t, err) + assert.ErrorContains(t, err, "looking up deployment at /Workspace/state/"+DeploymentNodeName) +} diff --git a/libs/testserver/bundle.go b/libs/testserver/bundle.go index a1b0cba24a9..6dffbe34a75 100644 --- a/libs/testserver/bundle.go +++ b/libs/testserver/bundle.go @@ -3,15 +3,23 @@ package testserver import ( "bytes" "encoding/json" + "path" "slices" "strconv" "github.com/databricks/databricks-sdk-go/service/bundledeployments" + "github.com/databricks/databricks-sdk-go/service/workspace" ) // Handlers for the Deployment Metadata Service (DMS) API under /api/2.0/bundle. // State is kept in FakeWorkspace.dmsDeployments, keyed by deployment ID. +// dmsDeploymentNodeName is the name of the workspace node the service creates +// for every deployment. It must match DEPLOYMENT_NODE_NAME on the service side +// (DeploymentWhsClient); the literal is repeated here rather than shared with +// the CLI so a test would catch the CLI drifting from the service. +const dmsDeploymentNodeName = "resources.deployment.json" + // dmsDeployment holds a deployment record together with the versions and // resources recorded under it, so the read APIs (ListVersions/ListResources) // can serve back what deploys wrote. @@ -27,29 +35,49 @@ type dmsDeployment struct { // value as "DMS owns the state". Tracked separately because the SDK // Deployment struct does not yet carry the field (still stage:DEVELOPMENT). lastSuccessfulVersionID string + // nodePath is the workspace node whose object ID is this deployment's ID. + // Kept so DeleteDeployment can trash the node, the way the service does. + nodePath string } func (s *FakeWorkspace) CreateDeployment(req Request) Response { - // The client either supplies the deployment ID or, in the server-generated - // flow, leaves it empty for the server to mint one. - deploymentID := req.URL.Query().Get("deployment_id") - if deploymentID == "" { - deploymentID = nextUUID() - } - var dep bundledeployments.Deployment if err := json.Unmarshal(req.Body, &dep); err != nil { return Response{StatusCode: 400, Body: map[string]string{"message": err.Error()}} } + if dep.InitialParentPath == "" { + return Response{ + StatusCode: 400, + Body: map[string]string{"error_code": "INVALID_PARAMETER_VALUE", "message": "initial_parent_path is required"}, + } + } defer s.LockUnlock()() + // The service registers the deployment as a workspace node under + // initial_parent_path and uses that node's ID as the deployment ID, so a + // get-status on the node path is how clients look the deployment back up. + nodePath := path.Join(dep.InitialParentPath, dmsDeploymentNodeName) + if resp, ok := s.requireParentDirectory(nodePath); !ok { + return resp + } + objectID := nextID() + s.files[nodePath] = FileEntry{ + Info: workspace.ObjectInfo{ + ObjectType: "FILE", + Path: nodePath, + ObjectId: objectID, + }, + } + + deploymentID := strconv.FormatInt(objectID, 10) dep.Name = "deployments/" + deploymentID dep.Status = bundledeployments.DeploymentStatusDeploymentStatusActive s.dmsDeployments[deploymentID] = &dmsDeployment{ deployment: dep, versions: map[string]*bundledeployments.Version{}, resources: map[string]bundledeployments.Resource{}, + nodePath: nodePath, } return Response{Body: dep} } @@ -99,6 +127,11 @@ func deploymentBody(d *dmsDeployment) (map[string]any, error) { func (s *FakeWorkspace) DeleteDeployment(deploymentID string) Response { defer s.LockUnlock()() + // The service trashes the deployment's workspace node, so a later get-status + // on the node path reports the deployment as absent. + if d, ok := s.dmsDeployments[deploymentID]; ok { + delete(s.files, d.nodePath) + } delete(s.dmsDeployments, deploymentID) return Response{Body: map[string]any{}} } From 4009eb4ae58bd899bfe6d8210995823e73a42f7a Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 28 Jul 2026 00:30:33 +0000 Subject: [PATCH 012/125] bundle: address review of the DMS state PR - Limit recorded state to 64 KB, checked when the payload is built so an oversized resource fails itself rather than the drain at close. - Collapse the operation queue's pending/inflight pair into one `owned` set. The two maps encoded a single question ("is this key already claimed?") and had to be read together; `take` now only releases ownership. - Only log "Coalescing" when an operation was actually merged. The old code logged it for in-flight keys too, where nothing was coalesced. - Restore mergeWalIntoState's `hasEntries` naming and comment from main. The `persist` rename existed for the headerDirty case, which is gone. - Trim the comments added by this PR. Also note in fetchDeploymentResources that DMS has no field for dependency edges and they cannot be recovered from the recorded state (references are resolved to literals before serialization), so depends_on is carried over from the local state file. Co-authored-by: Isaac --- bundle/direct/dstate/dms.go | 41 ++++++++----------- bundle/direct/dstate/state.go | 74 +++++++++++++---------------------- bundle/direct/opqueue.go | 64 +++++++++++++++--------------- bundle/direct/opqueue_test.go | 15 ++++++- bundle/direct/oprecorder.go | 23 +++++++---- bundle/phases/dms.go | 9 ++--- libs/dms/recorder.go | 30 ++++++-------- libs/dms/resolve.go | 19 ++++----- 8 files changed, 129 insertions(+), 146 deletions(-) diff --git a/bundle/direct/dstate/dms.go b/bundle/direct/dstate/dms.go index 0941d87d35c..8709ceec57f 100644 --- a/bundle/direct/dstate/dms.go +++ b/bundle/direct/dstate/dms.go @@ -15,12 +15,9 @@ import ( ) // overlayDMSState replaces the file-derived resource state with the state -// recorded in the deployment metadata service (DMS), when DMS owns this -// deployment. Once DMS is authoritative its resource set is trusted even when -// empty (a successful deploy with no resources); the file's resources are only -// used when DMS has no successful version, or when the user opts out of -// recording deployment history. The caller holds db.mu, has already populated -// db.Data from the file, and has resolved src.DeploymentID. +// recorded in DMS, when DMS owns this deployment. An authoritative DMS is +// trusted even when its resource set is empty (a successful deploy of nothing). +// The caller holds db.mu and has already populated db.Data from the file. func (db *DeploymentState) overlayDMSState(ctx context.Context, src *DMSSource) error { authoritative, err := deploymentHasSuccessfulVersion(ctx, src.Config, src.DeploymentID) if err != nil { @@ -45,21 +42,13 @@ func (db *DeploymentState) overlayDMSState(ctx context.Context, src *DMSSource) return nil } -// deploymentHasSuccessfulVersion reports whether DMS holds a successfully -// completed version for the deployment. It is the signal that DMS owns the -// state: if the deployment was never recorded to DMS, or its initial DMS deploy -// did not complete successfully, DMS state is absent or partial and Open keeps -// the local file's resources instead. +// deploymentHasSuccessfulVersion reports whether DMS owns the state. The server +// advances last_successful_version_id only when a version completes (unlike +// last_version_id, which also advances on failure), so a non-empty value means +// DMS holds a complete resource set. Otherwise Open keeps the file's resources. // -// The deployment carries last_successful_version_id, which the server advances -// only when a version completes successfully (unlike last_version_id, which -// also advances on failure). So a non-empty value is exactly the "DMS owns the -// state" signal, readable in a single GetDeployment. -// -// TODO(DMS): this reads the deployment via a raw GET into a local struct -// because last_successful_version_id is still stage:DEVELOPMENT in the proto -// and therefore stripped from the generated SDK. Once the field is promoted to -// PRIVATE_PREVIEW and regenerated, replace the raw call with +// TODO(DMS): raw GET because last_successful_version_id is stage:DEVELOPMENT and +// stripped from the generated SDK. Once it ships, use // client.GetDeployment(...).LastSuccessfulVersionId and drop DMSSource.Config. func deploymentHasSuccessfulVersion(ctx context.Context, cfg *sdkconfig.Config, deploymentID string) (bool, error) { apiClient, err := client.New(cfg) @@ -88,10 +77,14 @@ func deploymentHasSuccessfulVersion(ctx context.Context, cfg *sdkconfig.Config, // fetchDeploymentResources lists every resource recorded for the deployment in // DMS and maps them into state entries keyed by the fully-qualified resource key. // -// DMS does not record dependency edges, so depends_on is carried over from the -// local state entry for the same key. It is derived from the local config on -// every deploy and is only consumed for delete ordering, so falling back to an -// empty list when the local state has no entry is safe. +// DMS has no field for dependency edges, and they cannot be recovered from the +// recorded state either: references are resolved to literals before it is +// serialized. So depends_on is carried over from the local state file. +// +// TODO(DMS): resources present in DMS but not in the local file therefore get no +// depends_on. Plan recomputes it from config for everything it still declares, +// so this only affects deletes of resources dropped from config, which are +// ordered arbitrarily among themselves. func fetchDeploymentResources(ctx context.Context, client bundledeployments.BundleDeploymentsInterface, deploymentID string, local map[string]ResourceEntry) (map[string]ResourceEntry, error) { it := client.ListResources(ctx, bundledeployments.ListResourcesRequest{ Parent: "deployments/" + deploymentID, diff --git a/bundle/direct/dstate/state.go b/bundle/direct/dstate/state.go index c27de3ca44d..0359feba89a 100644 --- a/bundle/direct/dstate/state.go +++ b/bundle/direct/dstate/state.go @@ -223,28 +223,22 @@ type ( // DMSSource tells Open to read resource state from the deployment metadata // service instead of the state file. A nil *DMSSource keeps Open file-only. type DMSSource struct { - // Client is the DMS client used to list the deployment's resources. Client bundledeployments.BundleDeploymentsInterface - // Config accompanies Client (both come from the same workspace client) and is - // used only for a temporary raw read of last_successful_version_id; see the - // TODO in deploymentHasSuccessfulVersion. + // Config is only for a temporary raw read of last_successful_version_id; see + // the TODO in deploymentHasSuccessfulVersion. Config *sdkconfig.Config - // DeploymentID identifies the deployment in DMS, resolved from the - // deployment's workspace node (see dms.ResolveDeploymentID). It is empty for a - // bundle that has not recorded a deployment yet. + // DeploymentID is resolved from the deployment's workspace node (see + // dms.ResolveDeploymentID), and empty before the first recorded deploy. DeploymentID string } // Open reads the deployment state from disk (and recovers the WAL when -// withRecovery is set). When dmsSource is non-nil, the deployment metadata -// service is the source of truth for resource state: if DMS holds a -// successfully completed version for this deployment, the resources read from -// the file are replaced with the ones recorded in DMS. The local identity -// (lineage and serial) always comes from the file, since that is what the write -// path increments and carries forward. A nil dmsSource keeps the behavior -// file-only. +// withRecovery is set). With a non-nil dmsSource, resources come from DMS +// instead of the file whenever DMS holds a successful version. Lineage and +// serial always come from the file, since that is what the write path +// increments. func (db *DeploymentState) Open(ctx context.Context, path string, withRecovery WithRecovery, withWrite WithWrite, dmsSource *DMSSource) error { db.mu.Lock() defer db.mu.Unlock() @@ -294,25 +288,15 @@ func (db *DeploymentState) Open(ctx context.Context, path string, withRecovery W } if dmsSource != nil { - // Only deployments that start out empty are recorded in DMS. Resources - // tracked in a state file that DMS does not know about are not in DMS and - // never will be: the first recorded deploy would create a deployment whose - // resource set covers only what that deploy touched, and DMS would then be - // authoritative for everything (see overlayDMSState). Resources this bundle - // already owns would look absent and be created a second time. + // Only bundles that start out empty can be recorded. Once DMS owns a + // deployment it is authoritative for the whole resource set (see + // overlayDMSState), so pre-existing resources it never saw would look absent + // and get created a second time. // - // A deployment DMS already owns (deploymentID is non-empty) is fine — that is - // a bundle that opted in while it was still empty. So is a state file with no - // resources, e.g. one left behind by a destroy. - // - // TODO(DMS): lift this restriction by upgrading an existing state in place. - // That means writing the state at featureStateVersion (3) with a feature flag - // recording that DMS owns it, plus a tombstone entry per resource so a CLI - // that predates DMS refuses the state instead of silently deploying against a - // resource set it cannot see. The feature-flag scaffolding for this already - // exists (see featureStateVersion and Header.Features); once it is written, - // this check goes away and record_deployment_history becomes usable on - // existing bundles. + // TODO(DMS): allow this by upgrading the state in place, writing it at + // featureStateVersion with a feature flag plus a tombstone per resource so an + // older CLI refuses the state instead of deploying against resources it + // cannot see. if dmsSource.DeploymentID == "" && len(db.Data.State) > 0 { return fmt.Errorf("cannot record deployment history for a bundle that already has deployed resources tracked in %s: only new deployments can be recorded. Remove experimental.record_deployment_history, or destroy the bundle and deploy it again", path) } @@ -364,7 +348,7 @@ func (db *DeploymentState) OpenWithData(path string, data Database) { func (db *DeploymentState) replayWAL(ctx context.Context) error { walPath := db.Path + walSuffix - persist, err := db.mergeWalIntoState(ctx) + hasEntries, err := db.mergeWalIntoState(ctx) if err != nil { if errors.Is(err, errStaleWAL) { log.Debugf(ctx, "Deleting stale WAL file %s", walPath) @@ -373,7 +357,7 @@ func (db *DeploymentState) replayWAL(ctx context.Context) error { } return fmt.Errorf("WAL recovery failed: %w", err) } - if persist { + if hasEntries { if err := db.unlockedSave(); err != nil { return err } @@ -384,9 +368,6 @@ func (db *DeploymentState) replayWAL(ctx context.Context) error { return nil } -// mergeWalIntoState replays the WAL into db.Data and reports whether the caller -// must persist the state file: either the WAL carried resource entries, or a -// header field changed in memory during this deployment. func (db *DeploymentState) mergeWalIntoState(ctx context.Context) (bool, error) { if db.walFile != nil { panic("internal error: walFile must be closed") @@ -462,20 +443,19 @@ func (db *DeploymentState) mergeWalIntoState(ctx context.Context) (bool, error) } } - persist := lineNumber > 1 + hasEntries := lineNumber > 1 - // Only advance the serial when the state file is actually written, because - // the caller (replayWAL) persists it only in that case. A header-only WAL - // that changed nothing is a deploy that started but committed nothing; - // advancing the serial for it leaves the in-memory serial ahead of the - // persisted one, so the next deploy writes its WAL header at serial+2 and - // recovery rejects it as "ahead of expected". - // See acceptance/bundle/deploy/wal/header-only-wal. - if persist { + // Only advance the serial when the WAL carried entries, because the caller + // (replayWAL) persists the new state file only in that case. A header-only + // WAL is a deploy that started but committed nothing; advancing the serial + // for it leaves the in-memory serial ahead of the persisted one, so the + // next deploy writes its WAL header at serial+2 and recovery rejects it as + // "ahead of expected". See acceptance/bundle/deploy/wal/header-only-wal. + if hasEntries { db.Data.Serial = newSerial } - return persist, nil + return hasEntries, nil } // Finalize replays the WAL (if open for write), captures the resulting state, and resets. diff --git a/bundle/direct/opqueue.go b/bundle/direct/opqueue.go index 72a2fa9c39e..68e8e2e9f96 100644 --- a/bundle/direct/opqueue.go +++ b/bundle/direct/opqueue.go @@ -22,19 +22,15 @@ const ( ) // operationQueue uploads recorded operations from background workers, so an apply -// worker does not wait for the CreateOperation round trip before moving on to the -// next resource. +// worker does not wait for the CreateOperation round trip. // -// It guarantees at most one upload in flight per resource key: within a key the -// worker that owns it uploads sequentially, so the last operation recorded for a -// resource is also the last one the service sees. +// At most one upload is in flight per resource key, so the last operation +// recorded for a resource is also the last one the service sees. // -// Uploads are not fire-and-forget: close drains the queue and returns the first -// failure, which fails the deploy. That matters because a successfully completed -// version makes DMS the source of truth for resource state (see -// dstate.overlayDMSState); silently dropping an operation would leave DMS with an -// incomplete resource set, and the next deploy would plan to create resources -// that already exist. +// Uploads are not fire-and-forget: close returns the first failure and fails the +// deploy. A dropped operation would leave DMS with an incomplete resource set, +// and since DMS then becomes the source of truth (see dstate.overlayDMSState), +// the next deploy would recreate resources that already exist. type operationQueue struct { uploader operationUploader @@ -51,10 +47,11 @@ type operationQueue struct { // picked up yet. pending map[string]recordedOperation - // inflight holds the resource keys a worker currently owns. A key that is - // in flight is not queued again: the owning worker re-checks pending after its - // upload and picks up anything recorded in the meantime. - inflight map[string]bool + // owned holds the resource keys that are already queued or being uploaded. + // Such a key is never queued a second time; recording writes to pending + // instead, which the owning worker re-checks after each upload. This is what + // keeps uploads for one resource sequential. + owned map[string]bool err error closed bool @@ -74,7 +71,7 @@ func newOperationQueue(ctx context.Context, uploader operationUploader) *operati uploader: uploader, queue: make(chan string, operationQueueSize), pending: make(map[string]recordedOperation), - inflight: make(map[string]bool), + owned: make(map[string]bool), } q.wg.Add(operationUploadWorkers) @@ -85,16 +82,14 @@ func newOperationQueue(ctx context.Context, uploader operationUploader) *operati return q } -// record serializes an operation and queues it for upload. It performs no API -// call, so upload failures surface from close rather than here; the error -// returned is only about turning the applied resource into a payload. +// record serializes an operation and queues it for upload. It makes no API call, +// so upload failures surface from close; an error here only means the applied +// resource could not be turned into a payload. // -// When an operation for the same resource is already waiting it is replaced -// instead of queued again: DMS keeps one state per resource key, so the later -// operation supersedes the earlier one and a single upload records both. The -// merged operation keeps the action of a queued create (see mergeAction), so -// collapsing a create and a later update still records a create. This is best -// effort - only operations that have not been picked up yet are collapsed. +// An operation for a resource that is still waiting replaces it rather than +// queueing again: DMS keeps one state per key, so one upload records both. The +// merge keeps a queued create's action (see mergeAction). Best effort — only +// operations no worker has picked up yet are collapsed. func (q *operationQueue) record(ctx context.Context, resourceKey string, action deployplan.ActionType, resourceID string, state any) error { if q == nil { return nil @@ -107,15 +102,21 @@ func (q *operationQueue) record(ctx context.Context, resourceKey string, action q.mu.Lock() queued, waiting := q.pending[resourceKey] - owned := waiting || q.inflight[resourceKey] if waiting { op.action = mergeAction(queued.action, op.action) } q.pending[resourceKey] = op + owned := q.owned[resourceKey] + q.owned[resourceKey] = true q.mu.Unlock() - if owned { + if waiting { log.Debugf(ctx, "Coalescing queued deployment operation for %s", resourceKey) + } + + // Someone already owns this key, so pending is enough: a worker will pick the + // operation up. Queueing again would upload the resource twice in parallel. + if owned { return nil } @@ -167,21 +168,20 @@ func (q *operationQueue) work(ctx context.Context) { } } -// take claims the operation waiting for resourceKey, marking the key in flight so -// record does not queue it a second time. It reports false, and releases the key, -// when nothing is waiting. +// take claims the operation waiting for resourceKey. It reports false and gives +// up ownership when nothing is waiting, which is what lets the next record +// queue the key again. func (q *operationQueue) take(resourceKey string) (recordedOperation, bool) { q.mu.Lock() defer q.mu.Unlock() op, ok := q.pending[resourceKey] if !ok { - delete(q.inflight, resourceKey) + delete(q.owned, resourceKey) return recordedOperation{}, false } delete(q.pending, resourceKey) - q.inflight[resourceKey] = true return op, true } diff --git a/bundle/direct/opqueue_test.go b/bundle/direct/opqueue_test.go index a8e05141fc7..cdd1083a06d 100644 --- a/bundle/direct/opqueue_test.go +++ b/bundle/direct/opqueue_test.go @@ -4,6 +4,7 @@ import ( "context" "errors" "strconv" + "strings" "sync" "testing" @@ -153,6 +154,18 @@ func TestOperationQueueRecordRejectsUnsupportedAction(t *testing.T) { assert.Empty(t, f.recorded()) } +func TestOperationQueueRecordRejectsOversizedState(t *testing.T) { + f := &fakeUploader{} + q := newOperationQueue(t.Context(), f) + + big := map[string]string{"name": strings.Repeat("x", maxOperationStateSize)} + err := q.record(t.Context(), "resources.jobs.foo", deployplan.Create, "id-1", big) + require.ErrorContains(t, err, "exceeds the 65536 byte limit") + + require.NoError(t, q.close()) + assert.Empty(t, f.recorded()) +} + func TestOperationQueueCloseIsIdempotent(t *testing.T) { f := &fakeUploader{err: errors.New("boom")} q := newOperationQueue(t.Context(), f) @@ -226,7 +239,7 @@ func TestOperationQueueUploadsOneResourceAtATime(t *testing.T) { // Every distinct key was recorded, and close drained all of them. assert.Len(t, u.last, distinctKeyMod) assert.Empty(t, q.pending) - assert.Empty(t, q.inflight) + assert.Empty(t, q.owned) } func TestNilOperationQueueIsNoOp(t *testing.T) { diff --git a/bundle/direct/oprecorder.go b/bundle/direct/oprecorder.go index 63d0c84a621..3b3d631f4a9 100644 --- a/bundle/direct/oprecorder.go +++ b/bundle/direct/oprecorder.go @@ -12,6 +12,11 @@ import ( "github.com/databricks/databricks-sdk-go/service/bundledeployments" ) +// maxOperationStateSize is the largest serialized state DMS accepts per +// operation. Uploading more is rejected server-side, so fail early with a +// message that names the resource. +const maxOperationStateSize = 64 * 1024 + // recordedOperation is an applied resource operation, serialized and waiting to be // uploaded to the deployment metadata service (DMS). // @@ -28,7 +33,8 @@ type recordedOperation struct { } // newRecordedOperation serializes an applied operation for upload. state is the -// local config after the operation and must be nil for delete operations. +// local config after the operation and must be nil for delete operations. It +// errors when the serialized state exceeds maxOperationStateSize. func newRecordedOperation(action deployplan.ActionType, resourceID string, state any) (recordedOperation, error) { actionType, err := deployActionToSDK(action) if err != nil { @@ -37,19 +43,20 @@ func newRecordedOperation(action deployplan.ActionType, resourceID string, state op := recordedOperation{action: actionType, resourceID: resourceID} - // The DMS Operation.State field carries the serialized config so the backend - // can serve it as resource state. It is intentionally left unset for delete, - // where the resource no longer exists. + // Operation.State carries the serialized config, which DMS serves back as + // resource state. Unset for delete: the resource is gone. // - // Redact sensitive fields, matching what dstate.SaveState writes to the local - // state file: DMS state is read back as resource state, so recording secrets - // in plaintext would both leak them to the service and reintroduce them into - // a local state file via the read path. + // Redact secrets, like dstate.SaveState does for the local state file: + // otherwise we leak them to the service and the read path writes them back + // into a local state file in plaintext. if state != nil { raw, err := structwalk.RedactSensitiveFields(state, dyn.SensitiveValueRedacted) if err != nil { return recordedOperation{}, fmt.Errorf("serializing state: %w", err) } + if len(raw) > maxOperationStateSize { + return recordedOperation{}, fmt.Errorf("serialized state is %d bytes, which exceeds the %d byte limit for recording deployment history", len(raw), maxOperationStateSize) + } op.state = raw } diff --git a/bundle/phases/dms.go b/bundle/phases/dms.go index 02254237595..3d2f4f54009 100644 --- a/bundle/phases/dms.go +++ b/bundle/phases/dms.go @@ -16,11 +16,10 @@ import ( // AND the engine is direct: DMS resource state is tracked per direct-engine // deployment. Returning nil for terraform leaves those deployments untouched. // -// The deployment ID is resolved from the workspace rather than from local state -// (see dms.ResolveDeploymentID). The lookup happens here, after the deployment -// lock has been acquired, so it observes any deployment a concurrent deploy -// created. It is empty on a bundle's first recorded deploy, in which case the -// recorder creates the deployment and the server assigns the ID. +// The deployment ID is resolved from the workspace, not local state (see +// dms.ResolveDeploymentID). The lookup happens here, after the deployment lock is +// held, so it sees any deployment a concurrent deploy created. It is empty on the +// first recorded deploy, where the recorder creates the deployment instead. func newDeploymentRecorder(ctx context.Context, b *bundle.Bundle, eng engine.EngineType, versionType dms.VersionType) (*dms.Recorder, error) { if b.Config.Experimental == nil || !b.Config.Experimental.RecordDeploymentHistory { return nil, nil diff --git a/libs/dms/recorder.go b/libs/dms/recorder.go index 6fa9a1a24be..83f180ba3be 100644 --- a/libs/dms/recorder.go +++ b/libs/dms/recorder.go @@ -29,12 +29,10 @@ const ( // Recorder records a single deploy/destroy as a version with DMS. // -// The deployment ID is assigned by the server on the first deploy: NewRecorder -// is given the ID resolved from the workspace (empty on a bundle's first-ever -// recorded deploy, see ResolveDeploymentID), and CreateVersion creates the -// deployment record when that ID is empty. Later deploys resolve the same ID -// from the deployment's workspace node and reuse the record; a destroy deletes -// the record and its node, so the next deploy starts over from empty. +// The server assigns the deployment ID on the first deploy, i.e. when the ID +// resolved from the workspace is empty (see ResolveDeploymentID). Later deploys +// resolve the same ID and reuse the record; a destroy deletes the record and its +// node, so the next deploy starts over from empty. type Recorder struct { svc bundledeployments.BundleDeploymentsInterface deploymentID string @@ -150,10 +148,10 @@ func (r *Recorder) CompleteVersion(ctx context.Context, success bool) error { func (r *Recorder) createDeploymentVersion(ctx context.Context) (versionID string, err error) { if r.deploymentID != "" { // Existing deployment: read it to compute the next version number. A 404 is - // not recovered from by creating a second deployment: the ID was just - // resolved from the deployment's workspace node, which the service trashes - // when it deletes the record, so a missing record here means the two are out - // of sync and creating another one would collide on the same node path. + // not recovered from by creating a second deployment. The service trashes the + // workspace node when it deletes the record, so a node that resolved but has + // no record means the two are out of sync, and creating another deployment + // would collide on the same node path. dep, getErr := r.svc.GetDeployment(ctx, bundledeployments.GetDeploymentRequest{ Name: "deployments/" + r.deploymentID, }) @@ -166,14 +164,12 @@ func (r *Recorder) createDeploymentVersion(ctx context.Context) (versionID strin } versionID = strconv.FormatInt(lastVersion+1, 10) } else { - // First deploy: create the deployment so the server assigns an ID, then - // start at version 1. + // First deploy: create the deployment so the server assigns an ID. // - // initial_parent_path is required: the service creates the deployment's - // BUNDLE_DEPLOYMENT node under it, and that node's ID becomes the - // deployment ID that ResolveDeploymentID reads back on later deploys. The - // folder must already exist, which it does by this point - the deployment - // lock lives in the same directory. + // initial_parent_path is required. The service creates the deployment node + // under it, and that node's ID is the deployment ID ResolveDeploymentID reads + // back later. The folder already exists by now: the deployment lock lives in + // the same directory. dep, createErr := r.svc.CreateDeployment(ctx, bundledeployments.CreateDeploymentRequest{ Deployment: bundledeployments.Deployment{ InitialParentPath: r.statePath, diff --git a/libs/dms/resolve.go b/libs/dms/resolve.go index 0d26448e558..9bbf5a84b56 100644 --- a/libs/dms/resolve.go +++ b/libs/dms/resolve.go @@ -11,22 +11,17 @@ import ( "github.com/databricks/databricks-sdk-go/apierr" ) -// DeploymentNodeName is the workspace node DMS creates for a deployment. The -// name is fixed for every deployment: the node *is* the bundle's state file. -// It must match DeploymentWhsClient.DEPLOYMENT_NODE_NAME on the service side. +// DeploymentNodeName is the workspace node DMS creates per deployment. Must +// match DeploymentWhsClient.DEPLOYMENT_NODE_NAME on the service side. const DeploymentNodeName = "resources.deployment.json" // ResolveDeploymentID returns the DMS deployment ID for the bundle whose state -// lives under statePath, or an empty string when the bundle has no deployment -// recorded yet. +// lives under statePath, or empty if it has never recorded a deployment. // -// The ID is not stored anywhere by the CLI. DMS registers each deployment as a -// BUNDLE_DEPLOYMENT node at statePath/resources.deployment.json, and the -// workspace-assigned node ID *is* the deployment ID (see DeploymentHandler: -// deploymentId = Long.toString(createdNode.getId())). So a get-status on that -// path is the lookup, which keeps the workspace the single source of truth: a -// deployment that was destroyed or deleted out of band reports absent here -// rather than leaving a dangling ID behind in the local state file. +// The CLI stores the ID nowhere: DMS registers the deployment as a workspace +// node and that node's ID *is* the deployment ID, so a get-status is the lookup. +// This keeps the workspace the single source of truth — a destroyed deployment +// reports absent instead of leaving a dangling ID in the local state file. func ResolveDeploymentID(ctx context.Context, w *databricks.WorkspaceClient, statePath string) (string, error) { nodePath := path.Join(statePath, DeploymentNodeName) From 70741aeceb38fd9155b952e9f2d8785d08555338 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 28 Jul 2026 10:38:59 +0000 Subject: [PATCH 013/125] bundle: record depends_on in the state uploaded to DMS The read path carried depends_on over from the local state file, which is empty exactly when it matters: a fresh checkout reconstructs state from DMS and got no dependency edges. Deletes are the one case that cannot recompute them, because the resource is gone from config, so two dropped resources with a real dependency could be deleted in the wrong order. DMS has no field for dependency edges, and they cannot be recovered from the recorded config either: references are resolved to literals before it is serialized. So Operation.State now carries an envelope, dstate.RecordedState, holding the config plus depends_on. Nesting depends_on inside the config would have collided with resource fields of the same name (jobs.Task.depends_on). The envelope mirrors the local ResourceEntry, so both sides of the round trip have the same shape. acceptance/bundle/dms/depends-on covers it: a job referencing another records its edge, and after wiping the local state a destroy still deletes the referencing job first. Co-authored-by: Isaac --- .../bundle/dms/depends-on/databricks.yml | 13 ++++++ .../bundle/dms/depends-on/out.test.toml | 3 ++ acceptance/bundle/dms/depends-on/output.txt | 26 ++++++++++++ acceptance/bundle/dms/depends-on/script | 8 ++++ .../bundle/dms/existing-state/output.txt | 2 +- acceptance/bundle/dms/record/output.txt | 22 +++++----- .../dms/redeploy-after-destroy/output.txt | 22 +++++----- bundle/direct/bundle_apply.go | 7 ++-- bundle/direct/dstate/dms.go | 40 ++++++++++++------- bundle/direct/dstate/dms_test.go | 38 ++++++++---------- bundle/direct/opqueue.go | 4 +- bundle/direct/opqueue_test.go | 20 +++++----- bundle/direct/oprecorder.go | 11 +++-- bundle/direct/oprecorder_test.go | 21 ++++++++-- 14 files changed, 157 insertions(+), 80 deletions(-) create mode 100644 acceptance/bundle/dms/depends-on/databricks.yml create mode 100644 acceptance/bundle/dms/depends-on/out.test.toml create mode 100644 acceptance/bundle/dms/depends-on/output.txt create mode 100644 acceptance/bundle/dms/depends-on/script diff --git a/acceptance/bundle/dms/depends-on/databricks.yml b/acceptance/bundle/dms/depends-on/databricks.yml new file mode 100644 index 00000000000..f97e3a38809 --- /dev/null +++ b/acceptance/bundle/dms/depends-on/databricks.yml @@ -0,0 +1,13 @@ +bundle: + name: dms-depends-on + +experimental: + record_deployment_history: true + +resources: + jobs: + parent: + name: parent + child: + name: child + description: depends on ${resources.jobs.parent.id} diff --git a/acceptance/bundle/dms/depends-on/out.test.toml b/acceptance/bundle/dms/depends-on/out.test.toml new file mode 100644 index 00000000000..e90b6d5d1ba --- /dev/null +++ b/acceptance/bundle/dms/depends-on/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/dms/depends-on/output.txt b/acceptance/bundle/dms/depends-on/output.txt new file mode 100644 index 00000000000..2bb8d06b9dc --- /dev/null +++ b/acceptance/bundle/dms/depends-on/output.txt @@ -0,0 +1,26 @@ + +=== Deploy a job that references another: depends_on is recorded alongside the config, since the reference is resolved to a literal in the config itself +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-depends-on/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> print_requests.py //versions/1/operations --sort --oneline +{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations", "q": {"resource_key": "jobs.child"}, "body": {"action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_id": "[NUMID]", "resource_key": "jobs.child", "state": {"state": {"deployment": {"kind": "BUNDLE", "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-depends-on/default/state/metadata.json"}, "description": "depends on [NUMID]", "edit_mode": "UI_LOCKED", "format": "MULTI_TASK", "max_concurrent_runs": 1, "name": "child", "queue": {"enabled": true}}, "depends_on": [{"node": "resources.jobs.parent", "label": "${resources.jobs.parent.id}"}]}, "status": "OPERATION_STATUS_SUCCEEDED"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations", "q": {"resource_key": "jobs.parent"}, "body": {"action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_id": "[NUMID]", "resource_key": "jobs.parent", "state": {"state": {"deployment": {"kind": "BUNDLE", "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-depends-on/default/state/metadata.json"}, "edit_mode": "UI_LOCKED", "format": "MULTI_TASK", "max_concurrent_runs": 1, "name": "parent", "queue": {"enabled": true}}}, "status": "OPERATION_STATUS_SUCCEEDED"}} + +=== Wipe the local state, then destroy: depends_on comes back from DMS, so the child is still deleted before the parent it references +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.jobs.child + delete resources.jobs.parent + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/dms-depends-on/default + +Deleting files... +Destroy complete! + +>>> print_requests.py //jobs --oneline +{"method": "POST", "path": "/api/2.2/jobs/delete", "body": {"job_id": [NUMID]}} +{"method": "POST", "path": "/api/2.2/jobs/delete", "body": {"job_id": [NUMID]}} diff --git a/acceptance/bundle/dms/depends-on/script b/acceptance/bundle/dms/depends-on/script new file mode 100644 index 00000000000..905d022619c --- /dev/null +++ b/acceptance/bundle/dms/depends-on/script @@ -0,0 +1,8 @@ +title "Deploy a job that references another: depends_on is recorded alongside the config, since the reference is resolved to a literal in the config itself" +trace $CLI bundle deploy +trace print_requests.py //versions/1/operations --sort --oneline + +title "Wipe the local state, then destroy: depends_on comes back from DMS, so the child is still deleted before the parent it references" +rm -rf .databricks +trace $CLI bundle destroy --auto-approve +trace print_requests.py //jobs --oneline diff --git a/acceptance/bundle/dms/existing-state/output.txt b/acceptance/bundle/dms/existing-state/output.txt index 6aeff83c04b..116584d10c1 100644 --- a/acceptance/bundle/dms/existing-state/output.txt +++ b/acceptance/bundle/dms/existing-state/output.txt @@ -42,4 +42,4 @@ Deployment complete! {"method": "POST", "path": "/api/2.0/bundle/deployments", "body": {"initial_parent_path": "/Workspace/Users/[USERNAME]/.bundle/dms-existing-state/default/state", "target_name": "default"}} {"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": {"version_id": "1"}, "body": {"cli_version": "[CLI_VERSION]", "target_name": "default", "version_type": "VERSION_TYPE_DEPLOY"}} {"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/complete", "body": {"completion_reason": "VERSION_COMPLETE_SUCCESS"}} -{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations", "q": {"resource_key": "jobs.one"}, "body": {"action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_id": "[NUMID]", "resource_key": "jobs.one", "state": {"deployment": {"kind": "BUNDLE", "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-existing-state/default/state/metadata.json"}, "edit_mode": "UI_LOCKED", "format": "MULTI_TASK", "max_concurrent_runs": 1, "name": "one", "queue": {"enabled": true}}, "status": "OPERATION_STATUS_SUCCEEDED"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations", "q": {"resource_key": "jobs.one"}, "body": {"action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_id": "[NUMID]", "resource_key": "jobs.one", "state": {"state": {"deployment": {"kind": "BUNDLE", "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-existing-state/default/state/metadata.json"}, "edit_mode": "UI_LOCKED", "format": "MULTI_TASK", "max_concurrent_runs": 1, "name": "one", "queue": {"enabled": true}}}, "status": "OPERATION_STATUS_SUCCEEDED"}} diff --git a/acceptance/bundle/dms/record/output.txt b/acceptance/bundle/dms/record/output.txt index ed10f06e699..e792d5d8d99 100644 --- a/acceptance/bundle/dms/record/output.txt +++ b/acceptance/bundle/dms/record/output.txt @@ -45,16 +45,18 @@ Deployment complete! "resource_id": "[NUMID]", "resource_key": "jobs.foo", "state": { - "deployment": { - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-record/default/state/metadata.json" - }, - "edit_mode": "UI_LOCKED", - "format": "MULTI_TASK", - "max_concurrent_runs": 1, - "name": "foo", - "queue": { - "enabled": true + "state": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-record/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "foo", + "queue": { + "enabled": true + } } }, "status": "OPERATION_STATUS_SUCCEEDED" diff --git a/acceptance/bundle/dms/redeploy-after-destroy/output.txt b/acceptance/bundle/dms/redeploy-after-destroy/output.txt index ed7b6ede989..ff8694b5424 100644 --- a/acceptance/bundle/dms/redeploy-after-destroy/output.txt +++ b/acceptance/bundle/dms/redeploy-after-destroy/output.txt @@ -70,16 +70,18 @@ Deployment complete! "resource_id": "[NUMID]", "resource_key": "jobs.foo", "state": { - "deployment": { - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-redeploy-after-destroy/default/state/metadata.json" - }, - "edit_mode": "UI_LOCKED", - "format": "MULTI_TASK", - "max_concurrent_runs": 1, - "name": "foo", - "queue": { - "enabled": true + "state": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-redeploy-after-destroy/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "foo", + "queue": { + "enabled": true + } } }, "status": "OPERATION_STATUS_SUCCEEDED" diff --git a/bundle/direct/bundle_apply.go b/bundle/direct/bundle_apply.go index b26861128b7..f29aa18a186 100644 --- a/bundle/direct/bundle_apply.go +++ b/bundle/direct/bundle_apply.go @@ -94,7 +94,7 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa return false } // Record the delete with DMS. State is nil: the resource is gone. - if err := opQueue.record(ctx, resourceKey, action, "", nil); err != nil { + if err := opQueue.record(ctx, resourceKey, action, "", nil, nil); err != nil { logdiag.LogError(ctx, fmt.Errorf("%s: %w", errorPrefix, err)) return false } @@ -129,8 +129,9 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa // Record the operation with DMS. The resource ID and applied config // (sv.Value) come from the write just performed; GetResourceID reads - // the ID assigned by Deploy. - if err := opQueue.record(ctx, resourceKey, action, b.StateDB.GetResourceID(resourceKey), sv.Value); err != nil { + // the ID assigned by Deploy. depends_on is recorded alongside the config + // because it cannot be recomputed from it (see dstate.RecordedState). + if err := opQueue.record(ctx, resourceKey, action, b.StateDB.GetResourceID(resourceKey), sv.Value, d.DependsOn); err != nil { logdiag.LogError(ctx, fmt.Errorf("%s: %w", errorPrefix, err)) return false } diff --git a/bundle/direct/dstate/dms.go b/bundle/direct/dstate/dms.go index 8709ceec57f..e6ae63f6095 100644 --- a/bundle/direct/dstate/dms.go +++ b/bundle/direct/dstate/dms.go @@ -7,6 +7,7 @@ import ( "fmt" "net/http" + "github.com/databricks/cli/bundle/deployplan" "github.com/databricks/cli/libs/auth" "github.com/databricks/databricks-sdk-go/apierr" "github.com/databricks/databricks-sdk-go/client" @@ -14,6 +15,22 @@ import ( "github.com/databricks/databricks-sdk-go/service/bundledeployments" ) +// RecordedState is what the CLI serializes into the DMS Operation.State field. +// +// It is an envelope rather than the bare resource config, because depends_on has +// to survive the round trip: DMS has no field for dependency edges, and they +// cannot be recomputed from the config once it is recorded (references are +// resolved to literals before serialization). Nesting depends_on inside the +// config instead would collide with resource fields of the same name, e.g. +// jobs.Task.depends_on. +// +// The shape deliberately matches the local ResourceEntry so both sides of the +// state round trip look the same. +type RecordedState struct { + State json.RawMessage `json:"state"` + DependsOn []deployplan.DependsOnEntry `json:"depends_on,omitempty"` +} + // overlayDMSState replaces the file-derived resource state with the state // recorded in DMS, when DMS owns this deployment. An authoritative DMS is // trusted even when its resource set is empty (a successful deploy of nothing). @@ -29,7 +46,7 @@ func (db *DeploymentState) overlayDMSState(ctx context.Context, src *DMSSource) return nil } - resources, err := fetchDeploymentResources(ctx, src.Client, src.DeploymentID, db.Data.State) + resources, err := fetchDeploymentResources(ctx, src.Client, src.DeploymentID) if err != nil { return err } @@ -76,16 +93,7 @@ func deploymentHasSuccessfulVersion(ctx context.Context, cfg *sdkconfig.Config, // fetchDeploymentResources lists every resource recorded for the deployment in // DMS and maps them into state entries keyed by the fully-qualified resource key. -// -// DMS has no field for dependency edges, and they cannot be recovered from the -// recorded state either: references are resolved to literals before it is -// serialized. So depends_on is carried over from the local state file. -// -// TODO(DMS): resources present in DMS but not in the local file therefore get no -// depends_on. Plan recomputes it from config for everything it still declares, -// so this only affects deletes of resources dropped from config, which are -// ordered arbitrarily among themselves. -func fetchDeploymentResources(ctx context.Context, client bundledeployments.BundleDeploymentsInterface, deploymentID string, local map[string]ResourceEntry) (map[string]ResourceEntry, error) { +func fetchDeploymentResources(ctx context.Context, client bundledeployments.BundleDeploymentsInterface, deploymentID string) (map[string]ResourceEntry, error) { it := client.ListResources(ctx, bundledeployments.ListResourcesRequest{ Parent: "deployments/" + deploymentID, }) @@ -102,15 +110,17 @@ func fetchDeploymentResources(ctx context.Context, client bundledeployments.Bund // ("resources.jobs.foo"), so prepend it here. key := "resources." + res.ResourceKey - var state json.RawMessage + var recorded RecordedState if res.State != nil { - state = *res.State + if err := json.Unmarshal(*res.State, &recorded); err != nil { + return nil, fmt.Errorf("interpreting state recorded for %s: %w", key, err) + } } out[key] = ResourceEntry{ ID: res.ResourceId, - State: state, - DependsOn: local[key].DependsOn, + State: recorded.State, + DependsOn: recorded.DependsOn, } } return out, nil diff --git a/bundle/direct/dstate/dms_test.go b/bundle/direct/dstate/dms_test.go index df1084b9de7..35fe7acbb0c 100644 --- a/bundle/direct/dstate/dms_test.go +++ b/bundle/direct/dstate/dms_test.go @@ -35,40 +35,34 @@ func (f *fakeResourceLister) ListResources(ctx context.Context, req bundledeploy ) } -func TestFetchDeploymentResourcesPreservesLocalDependsOn(t *testing.T) { - state := json.RawMessage(`{"name":"foo"}`) +func TestFetchDeploymentResourcesUnwrapsEnvelope(t *testing.T) { + recorded := json.RawMessage(`{"state":{"name":"foo"},"depends_on":[{"node":"resources.pipelines.bar","label":"${resources.pipelines.bar.id}"}]}`) f := &fakeResourceLister{resources: []bundledeployments.Resource{ - {ResourceKey: "jobs.foo", ResourceId: "123", State: &state}, + {ResourceKey: "jobs.foo", ResourceId: "123", State: &recorded}, {ResourceKey: "pipelines.bar", ResourceId: "456"}, }} - dependsOn := []deployplan.DependsOnEntry{{Node: "resources.pipelines.bar", Label: "pipeline_id"}} - local := map[string]ResourceEntry{ - "resources.jobs.foo": {ID: "stale", DependsOn: dependsOn}, - } - - got, err := fetchDeploymentResources(t.Context(), f, "dep-1", local) + got, err := fetchDeploymentResources(t.Context(), f, "dep-1") require.NoError(t, err) - // DMS owns the ID and state, but it does not record dependency edges, so - // depends_on must survive from the local entry. Losing it breaks delete - // ordering and --select expansion. + // depends_on comes back from the envelope, so a bundle whose local state was + // wiped still has the edges needed for delete ordering. assert.Equal(t, map[string]ResourceEntry{ - "resources.jobs.foo": {ID: "123", State: state, DependsOn: dependsOn}, + "resources.jobs.foo": { + ID: "123", + State: json.RawMessage(`{"name":"foo"}`), + DependsOn: []deployplan.DependsOnEntry{{Node: "resources.pipelines.bar", Label: "${resources.pipelines.bar.id}"}}, + }, "resources.pipelines.bar": {ID: "456"}, }, got) } -func TestFetchDeploymentResourcesWithNoLocalState(t *testing.T) { +func TestFetchDeploymentResourcesRejectsMalformedState(t *testing.T) { + recorded := json.RawMessage(`not json`) f := &fakeResourceLister{resources: []bundledeployments.Resource{ - {ResourceKey: "jobs.foo", ResourceId: "123"}, + {ResourceKey: "jobs.foo", ResourceId: "123", State: &recorded}, }} - // A bundle whose local state was wiped has no entry to carry depends_on from; - // the resource is still recovered from DMS. - got, err := fetchDeploymentResources(t.Context(), f, "dep-1", nil) - require.NoError(t, err) - assert.Equal(t, map[string]ResourceEntry{ - "resources.jobs.foo": {ID: "123"}, - }, got) + _, err := fetchDeploymentResources(t.Context(), f, "dep-1") + assert.ErrorContains(t, err, "interpreting state recorded for resources.jobs.foo") } diff --git a/bundle/direct/opqueue.go b/bundle/direct/opqueue.go index 68e8e2e9f96..1f38c2b7dd6 100644 --- a/bundle/direct/opqueue.go +++ b/bundle/direct/opqueue.go @@ -90,12 +90,12 @@ func newOperationQueue(ctx context.Context, uploader operationUploader) *operati // queueing again: DMS keeps one state per key, so one upload records both. The // merge keeps a queued create's action (see mergeAction). Best effort — only // operations no worker has picked up yet are collapsed. -func (q *operationQueue) record(ctx context.Context, resourceKey string, action deployplan.ActionType, resourceID string, state any) error { +func (q *operationQueue) record(ctx context.Context, resourceKey string, action deployplan.ActionType, resourceID string, state any, dependsOn []deployplan.DependsOnEntry) error { if q == nil { return nil } - op, err := newRecordedOperation(action, resourceID, state) + op, err := newRecordedOperation(action, resourceID, state, dependsOn) if err != nil { return err } diff --git a/bundle/direct/opqueue_test.go b/bundle/direct/opqueue_test.go index cdd1083a06d..6bf0da8107b 100644 --- a/bundle/direct/opqueue_test.go +++ b/bundle/direct/opqueue_test.go @@ -59,7 +59,7 @@ func (f *fakeUploader) actionFor(resourceKey string) bundledeployments.Operation func recordState(t *testing.T, q *operationQueue, resourceKey, name string) { t.Helper() - require.NoError(t, q.record(t.Context(), resourceKey, deployplan.Update, "id-1", map[string]string{"name": name})) + require.NoError(t, q.record(t.Context(), resourceKey, deployplan.Update, "id-1", map[string]string{"name": name}, nil)) } func TestOperationQueueUploadsEachOperation(t *testing.T) { @@ -94,8 +94,8 @@ func TestOperationQueueCoalescesQueuedOperationsForSameResource(t *testing.T) { // Two uploads, not three: v2 was superseded by v3 while both were queued, and // the last recorded state is the one the service ends up with. assert.Equal(t, []string{ - `resources.jobs.foo={"name":"v1"}`, - `resources.jobs.foo={"name":"v3"}`, + `resources.jobs.foo={"state":{"name":"v1"}}`, + `resources.jobs.foo={"state":{"name":"v3"}}`, }, f.recorded()) } @@ -114,15 +114,15 @@ func TestOperationQueueCoalescingKeepsCreateAction(t *testing.T) { assert.Equal(t, "resources.jobs.hold"+strconv.Itoa(i), <-f.started) } - require.NoError(t, q.record(t.Context(), "resources.jobs.foo", deployplan.Create, "id-1", map[string]string{"name": "created"})) - require.NoError(t, q.record(t.Context(), "resources.jobs.foo", deployplan.Update, "id-1", map[string]string{"name": "updated"})) + require.NoError(t, q.record(t.Context(), "resources.jobs.foo", deployplan.Create, "id-1", map[string]string{"name": "created"}, nil)) + require.NoError(t, q.record(t.Context(), "resources.jobs.foo", deployplan.Update, "id-1", map[string]string{"name": "updated"}, nil)) close(f.block) require.NoError(t, q.close()) // The state is the later one, but the action stays CREATE: recording an update // would tell DMS the resource already existed before this deploy. - assert.Contains(t, f.recorded(), `resources.jobs.foo={"name":"updated"}`) + assert.Contains(t, f.recorded(), `resources.jobs.foo={"state":{"name":"updated"}}`) assert.Equal(t, bundledeployments.OperationActionTypeOperationActionTypeCreate, f.actionFor("resources.jobs.foo")) @@ -147,7 +147,7 @@ func TestOperationQueueRecordRejectsUnsupportedAction(t *testing.T) { // Serialization failures surface at record time, on the resource that caused // them, rather than from the drain at the end of apply. - err := q.record(t.Context(), "resources.jobs.foo", deployplan.Skip, "id-1", nil) + err := q.record(t.Context(), "resources.jobs.foo", deployplan.Skip, "id-1", nil, nil) require.Error(t, err) require.NoError(t, q.close()) @@ -159,7 +159,7 @@ func TestOperationQueueRecordRejectsOversizedState(t *testing.T) { q := newOperationQueue(t.Context(), f) big := map[string]string{"name": strings.Repeat("x", maxOperationStateSize)} - err := q.record(t.Context(), "resources.jobs.foo", deployplan.Create, "id-1", big) + err := q.record(t.Context(), "resources.jobs.foo", deployplan.Create, "id-1", big, nil) require.ErrorContains(t, err, "exceeds the 65536 byte limit") require.NoError(t, q.close()) @@ -224,7 +224,7 @@ func TestOperationQueueUploadsOneResourceAtATime(t *testing.T) { wg.Go(func() { for i := range perWorker { key := "resources.jobs.job" + strconv.Itoa((w*perWorker+i)%distinctKeyMod) - errs <- q.record(ctx, key, deployplan.Update, "id-1", map[string]string{"name": strconv.Itoa(w)}) + errs <- q.record(ctx, key, deployplan.Update, "id-1", map[string]string{"name": strconv.Itoa(w)}, nil) } }) } @@ -247,6 +247,6 @@ func TestNilOperationQueueIsNoOp(t *testing.T) { // no-op, so Apply does not have to branch. q := newOperationQueue(t.Context(), nil) require.Nil(t, q) - require.NoError(t, q.record(t.Context(), "resources.jobs.foo", deployplan.Create, "id-1", nil)) + require.NoError(t, q.record(t.Context(), "resources.jobs.foo", deployplan.Create, "id-1", nil, nil)) require.NoError(t, q.close()) } diff --git a/bundle/direct/oprecorder.go b/bundle/direct/oprecorder.go index 3b3d631f4a9..91bcb65a4f6 100644 --- a/bundle/direct/oprecorder.go +++ b/bundle/direct/oprecorder.go @@ -7,6 +7,7 @@ import ( "strings" "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/cli/bundle/direct/dstate" "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/structs/structwalk" "github.com/databricks/databricks-sdk-go/service/bundledeployments" @@ -35,7 +36,7 @@ type recordedOperation struct { // newRecordedOperation serializes an applied operation for upload. state is the // local config after the operation and must be nil for delete operations. It // errors when the serialized state exceeds maxOperationStateSize. -func newRecordedOperation(action deployplan.ActionType, resourceID string, state any) (recordedOperation, error) { +func newRecordedOperation(action deployplan.ActionType, resourceID string, state any, dependsOn []deployplan.DependsOnEntry) (recordedOperation, error) { actionType, err := deployActionToSDK(action) if err != nil { return recordedOperation{}, err @@ -43,14 +44,18 @@ func newRecordedOperation(action deployplan.ActionType, resourceID string, state op := recordedOperation{action: actionType, resourceID: resourceID} - // Operation.State carries the serialized config, which DMS serves back as + // Operation.State carries the serialized state, which DMS serves back as // resource state. Unset for delete: the resource is gone. // // Redact secrets, like dstate.SaveState does for the local state file: // otherwise we leak them to the service and the read path writes them back // into a local state file in plaintext. if state != nil { - raw, err := structwalk.RedactSensitiveFields(state, dyn.SensitiveValueRedacted) + config, err := structwalk.RedactSensitiveFields(state, dyn.SensitiveValueRedacted) + if err != nil { + return recordedOperation{}, fmt.Errorf("serializing state: %w", err) + } + raw, err := json.Marshal(dstate.RecordedState{State: config, DependsOn: dependsOn}) if err != nil { return recordedOperation{}, fmt.Errorf("serializing state: %w", err) } diff --git a/bundle/direct/oprecorder_test.go b/bundle/direct/oprecorder_test.go index 70afb788cd3..556f7f59f90 100644 --- a/bundle/direct/oprecorder_test.go +++ b/bundle/direct/oprecorder_test.go @@ -30,7 +30,7 @@ func (f *fakeOpClient) CreateOperation(ctx context.Context, req bundledeployment // an operationQueue worker does. func uploadOne(t *testing.T, u operationUploader, resourceKey string, action deployplan.ActionType, resourceID string, state any) { t.Helper() - op, err := newRecordedOperation(action, resourceID, state) + op, err := newRecordedOperation(action, resourceID, state, nil) require.NoError(t, err) require.NoError(t, u.upload(t.Context(), resourceKey, op)) } @@ -71,18 +71,31 @@ func TestNewRecordedOperationRedactsSensitiveFields(t *testing.T) { Token string `json:"token" bundle:"sensitive"` }{Name: "foo", Token: "super-secret"} - op, err := newRecordedOperation(deployplan.Create, "job-123", state) + op, err := newRecordedOperation(deployplan.Create, "job-123", state, nil) require.NoError(t, err) // Sensitive fields are redacted before leaving the CLI, matching what // dstate.SaveState writes to the local state file. assert.JSONEq(t, - `{"name":"foo","token":"`+dyn.SensitiveValueRedacted+`"}`, + `{"state":{"name":"foo","token":"`+dyn.SensitiveValueRedacted+`"}}`, + string(op.state)) +} + +func TestNewRecordedOperationRecordsDependsOn(t *testing.T) { + // depends_on rides in an envelope alongside the config: it cannot be + // recomputed from the config, whose references are already resolved. + dependsOn := []deployplan.DependsOnEntry{{Node: "resources.jobs.bar", Label: "${resources.jobs.bar.id}"}} + + op, err := newRecordedOperation(deployplan.Create, "job-123", map[string]string{"name": "foo"}, dependsOn) + require.NoError(t, err) + + assert.JSONEq(t, + `{"state":{"name":"foo"},"depends_on":[{"node":"resources.jobs.bar","label":"${resources.jobs.bar.id}"}]}`, string(op.state)) } func TestNewRecordedOperationRejectsUnsupportedAction(t *testing.T) { - _, err := newRecordedOperation(deployplan.Skip, "job-123", nil) + _, err := newRecordedOperation(deployplan.Skip, "job-123", nil, nil) assert.Error(t, err) } From 386a0b616f6a409ab1c31fb34383b74bcd098547 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 28 Jul 2026 11:03:26 +0000 Subject: [PATCH 014/125] bundle: drop the DMS state overlay check Migrating an existing deployment is not supported: Open rejects a bundle that already has resources in state, so a deployment that exists in DMS was created by an opted-in CLI and DMS owns its resource set outright. The last_successful_version_id probe that decided whether to trust DMS was therefore always true by the time it ran. Removing it takes with it the raw GET that read the field (it is stage:DEVELOPMENT and stripped from the generated SDK) and DMSSource.Config, which existed only to make that call. overlayDMSState is now readDMSState, since it no longer overlays anything conditionally: it just reads. Recording stays opt-in via experimental.record_deployment_history; that flag is what makes the caller pass a DMSSource at all. acceptance/bundle/dms/existing-state also now covers wiping the local cache: deploy pulls the state file back from the workspace, so the resources stay tracked and opting in is still rejected. Co-authored-by: Isaac --- .../bundle/dms/existing-state/output.txt | 7 +++ acceptance/bundle/dms/existing-state/script | 5 ++ acceptance/bundle/dms/no-resources/output.txt | 4 -- bundle/direct/dstate/dms.go | 59 ++----------------- bundle/direct/dstate/state.go | 21 +++---- cmd/bundle/utils/process.go | 8 +-- libs/dms/resolve_test.go | 4 +- 7 files changed, 33 insertions(+), 75 deletions(-) diff --git a/acceptance/bundle/dms/existing-state/output.txt b/acceptance/bundle/dms/existing-state/output.txt index 116584d10c1..a793cc0fae1 100644 --- a/acceptance/bundle/dms/existing-state/output.txt +++ b/acceptance/bundle/dms/existing-state/output.txt @@ -18,6 +18,13 @@ Error: cannot record deployment history for a bundle that already has deployed r === No deployment was created in DMS >>> print_requests.py //api/2.0/bundle --sort --oneline +=== Still an error after wiping the local cache: deploy pulls the state file back from the workspace, so the resources are still tracked +>>> musterr [CLI] bundle deploy +Error: cannot record deployment history for a bundle that already has deployed resources tracked in [TEST_TMP_DIR]/.databricks/bundle/default/resources.json: only new deployments can be recorded. Remove experimental.record_deployment_history, or destroy the bundle and deploy it again + + +>>> print_requests.py //api/2.0/bundle --sort --oneline + === Destroy clears the tracked resources, so recording can be enabled afterwards >>> update_file.py databricks.yml record_deployment_history: true record_deployment_history: false diff --git a/acceptance/bundle/dms/existing-state/script b/acceptance/bundle/dms/existing-state/script index 7bc296465d5..ae9be95f701 100644 --- a/acceptance/bundle/dms/existing-state/script +++ b/acceptance/bundle/dms/existing-state/script @@ -9,6 +9,11 @@ trace musterr $CLI bundle deploy title "No deployment was created in DMS" trace print_requests.py //api/2.0/bundle --sort --oneline +title "Still an error after wiping the local cache: deploy pulls the state file back from the workspace, so the resources are still tracked" +rm -rf .databricks +trace musterr $CLI bundle deploy +trace print_requests.py //api/2.0/bundle --sort --oneline + title "Destroy clears the tracked resources, so recording can be enabled afterwards" trace update_file.py databricks.yml "record_deployment_history: true" "record_deployment_history: false" trace $CLI bundle destroy --auto-approve diff --git a/acceptance/bundle/dms/no-resources/output.txt b/acceptance/bundle/dms/no-resources/output.txt index 73bd3adfa11..b400fb9a67e 100644 --- a/acceptance/bundle/dms/no-resources/output.txt +++ b/acceptance/bundle/dms/no-resources/output.txt @@ -51,10 +51,6 @@ Deployment complete! "method": "GET", "path": "/api/2.0/bundle/deployments/[NUMID]" } -{ - "method": "GET", - "path": "/api/2.0/bundle/deployments/[NUMID]" -} { "method": "GET", "path": "/api/2.0/bundle/deployments/[NUMID]/resources" diff --git a/bundle/direct/dstate/dms.go b/bundle/direct/dstate/dms.go index e6ae63f6095..ec51e2c4479 100644 --- a/bundle/direct/dstate/dms.go +++ b/bundle/direct/dstate/dms.go @@ -3,15 +3,9 @@ package dstate import ( "context" "encoding/json" - "errors" "fmt" - "net/http" "github.com/databricks/cli/bundle/deployplan" - "github.com/databricks/cli/libs/auth" - "github.com/databricks/databricks-sdk-go/apierr" - "github.com/databricks/databricks-sdk-go/client" - sdkconfig "github.com/databricks/databricks-sdk-go/config" "github.com/databricks/databricks-sdk-go/service/bundledeployments" ) @@ -31,21 +25,12 @@ type RecordedState struct { DependsOn []deployplan.DependsOnEntry `json:"depends_on,omitempty"` } -// overlayDMSState replaces the file-derived resource state with the state -// recorded in DMS, when DMS owns this deployment. An authoritative DMS is -// trusted even when its resource set is empty (a successful deploy of nothing). -// The caller holds db.mu and has already populated db.Data from the file. -func (db *DeploymentState) overlayDMSState(ctx context.Context, src *DMSSource) error { - authoritative, err := deploymentHasSuccessfulVersion(ctx, src.Config, src.DeploymentID) - if err != nil { - return err - } - if !authoritative { - // DMS has no completed version for this deployment: a prior direct deploy - // that has not yet successfully recorded to DMS. Keep the file state. - return nil - } - +// readDMSState replaces the file-derived resource state with the state recorded +// in DMS. Recording is only enabled for net-new deployments, so once a +// deployment exists DMS owns its resource set outright - including when that set +// is empty, which is a successful deploy of nothing rather than missing data. +// The caller holds db.mu. +func (db *DeploymentState) readDMSState(ctx context.Context, src *DMSSource) error { resources, err := fetchDeploymentResources(ctx, src.Client, src.DeploymentID) if err != nil { return err @@ -59,38 +44,6 @@ func (db *DeploymentState) overlayDMSState(ctx context.Context, src *DMSSource) return nil } -// deploymentHasSuccessfulVersion reports whether DMS owns the state. The server -// advances last_successful_version_id only when a version completes (unlike -// last_version_id, which also advances on failure), so a non-empty value means -// DMS holds a complete resource set. Otherwise Open keeps the file's resources. -// -// TODO(DMS): raw GET because last_successful_version_id is stage:DEVELOPMENT and -// stripped from the generated SDK. Once it ships, use -// client.GetDeployment(...).LastSuccessfulVersionId and drop DMSSource.Config. -func deploymentHasSuccessfulVersion(ctx context.Context, cfg *sdkconfig.Config, deploymentID string) (bool, error) { - apiClient, err := client.New(cfg) - if err != nil { - return false, fmt.Errorf("creating API client for deployment metadata service: %w", err) - } - - // Mirrors the SDK's GetDeployment path (/api/2.0/bundle/{name} with - // name=deployments/{id}); we unmarshal into a local struct so we can read - // last_successful_version_id, which the typed SDK response drops. - var dep struct { - LastSuccessfulVersionID string `json:"last_successful_version_id"` - } - err = apiClient.Do(ctx, http.MethodGet, "/api/2.0/bundle/deployments/"+deploymentID, auth.WorkspaceIDHeaders(cfg), nil, nil, &dep) - if err != nil { - // A deployment that was never recorded to DMS is not an error here: it - // just means DMS is not (yet) the source of truth. - if errors.Is(err, apierr.ErrNotFound) { - return false, nil - } - return false, fmt.Errorf("reading deployment from deployment metadata service: %w", err) - } - return dep.LastSuccessfulVersionID != "", nil -} - // fetchDeploymentResources lists every resource recorded for the deployment in // DMS and maps them into state entries keyed by the fully-qualified resource key. func fetchDeploymentResources(ctx context.Context, client bundledeployments.BundleDeploymentsInterface, deploymentID string) (map[string]ResourceEntry, error) { diff --git a/bundle/direct/dstate/state.go b/bundle/direct/dstate/state.go index 0359feba89a..bad87a6a293 100644 --- a/bundle/direct/dstate/state.go +++ b/bundle/direct/dstate/state.go @@ -19,7 +19,6 @@ import ( "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/structs/structwalk" - sdkconfig "github.com/databricks/databricks-sdk-go/config" "github.com/databricks/databricks-sdk-go/service/bundledeployments" "github.com/google/uuid" ) @@ -221,24 +220,20 @@ type ( ) // DMSSource tells Open to read resource state from the deployment metadata -// service instead of the state file. A nil *DMSSource keeps Open file-only. +// service instead of the state file. Callers pass it only when the bundle set +// experimental.record_deployment_history; a nil *DMSSource keeps Open file-only. type DMSSource struct { Client bundledeployments.BundleDeploymentsInterface - // Config is only for a temporary raw read of last_successful_version_id; see - // the TODO in deploymentHasSuccessfulVersion. - Config *sdkconfig.Config - // DeploymentID is resolved from the deployment's workspace node (see // dms.ResolveDeploymentID), and empty before the first recorded deploy. DeploymentID string } // Open reads the deployment state from disk (and recovers the WAL when -// withRecovery is set). With a non-nil dmsSource, resources come from DMS -// instead of the file whenever DMS holds a successful version. Lineage and -// serial always come from the file, since that is what the write path -// increments. +// withRecovery is set). With a non-nil dmsSource, resources come from DMS rather +// than the file. Lineage and serial always come from the file, since that is +// what the write path increments. func (db *DeploymentState) Open(ctx context.Context, path string, withRecovery WithRecovery, withWrite WithWrite, dmsSource *DMSSource) error { db.mu.Lock() defer db.mu.Unlock() @@ -290,8 +285,8 @@ func (db *DeploymentState) Open(ctx context.Context, path string, withRecovery W if dmsSource != nil { // Only bundles that start out empty can be recorded. Once DMS owns a // deployment it is authoritative for the whole resource set (see - // overlayDMSState), so pre-existing resources it never saw would look absent - // and get created a second time. + // readDMSState), so pre-existing resources it never saw would look absent and + // get created a second time. // // TODO(DMS): allow this by upgrading the state in place, writing it at // featureStateVersion with a feature flag plus a tombstone per resource so an @@ -301,7 +296,7 @@ func (db *DeploymentState) Open(ctx context.Context, path string, withRecovery W return fmt.Errorf("cannot record deployment history for a bundle that already has deployed resources tracked in %s: only new deployments can be recorded. Remove experimental.record_deployment_history, or destroy the bundle and deploy it again", path) } if dmsSource.DeploymentID != "" { - if err := db.overlayDMSState(ctx, dmsSource); err != nil { + if err := db.readDMSState(ctx, dmsSource); err != nil { return err } } diff --git a/cmd/bundle/utils/process.go b/cmd/bundle/utils/process.go index e9840188db4..209dc874403 100644 --- a/cmd/bundle/utils/process.go +++ b/cmd/bundle/utils/process.go @@ -214,9 +214,10 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle _, localPath := b.StateFilenameDirect(ctx) // When the bundle records deployment history, the deployment metadata - // service owns resource state, so hand Open a DMS source to overlay that - // state on top of the local identity (lineage/serial). Reads open the - // state write-disabled, so no lineage is minted here. + // service owns resource state, so hand Open a DMS source to read it from + // there instead of the file. The local identity (lineage/serial) still + // comes from the file. Reads open the state write-disabled, so no lineage + // is minted here. var dmsSource *dstate.DMSSource if b.Config.Experimental != nil && b.Config.Experimental.RecordDeploymentHistory { w := b.WorkspaceClient(ctx) @@ -227,7 +228,6 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle } dmsSource = &dstate.DMSSource{ Client: w.BundleDeployments, - Config: w.Config, DeploymentID: deploymentID, } } diff --git a/libs/dms/resolve_test.go b/libs/dms/resolve_test.go index 24d5303983d..6838f4dd15c 100644 --- a/libs/dms/resolve_test.go +++ b/libs/dms/resolve_test.go @@ -25,7 +25,7 @@ func newTestClient(t *testing.T, statusCode int, body string) *databricks.Worksp })) t.Cleanup(srv.Close) t.Cleanup(func() { - assert.Equal(t, "/Workspace/state/"+DeploymentNodeName, gotPath) + assert.Equal(t, nodePath, gotPath) }) w, err := databricks.NewWorkspaceClient(&databricks.Config{ @@ -37,6 +37,8 @@ func newTestClient(t *testing.T, statusCode int, body string) *databricks.Worksp return w } +const nodePath = "/Workspace/state/" + DeploymentNodeName + func TestResolveDeploymentIDReturnsNodeID(t *testing.T) { w := newTestClient(t, http.StatusOK, `{"object_type":"FILE","object_id":123456789,"path":"/Workspace/state/`+DeploymentNodeName+`"}`) From efe7320cbeeaacecde8dc76ef68eed74b9e6d658 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 28 Jul 2026 23:44:57 +0000 Subject: [PATCH 015/125] bundle: gate DMS state on a state feature flag Reading resource state from DMS now requires the state file to record a "deployment_history" feature flag, rather than inferring eligibility from the resource count. The old check refused a state that had resources and no DMS deployment ID. That happened to be right, but it inferred intent from a side effect: a state with resources could equally be one DMS already owns. The flag says so directly. Header.Features was already scaffolded for exactly this, so this fills it in: - Open writes the flag when recording is enabled, and refuses a state that has resources without it. Migrating such a target is not supported, so the error names the target and the three ways out: use a new target, destroy this one and redeploy, or unset the feature. - A state recording any feature is written at featureStateVersion, so a CLI that predates the flag refuses it (see migrateState) instead of deploying against a resource set that lives in DMS and looks empty on disk. - migrateState accepts features it implements and refuses only unknown ones, naming just those in the error. - WAL replay carries the features forward, so recovering a WAL from a recording deploy still produces a state marked as DMS-owned. The flag is per target, which matches how experimental.record_deployment_history is set. Co-authored-by: Isaac --- .../bundle/dms/existing-state/output.txt | 4 +- acceptance/bundle/dms/record/output.txt | 9 ++ acceptance/bundle/dms/record/script | 3 + bundle/direct/dstate/migrate.go | 35 +++--- bundle/direct/dstate/state.go | 110 ++++++++++++------ bundle/direct/dstate/state_test.go | 21 +++- cmd/bundle/utils/process.go | 1 + 7 files changed, 132 insertions(+), 51 deletions(-) diff --git a/acceptance/bundle/dms/existing-state/output.txt b/acceptance/bundle/dms/existing-state/output.txt index a793cc0fae1..229b3bea03f 100644 --- a/acceptance/bundle/dms/existing-state/output.txt +++ b/acceptance/bundle/dms/existing-state/output.txt @@ -12,7 +12,7 @@ Deployment complete! >>> update_file.py databricks.yml record_deployment_history: false record_deployment_history: true >>> musterr [CLI] bundle deploy -Error: cannot record deployment history for a bundle that already has deployed resources tracked in [TEST_TMP_DIR]/.databricks/bundle/default/resources.json: only new deployments can be recorded. Remove experimental.record_deployment_history, or destroy the bundle and deploy it again +Error: target "default" was deployed without experimental.record_deployment_history and cannot be migrated to it: [TEST_TMP_DIR]/.databricks/bundle/default/resources.json tracks resources this deployment recorded before the feature was enabled. Use a new target, destroy this one and deploy it again, or unset experimental.record_deployment_history === No deployment was created in DMS @@ -20,7 +20,7 @@ Error: cannot record deployment history for a bundle that already has deployed r === Still an error after wiping the local cache: deploy pulls the state file back from the workspace, so the resources are still tracked >>> musterr [CLI] bundle deploy -Error: cannot record deployment history for a bundle that already has deployed resources tracked in [TEST_TMP_DIR]/.databricks/bundle/default/resources.json: only new deployments can be recorded. Remove experimental.record_deployment_history, or destroy the bundle and deploy it again +Error: target "default" was deployed without experimental.record_deployment_history and cannot be migrated to it: [TEST_TMP_DIR]/.databricks/bundle/default/resources.json tracks resources this deployment recorded before the feature was enabled. Use a new target, destroy this one and deploy it again, or unset experimental.record_deployment_history >>> print_requests.py //api/2.0/bundle --sort --oneline diff --git a/acceptance/bundle/dms/record/output.txt b/acceptance/bundle/dms/record/output.txt index e792d5d8d99..3266e0d3af8 100644 --- a/acceptance/bundle/dms/record/output.txt +++ b/acceptance/bundle/dms/record/output.txt @@ -73,6 +73,15 @@ Deployment complete! >>> jq has("deployment_id") .databricks/bundle/default/resources.json false +=== The state records the feature flag, at the version that makes a CLI without it refuse the state rather than deploy against resources it cannot see +>>> jq {state_version, features} .databricks/bundle/default/resources.json +{ + "state_version": 3, + "features": { + "deployment_history": {} + } +} + === Redeploy after deleting the local cache: the deployment ID is resolved from that node, the same deployment is reused, and the version increments (no new CreateDeployment) >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-record/default/files... diff --git a/acceptance/bundle/dms/record/script b/acceptance/bundle/dms/record/script index 63eaa323b1b..3298c8fb131 100644 --- a/acceptance/bundle/dms/record/script +++ b/acceptance/bundle/dms/record/script @@ -6,6 +6,9 @@ title "The deployment ID is the ID of the workspace node the service registered trace $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-record/default/state/resources.deployment.json" | jq '{object_type,path}' trace jq 'has("deployment_id")' .databricks/bundle/default/resources.json +title "The state records the feature flag, at the version that makes a CLI without it refuse the state rather than deploy against resources it cannot see" +trace jq '{state_version, features}' .databricks/bundle/default/resources.json + title "Redeploy after deleting the local cache: the deployment ID is resolved from that node, the same deployment is reused, and the version increments (no new CreateDeployment)" rm -rf .databricks trace $CLI bundle deploy diff --git a/bundle/direct/dstate/migrate.go b/bundle/direct/dstate/migrate.go index e4d21a7054a..288fdcf863f 100644 --- a/bundle/direct/dstate/migrate.go +++ b/bundle/direct/dstate/migrate.go @@ -12,25 +12,34 @@ import ( "github.com/databricks/databricks-sdk-go/service/iam" ) +// knownFeatures lists the state feature flags this CLI implements. A state that +// records anything outside this set is refused by migrateState. +var knownFeatures = map[string]bool{ + FeatureDeploymentHistory: true, +} + // migrateState runs all necessary migrations on the database. // It is called after loading state from disk. func migrateState(db *Database) error { - // featureStateVersion states carry a feature list this CLI does not yet write or - // understand (see the featureStateVersion doc comment). A featureStateVersion - // state with no features is equivalent to currentStateVersion, so accept it and - // return without running the migrations below, leaving the on-disk version at - // featureStateVersion rather than flipping it down. One that records any feature - // depends on capabilities this CLI lacks, so refuse it and tell the user to upgrade. + // featureStateVersion states carry a feature list (see the featureStateVersion + // doc comment). A featureStateVersion state with no features is equivalent to + // currentStateVersion, so accept it and return without running the migrations + // below, leaving the on-disk version at featureStateVersion rather than flipping + // it down. Same for a state whose features this CLI implements. One that records + // a feature this CLI does not know depends on capabilities it lacks, so refuse it + // and tell the user to upgrade. if db.StateVersion == featureStateVersion { - if len(db.Features) == 0 { - return nil - } - features := make([]string, 0, len(db.Features)) + unknown := make([]string, 0, len(db.Features)) for name := range db.Features { - features = append(features, name) + if !knownFeatures[name] { + unknown = append(unknown, name) + } + } + if len(unknown) == 0 { + return nil } - slices.Sort(features) - return fmt.Errorf("the deployment state requires features this CLI does not support: %s; upgrade to the latest CLI version and see %s for more information", strings.Join(features, ", "), featuresDocURL) + slices.Sort(unknown) + return fmt.Errorf("the deployment state requires features this CLI does not support: %s; upgrade to the latest CLI version and see %s for more information", strings.Join(unknown, ", "), featuresDocURL) } if db.StateVersion == currentStateVersion { diff --git a/bundle/direct/dstate/state.go b/bundle/direct/dstate/state.go index bad87a6a293..b367e97001b 100644 --- a/bundle/direct/dstate/state.go +++ b/bundle/direct/dstate/state.go @@ -31,22 +31,28 @@ const ( maxWalEntrySize = 10 * 1024 * 1024 walSuffix = ".wal" - // featureStateVersion is the schema version a future CLI will write once it - // records deployment state "feature flags" (see Header.Features). This CLI does - // not write it and records no features; it exists now only so this CLI reads - // such states correctly (see migrateState): - // - featureStateVersion with no features -> accept and leave the version as-is - // - featureStateVersion with any feature -> refuse, tell the user to upgrade + // featureStateVersion is the schema version written once a state records + // deployment state "feature flags" (see Header.Features). Reading such a state + // (see migrateState): + // - featureStateVersion with no features -> accept, leave the version as-is + // - featureStateVersion with known features -> accept + // - featureStateVersion with unknown features -> refuse, tell the user to upgrade // // A featureStateVersion state with no features is equivalent to // currentStateVersion, but we deliberately do not flip the on-disk version down // to currentStateVersion: a state written at featureStateVersion stays at - // featureStateVersion. This is forward-compat scaffolding so that a later release - // can start writing featureStateVersion + features without older CLIs (with this - // change) either mishandling a feature they lack or rejecting a featureless state - // outright. featureStateVersion is always 3. + // featureStateVersion. That way a release can start writing + // featureStateVersion + features without older CLIs either mishandling a feature + // they lack or rejecting a featureless state outright. featureStateVersion is + // always 3. featureStateVersion = 3 + // FeatureDeploymentHistory marks a state whose resources live in the deployment + // metadata service rather than in the state file. A CLI that does not know this + // feature refuses the state instead of deploying against a resource set it + // cannot see - the file's resources are not authoritative. + FeatureDeploymentHistory = "deployment_history" + // supportedStateVersion is the highest schema version this CLI can read. It is // normally equal to currentStateVersion — the version this CLI reads is the // version it writes — and exceeds it only during a two-phase version bump like @@ -82,13 +88,27 @@ type Header struct { Serial int `json:"serial"` // Features maps each feature flag this state depends on to a (currently empty) - // value. This CLI writes no features; it only reads the field to detect a state - // that depends on features it lacks and refuse it (see migrateState). It is a - // map so a future CLI can attach per-feature data without reshaping the state. - // Empty/omitted for states that use no features. + // value. A CLI that does not implement one of them refuses the state rather than + // deploying against it (see migrateState). It is a map so a future CLI can attach + // per-feature data without reshaping the state. Empty/omitted for states that use + // no features. Features map[string]struct{} `json:"features,omitempty"` } +// hasFeature reports whether the state records the given feature flag. +func (h *Header) hasFeature(name string) bool { + _, ok := h.Features[name] + return ok +} + +// setFeature records a feature flag in the state. +func (h *Header) setFeature(name string) { + if h.Features == nil { + h.Features = make(map[string]struct{}) + } + h.Features[name] = struct{}{} +} + type Database struct { Header @@ -228,6 +248,10 @@ type DMSSource struct { // DeploymentID is resolved from the deployment's workspace node (see // dms.ResolveDeploymentID), and empty before the first recorded deploy. DeploymentID string + + // TargetName names the bundle target in the error Open returns for a state that + // predates the opt-in, since the feature is enabled per target. + TargetName string } // Open reads the deployment state from disk (and recovers the WAL when @@ -283,18 +307,18 @@ func (db *DeploymentState) Open(ctx context.Context, path string, withRecovery W } if dmsSource != nil { - // Only bundles that start out empty can be recorded. Once DMS owns a - // deployment it is authoritative for the whole resource set (see - // readDMSState), so pre-existing resources it never saw would look absent and - // get created a second time. - // - // TODO(DMS): allow this by upgrading the state in place, writing it at - // featureStateVersion with a feature flag plus a tombstone per resource so an - // older CLI refuses the state instead of deploying against resources it - // cannot see. - if dmsSource.DeploymentID == "" && len(db.Data.State) > 0 { - return fmt.Errorf("cannot record deployment history for a bundle that already has deployed resources tracked in %s: only new deployments can be recorded. Remove experimental.record_deployment_history, or destroy the bundle and deploy it again", path) + // An existing state file has to be marked as DMS-owned before its resources + // can be read from DMS. Recording only starts on an empty state, so a state + // with resources and no feature flag predates the opt-in: DMS never saw those + // resources and is authoritative for the whole set once enabled (see + // readDMSState), so they would look absent and be created a second time. + // Migrating such a target is not supported yet. + if len(db.Data.State) > 0 && !db.Data.hasFeature(FeatureDeploymentHistory) { + return fmt.Errorf("target %q was deployed without experimental.record_deployment_history and cannot be migrated to it: %s tracks resources this deployment recorded before the feature was enabled. Use a new target, destroy this one and deploy it again, or unset experimental.record_deployment_history", dmsSource.TargetName, path) } + // Mark the state as DMS-owned so a CLI without this feature refuses it rather + // than deploying against a resource set it cannot see. + db.Data.setFeature(FeatureDeploymentHistory) if dmsSource.DeploymentID != "" { if err := db.readDMSState(ctx, dmsSource); err != nil { return err @@ -311,13 +335,7 @@ func (db *DeploymentState) Open(ctx context.Context, path string, withRecovery W return fmt.Errorf("failed to open WAL file %s: %w", walPath, err) } db.walFile = walFile - walHead := Header{ - Lineage: db.GetOrInitLineage(), - Serial: db.Data.Serial + 1, - StateVersion: currentStateVersion, - CLIVersion: build.GetInfo().Version, - } - return appendJSONLine(db.walFile, walHead) + return appendJSONLine(db.walFile, db.newWalHeader()) } return nil @@ -404,6 +422,16 @@ func (db *DeploymentState) mergeWalIntoState(ctx context.Context) (bool, error) return false, fmt.Errorf("WAL serial (%d) is ahead of expected (%d), state may be corrupted", header.Serial, expectedSerial) } newSerial = header.Serial + + // Carry the WAL's features (and the version that goes with them) into the + // state being written, so recovering a WAL written by a DMS-recording deploy + // still produces a state marked as DMS-owned. + for name := range header.Features { + db.Data.setFeature(name) + } + if len(db.Data.Features) > 0 { + db.Data.StateVersion = featureStateVersion + } } else { var entry WALEntry if err := json.Unmarshal(line, &entry); err != nil { @@ -507,13 +535,25 @@ func (db *DeploymentState) UpgradeToWrite() error { } db.walFile = walFile - walHead := Header{ + return appendJSONLine(db.walFile, db.newWalHeader()) +} + +// newWalHeader builds the header for a fresh WAL. Features carry over from the +// state being written, and a state that records any feature is written at +// featureStateVersion so a CLI that lacks the feature refuses it instead of +// deploying against it. The caller holds db.mu. +func (db *DeploymentState) newWalHeader() Header { + version := currentStateVersion + if len(db.Data.Features) > 0 { + version = featureStateVersion + } + return Header{ Lineage: db.GetOrInitLineage(), Serial: db.Data.Serial + 1, - StateVersion: currentStateVersion, + StateVersion: version, CLIVersion: build.GetInfo().Version, + Features: db.Data.Features, } - return appendJSONLine(db.walFile, walHead) } func (db *DeploymentState) AssertOpenedForReadOrWrite() { diff --git a/bundle/direct/dstate/state_test.go b/bundle/direct/dstate/state_test.go index a9c90530514..35050575121 100644 --- a/bundle/direct/dstate/state_test.go +++ b/bundle/direct/dstate/state_test.go @@ -154,7 +154,16 @@ func TestEmptyFeatureStateAcceptedWithoutFlippingVersion(t *testing.T) { require.NoError(t, migrateState(empty)) assert.Equal(t, featureStateVersion, empty.StateVersion, "v3 + no features keeps its on-disk version, not flipped to v2") - // v3 that records a feature is refused: this CLI does not understand features. + // v3 recording a feature this CLI implements is accepted. + known := &Database{Header: Header{ + StateVersion: featureStateVersion, + Features: map[string]struct{}{FeatureDeploymentHistory: {}}, + }} + require.NoError(t, migrateState(known)) + assert.Equal(t, featureStateVersion, known.StateVersion) + + // v3 recording a feature this CLI does not know is refused: its resources may + // live somewhere this CLI cannot see. withFeature := &Database{Header: Header{ StateVersion: featureStateVersion, Features: map[string]struct{}{"future_feature": {}}, @@ -165,6 +174,16 @@ func TestEmptyFeatureStateAcceptedWithoutFlippingVersion(t *testing.T) { assert.Contains(t, err.Error(), "future_feature") assert.Contains(t, err.Error(), "upgrade to the latest CLI version") assert.Contains(t, err.Error(), featuresDocURL) + + // Only the unknown feature is named, so the message tells the user what to do. + mixed := &Database{Header: Header{ + StateVersion: featureStateVersion, + Features: map[string]struct{}{FeatureDeploymentHistory: {}, "future_feature": {}}, + }} + err = migrateState(mixed) + require.Error(t, err) + assert.Contains(t, err.Error(), "future_feature") + assert.NotContains(t, err.Error(), FeatureDeploymentHistory) } func TestDeleteState(t *testing.T) { diff --git a/cmd/bundle/utils/process.go b/cmd/bundle/utils/process.go index 209dc874403..b000da60264 100644 --- a/cmd/bundle/utils/process.go +++ b/cmd/bundle/utils/process.go @@ -229,6 +229,7 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle dmsSource = &dstate.DMSSource{ Client: w.BundleDeployments, DeploymentID: deploymentID, + TargetName: b.Config.Bundle.Target, } } if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WithRecovery(true), dstate.WithWrite(false), dmsSource); err != nil { From 92eac95b722c6148fedcc03f79db081257c5e152 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 29 Jul 2026 09:49:01 +0000 Subject: [PATCH 016/125] bundle: combine queued operations into a single Create action Coalescing now keeps the newest operation outright instead of merging fields. Each operation carries the resource's full state rather than a delta, so a newer one entirely supersedes an older one - including its resource_id, which is the field that actually changes between two records (a create learns the ID only after the API call returns it). The previous code merged the action and overwrote the ID, which is backwards: the action cannot differ between records of the same resource, while the ID can. mergeAction is gone. Also renames `owned` to `queuedOrUploading`. Nothing is owned by a particular worker: a key can be handled by one worker, released, and picked up later by another. The mark only means "some worker will get to this", which is all record needs to know in order to not queue the key twice. The doc comments now lead with the two rules that shape the design (no overlapping uploads per resource; only the newest operation matters) so the mechanism reads as a consequence of them rather than as bookkeeping. Co-authored-by: Isaac --- bundle/direct/opqueue.go | 94 ++++++++++++++++++-------------- bundle/direct/opqueue_test.go | 33 +++++++---- bundle/direct/oprecorder.go | 18 ------ bundle/direct/oprecorder_test.go | 30 ---------- 4 files changed, 74 insertions(+), 101 deletions(-) diff --git a/bundle/direct/opqueue.go b/bundle/direct/opqueue.go index 1f38c2b7dd6..9a1a1cd918e 100644 --- a/bundle/direct/opqueue.go +++ b/bundle/direct/opqueue.go @@ -21,37 +21,48 @@ const ( operationUploadWorkers = 4 ) -// operationQueue uploads recorded operations from background workers, so an apply -// worker does not wait for the CreateOperation round trip. +// operationQueue hands recorded operations to background workers, so an apply +// worker does not wait for the CreateOperation round trip before deploying the +// next resource. // -// At most one upload is in flight per resource key, so the last operation -// recorded for a resource is also the last one the service sees. +// Two rules shape the design: +// +// - Uploads for one resource never overlap. DMS stores one state per resource +// key, so concurrent uploads could land out of order and leave stale state. +// - Only the newest operation for a resource matters. Each operation carries the +// resource's full state, not a delta, so a newer one entirely supersedes an +// older one. When both are still waiting, the older is dropped ("coalesced") +// and one upload records the result. // // Uploads are not fire-and-forget: close returns the first failure and fails the // deploy. A dropped operation would leave DMS with an incomplete resource set, -// and since DMS then becomes the source of truth (see dstate.overlayDMSState), -// the next deploy would recreate resources that already exist. +// and since DMS then becomes the source of truth (see dstate.readDMSState), the +// next deploy would recreate resources that already exist. type operationQueue struct { uploader operationUploader - // queue carries resource keys, not the operations themselves: a worker looks - // the operation up in pending when it picks the key up, which is what lets - // record collapse repeated writes to the same resource. + // queue carries resource keys, not operations. A worker looks the operation up + // when it picks the key up, so recording again before then just overwrites the + // entry in pending - that is what makes coalescing work. queue chan string wg sync.WaitGroup // mu guards the fields below. mu sync.Mutex - // pending is the latest operation recorded per resource key that no worker has - // picked up yet. + // pending holds the newest operation per resource key that no worker has taken + // yet. Empty for a key means everything recorded for it has been uploaded. pending map[string]recordedOperation - // owned holds the resource keys that are already queued or being uploaded. - // Such a key is never queued a second time; recording writes to pending - // instead, which the owning worker re-checks after each upload. This is what - // keeps uploads for one resource sequential. - owned map[string]bool + // queuedOrUploading marks keys that are already in the queue channel or being + // uploaded right now. Such a key must not be queued again, or two workers could + // upload the same resource at once; recording writes to pending instead, and + // the worker handling the key picks it up when its current upload finishes. + // + // No single worker "owns" a key for the whole time it is marked: a key can be + // handled by one worker, released, and later picked up by another. The mark only + // means "some worker will get to this", which is all record needs to know. + queuedOrUploading map[string]bool err error closed bool @@ -68,10 +79,10 @@ func newOperationQueue(ctx context.Context, uploader operationUploader) *operati } q := &operationQueue{ - uploader: uploader, - queue: make(chan string, operationQueueSize), - pending: make(map[string]recordedOperation), - owned: make(map[string]bool), + uploader: uploader, + queue: make(chan string, operationQueueSize), + pending: make(map[string]recordedOperation), + queuedOrUploading: make(map[string]bool), } q.wg.Add(operationUploadWorkers) @@ -82,14 +93,12 @@ func newOperationQueue(ctx context.Context, uploader operationUploader) *operati return q } -// record serializes an operation and queues it for upload. It makes no API call, -// so upload failures surface from close; an error here only means the applied -// resource could not be turned into a payload. +// record serializes an operation and hands it to the upload workers. It makes no +// API call, so upload failures surface from close; an error here only means the +// applied resource could not be turned into a payload. // -// An operation for a resource that is still waiting replaces it rather than -// queueing again: DMS keeps one state per key, so one upload records both. The -// merge keeps a queued create's action (see mergeAction). Best effort — only -// operations no worker has picked up yet are collapsed. +// Recording a resource that is still waiting replaces the waiting operation +// outright, since the newer one carries the resource's full state. func (q *operationQueue) record(ctx context.Context, resourceKey string, action deployplan.ActionType, resourceID string, state any, dependsOn []deployplan.DependsOnEntry) error { if q == nil { return nil @@ -101,22 +110,20 @@ func (q *operationQueue) record(ctx context.Context, resourceKey string, action } q.mu.Lock() - queued, waiting := q.pending[resourceKey] - if waiting { - op.action = mergeAction(queued.action, op.action) - } + _, replaced := q.pending[resourceKey] q.pending[resourceKey] = op - owned := q.owned[resourceKey] - q.owned[resourceKey] = true + alreadyHandled := q.queuedOrUploading[resourceKey] + q.queuedOrUploading[resourceKey] = true q.mu.Unlock() - if waiting { + if replaced { log.Debugf(ctx, "Coalescing queued deployment operation for %s", resourceKey) } - // Someone already owns this key, so pending is enough: a worker will pick the - // operation up. Queueing again would upload the resource twice in parallel. - if owned { + // A worker is already going to handle this key, and it re-reads pending before + // finishing, so it will see the operation written above. Queueing the key again + // would let a second worker upload the same resource concurrently. + if alreadyHandled { return nil } @@ -152,7 +159,7 @@ func (q *operationQueue) work(ctx context.Context) { defer q.wg.Done() for resourceKey := range q.queue { - // Keep uploading this key until nothing new was recorded for it, instead of + // Keep uploading this key until nothing new was recorded for it, rather than // putting it back on the queue: a worker sending to the channel it consumes // from can deadlock once the queue is full. for { @@ -168,16 +175,19 @@ func (q *operationQueue) work(ctx context.Context) { } } -// take claims the operation waiting for resourceKey. It reports false and gives -// up ownership when nothing is waiting, which is what lets the next record -// queue the key again. +// take claims the operation waiting for resourceKey. It reports false and clears +// the queuedOrUploading mark when nothing is waiting, which is what lets the next +// record queue the key again. +// +// Clearing the mark and observing pending empty happen under one lock, so record +// can never skip queueing a key that no worker is going to look at again. func (q *operationQueue) take(resourceKey string) (recordedOperation, bool) { q.mu.Lock() defer q.mu.Unlock() op, ok := q.pending[resourceKey] if !ok { - delete(q.owned, resourceKey) + delete(q.queuedOrUploading, resourceKey) return recordedOperation{}, false } diff --git a/bundle/direct/opqueue_test.go b/bundle/direct/opqueue_test.go index 6bf0da8107b..356a7cfb319 100644 --- a/bundle/direct/opqueue_test.go +++ b/bundle/direct/opqueue_test.go @@ -22,9 +22,10 @@ type fakeUploader struct { started chan string err error - mu sync.Mutex - uploads []string - actions map[string]bundledeployments.OperationActionType + mu sync.Mutex + uploads []string + actions map[string]bundledeployments.OperationActionType + resourceIDs map[string]string } func (f *fakeUploader) upload(ctx context.Context, resourceKey string, op recordedOperation) error { @@ -40,8 +41,10 @@ func (f *fakeUploader) upload(ctx context.Context, resourceKey string, op record f.uploads = append(f.uploads, resourceKey+"="+string(op.state)) if f.actions == nil { f.actions = map[string]bundledeployments.OperationActionType{} + f.resourceIDs = map[string]string{} } f.actions[resourceKey] = op.action + f.resourceIDs[resourceKey] = op.resourceID return f.err } @@ -57,6 +60,12 @@ func (f *fakeUploader) actionFor(resourceKey string) bundledeployments.Operation return f.actions[resourceKey] } +func (f *fakeUploader) resourceIDFor(resourceKey string) string { + f.mu.Lock() + defer f.mu.Unlock() + return f.resourceIDs[resourceKey] +} + func recordState(t *testing.T, q *operationQueue, resourceKey, name string) { t.Helper() require.NoError(t, q.record(t.Context(), resourceKey, deployplan.Update, "id-1", map[string]string{"name": name}, nil)) @@ -99,9 +108,8 @@ func TestOperationQueueCoalescesQueuedOperationsForSameResource(t *testing.T) { }, f.recorded()) } -func TestOperationQueueCoalescingKeepsCreateAction(t *testing.T) { - // Hold the first upload so the create below stays queued and the update - // coalesces into it. +func TestOperationQueueCoalescingKeepsLatestOperation(t *testing.T) { + // Hold the first upload so the operations below stay queued and coalesce. f := &fakeUploader{block: make(chan struct{}), started: make(chan string, 1)} q := newOperationQueue(t.Context(), f) @@ -114,15 +122,18 @@ func TestOperationQueueCoalescingKeepsCreateAction(t *testing.T) { assert.Equal(t, "resources.jobs.hold"+strconv.Itoa(i), <-f.started) } - require.NoError(t, q.record(t.Context(), "resources.jobs.foo", deployplan.Create, "id-1", map[string]string{"name": "created"}, nil)) - require.NoError(t, q.record(t.Context(), "resources.jobs.foo", deployplan.Update, "id-1", map[string]string{"name": "updated"}, nil)) + // A resource whose ID is only known after it was created: the first operation + // has no ID, the second fills it in. + require.NoError(t, q.record(t.Context(), "resources.jobs.foo", deployplan.Create, "", map[string]string{"name": "created"}, nil)) + require.NoError(t, q.record(t.Context(), "resources.jobs.foo", deployplan.Create, "id-1", map[string]string{"name": "updated"}, nil)) close(f.block) require.NoError(t, q.close()) - // The state is the later one, but the action stays CREATE: recording an update - // would tell DMS the resource already existed before this deploy. + // Everything comes from the newest operation: it carries the resource's full + // state, and the ID it learned after the create. assert.Contains(t, f.recorded(), `resources.jobs.foo={"state":{"name":"updated"}}`) + assert.Equal(t, "id-1", f.resourceIDFor("resources.jobs.foo")) assert.Equal(t, bundledeployments.OperationActionTypeOperationActionTypeCreate, f.actionFor("resources.jobs.foo")) @@ -239,7 +250,7 @@ func TestOperationQueueUploadsOneResourceAtATime(t *testing.T) { // Every distinct key was recorded, and close drained all of them. assert.Len(t, u.last, distinctKeyMod) assert.Empty(t, q.pending) - assert.Empty(t, q.owned) + assert.Empty(t, q.queuedOrUploading) } func TestNilOperationQueueIsNoOp(t *testing.T) { diff --git a/bundle/direct/oprecorder.go b/bundle/direct/oprecorder.go index 91bcb65a4f6..2401e34fcf8 100644 --- a/bundle/direct/oprecorder.go +++ b/bundle/direct/oprecorder.go @@ -68,24 +68,6 @@ func newRecordedOperation(action deployplan.ActionType, resourceID string, state return op, nil } -// mergeAction returns the action to record when a later operation coalesces into -// one still queued for the same resource (see operationQueue.record). The state -// uploaded is the later one, but the action must not be downgraded: Create and -// Recreate tell DMS the resource ID is new, and a subsequent Update only refines -// the state of that same new resource. Recording the pair as an Update would -// claim the resource already existed. A Delete is the exception - the resource is -// gone, so nothing earlier is worth reporting. -func mergeAction(queued, next bundledeployments.OperationActionType) bundledeployments.OperationActionType { - if next == bundledeployments.OperationActionTypeOperationActionTypeDelete { - return next - } - if queued == bundledeployments.OperationActionTypeOperationActionTypeCreate || - queued == bundledeployments.OperationActionTypeOperationActionTypeRecreate { - return queued - } - return next -} - // operationUploader records an applied resource operation with DMS. Uploads run // on the operationQueue workers, off the apply path. type operationUploader interface { diff --git a/bundle/direct/oprecorder_test.go b/bundle/direct/oprecorder_test.go index 556f7f59f90..f7262c04632 100644 --- a/bundle/direct/oprecorder_test.go +++ b/bundle/direct/oprecorder_test.go @@ -99,36 +99,6 @@ func TestNewRecordedOperationRejectsUnsupportedAction(t *testing.T) { assert.Error(t, err) } -func TestMergeAction(t *testing.T) { - const ( - create = bundledeployments.OperationActionTypeOperationActionTypeCreate - recreate = bundledeployments.OperationActionTypeOperationActionTypeRecreate - update = bundledeployments.OperationActionTypeOperationActionTypeUpdate - resize = bundledeployments.OperationActionTypeOperationActionTypeResize - del = bundledeployments.OperationActionTypeOperationActionTypeDelete - ) - - cases := []struct { - queued, next, want bundledeployments.OperationActionType - }{ - // A queued create is not downgraded: the resource is still new. - {create, update, create}, - {create, resize, create}, - {recreate, update, recreate}, - {create, create, create}, - // A delete wins: the resource is gone, so the earlier action is moot. - {create, del, del}, - {update, del, del}, - // Neither side is a create, so the later action stands. - {update, resize, resize}, - {resize, update, update}, - {del, create, create}, - } - for _, c := range cases { - assert.Equal(t, c.want, mergeAction(c.queued, c.next), "queued %s, next %s", c.queued, c.next) - } -} - func TestDeployActionToSDK(t *testing.T) { cases := []struct { action deployplan.ActionType From 96c0826849483f30565a609174637834bc909bc8 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 29 Jul 2026 10:13:16 +0000 Subject: [PATCH 017/125] acceptance: set MSYS_NO_PATHCONV for the DMS tests The dms tests pass workspace paths to $CLI (`workspace get-status /Workspace/...`). On Windows, Git Bash rewrites a leading-'/' argument into a Windows path before the CLI sees it, so the lookup goes to C:/Program Files/Git/Workspace/... and 404s, failing record, no-resources and redeploy-after-destroy on that platform only. Same fix and reason as acceptance/cmd/workspace/export-dir-*/test.toml. Co-authored-by: Isaac --- acceptance/bundle/dms/test.toml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/acceptance/bundle/dms/test.toml b/acceptance/bundle/dms/test.toml index 1e36331a16f..40cc45a44a5 100644 --- a/acceptance/bundle/dms/test.toml +++ b/acceptance/bundle/dms/test.toml @@ -10,3 +10,11 @@ RecordRequests = true Ignore = [ '.databricks', ] + +# These tests pass workspace paths to $CLI. On Windows, Git Bash rewrites a +# leading-'/' argument into a Windows path before the CLI sees it, so +# `workspace get-status /Workspace/...` looks up C:/Program Files/Git/Workspace/... +# and 404s. Quoting the argument does not help - the conversion happens in the +# Windows binary's argument processing. +[Env] +MSYS_NO_PATHCONV = "1" From b2ef0e83a76c48527379f2b5f39edb82a87169e4 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 29 Jul 2026 10:39:34 +0000 Subject: [PATCH 018/125] acceptance: scope MSYS_NO_PATHCONV to the get-status commands Setting it in test.toml applied it to the whole test, which stopped Git Bash converting the PATH too - so python3 could not find print_requests.py ("can't open file 'C:\\c\\a\\cli\\cli\\acceptance\\bin\\print_requests.py'") and every dms test failed on Windows, including the three that were passing. trace exports leading KEY=value pairs in a subshell, so setting it there fixes the CLI's leading-'/' argument without reaching the helpers. The precedent this was copied from (acceptance/cmd/workspace/export-dir-*/test.toml) uses no python helpers, which is why the test.toml form is safe there but not here. Co-authored-by: Isaac --- acceptance/bundle/dms/no-resources/output.txt | 2 +- acceptance/bundle/dms/no-resources/script | 2 +- acceptance/bundle/dms/record/output.txt | 2 +- acceptance/bundle/dms/record/script | 6 +++++- acceptance/bundle/dms/redeploy-after-destroy/output.txt | 4 ++-- acceptance/bundle/dms/redeploy-after-destroy/script | 4 ++-- acceptance/bundle/dms/test.toml | 8 -------- 7 files changed, 12 insertions(+), 16 deletions(-) diff --git a/acceptance/bundle/dms/no-resources/output.txt b/acceptance/bundle/dms/no-resources/output.txt index b400fb9a67e..86009c71b94 100644 --- a/acceptance/bundle/dms/no-resources/output.txt +++ b/acceptance/bundle/dms/no-resources/output.txt @@ -34,7 +34,7 @@ Deployment complete! } } ->>> [CLI] workspace get-status /Workspace/Users/[USERNAME]/.bundle/dms-no-resources/default/state/resources.deployment.json +>>> MSYS_NO_PATHCONV=1 [CLI] workspace get-status /Workspace/Users/[USERNAME]/.bundle/dms-no-resources/default/state/resources.deployment.json { "object_type": "FILE", "path": "/Workspace/Users/[USERNAME]/.bundle/dms-no-resources/default/state/resources.deployment.json" diff --git a/acceptance/bundle/dms/no-resources/script b/acceptance/bundle/dms/no-resources/script index 847935af5d1..9b14355bd27 100644 --- a/acceptance/bundle/dms/no-resources/script +++ b/acceptance/bundle/dms/no-resources/script @@ -1,7 +1,7 @@ title "First deploy of a bundle with no resources: the deployment is created, and its workspace node identifies it even though no resource state was written" trace $CLI bundle deploy trace print_requests.py //api/2.0/bundle --sort --get -trace $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-no-resources/default/state/resources.deployment.json" | jq '{object_type,path}' +trace MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-no-resources/default/state/resources.deployment.json" | jq '{object_type,path}' title "Redeploy: the deployment is resolved from that node, so no second deployment is created" trace $CLI bundle deploy diff --git a/acceptance/bundle/dms/record/output.txt b/acceptance/bundle/dms/record/output.txt index 3266e0d3af8..c2b3333e587 100644 --- a/acceptance/bundle/dms/record/output.txt +++ b/acceptance/bundle/dms/record/output.txt @@ -64,7 +64,7 @@ Deployment complete! } === The deployment ID is the ID of the workspace node the service registered under initial_parent_path; the CLI stores nothing locally ->>> [CLI] workspace get-status /Workspace/Users/[USERNAME]/.bundle/dms-record/default/state/resources.deployment.json +>>> MSYS_NO_PATHCONV=1 [CLI] workspace get-status /Workspace/Users/[USERNAME]/.bundle/dms-record/default/state/resources.deployment.json { "object_type": "FILE", "path": "/Workspace/Users/[USERNAME]/.bundle/dms-record/default/state/resources.deployment.json" diff --git a/acceptance/bundle/dms/record/script b/acceptance/bundle/dms/record/script index 3298c8fb131..0ad4dd96d98 100644 --- a/acceptance/bundle/dms/record/script +++ b/acceptance/bundle/dms/record/script @@ -3,7 +3,11 @@ trace $CLI bundle deploy trace print_requests.py //api/2.0/bundle --sort title "The deployment ID is the ID of the workspace node the service registered under initial_parent_path; the CLI stores nothing locally" -trace $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-record/default/state/resources.deployment.json" | jq '{object_type,path}' +# MSYS_NO_PATHCONV stops Git Bash on Windows from rewriting the leading-'/' path +# into C:/Program Files/Git/Workspace/... before the CLI sees it. Set per command +# rather than in test.toml: trace exports it in a subshell, so it cannot reach the +# python helpers, whose PATH does need converting. +trace MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-record/default/state/resources.deployment.json" | jq '{object_type,path}' trace jq 'has("deployment_id")' .databricks/bundle/default/resources.json title "The state records the feature flag, at the version that makes a CLI without it refuse the state rather than deploy against resources it cannot see" diff --git a/acceptance/bundle/dms/redeploy-after-destroy/output.txt b/acceptance/bundle/dms/redeploy-after-destroy/output.txt index ff8694b5424..b8b6f3c630d 100644 --- a/acceptance/bundle/dms/redeploy-after-destroy/output.txt +++ b/acceptance/bundle/dms/redeploy-after-destroy/output.txt @@ -15,7 +15,7 @@ All files and directories at the following location will be deleted: /Workspace/ Deleting files... Destroy complete! ->>> musterr [CLI] workspace get-status /Workspace/Users/[USERNAME]/.bundle/dms-redeploy-after-destroy/default/state/resources.deployment.json +>>> MSYS_NO_PATHCONV=1 musterr [CLI] workspace get-status /Workspace/Users/[USERNAME]/.bundle/dms-redeploy-after-destroy/default/state/resources.deployment.json Error: Path (/Workspace/Users/[USERNAME]/.bundle/dms-redeploy-after-destroy/default/state/resources.deployment.json) doesn't exist. === Deploy again: with no node to resolve, a fresh deployment is created at version 1, with its own workspace node @@ -25,7 +25,7 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> [CLI] workspace get-status /Workspace/Users/[USERNAME]/.bundle/dms-redeploy-after-destroy/default/state/resources.deployment.json +>>> MSYS_NO_PATHCONV=1 [CLI] workspace get-status /Workspace/Users/[USERNAME]/.bundle/dms-redeploy-after-destroy/default/state/resources.deployment.json { "object_type": "FILE", "path": "/Workspace/Users/[USERNAME]/.bundle/dms-redeploy-after-destroy/default/state/resources.deployment.json" diff --git a/acceptance/bundle/dms/redeploy-after-destroy/script b/acceptance/bundle/dms/redeploy-after-destroy/script index 04c639019c9..a39edf3c0d8 100644 --- a/acceptance/bundle/dms/redeploy-after-destroy/script +++ b/acceptance/bundle/dms/redeploy-after-destroy/script @@ -3,9 +3,9 @@ trace $CLI bundle deploy trace $CLI bundle destroy --auto-approve print_requests.py //api/2.0/bundle --sort --get > /dev/null -trace musterr $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-redeploy-after-destroy/default/state/resources.deployment.json" +trace MSYS_NO_PATHCONV=1 musterr $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-redeploy-after-destroy/default/state/resources.deployment.json" title "Deploy again: with no node to resolve, a fresh deployment is created at version 1, with its own workspace node" trace $CLI bundle deploy -trace $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-redeploy-after-destroy/default/state/resources.deployment.json" | jq '{object_type,path}' +trace MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-redeploy-after-destroy/default/state/resources.deployment.json" | jq '{object_type,path}' trace print_requests.py //api/2.0/bundle --sort --get diff --git a/acceptance/bundle/dms/test.toml b/acceptance/bundle/dms/test.toml index 40cc45a44a5..1e36331a16f 100644 --- a/acceptance/bundle/dms/test.toml +++ b/acceptance/bundle/dms/test.toml @@ -10,11 +10,3 @@ RecordRequests = true Ignore = [ '.databricks', ] - -# These tests pass workspace paths to $CLI. On Windows, Git Bash rewrites a -# leading-'/' argument into a Windows path before the CLI sees it, so -# `workspace get-status /Workspace/...` looks up C:/Program Files/Git/Workspace/... -# and 404s. Quoting the argument does not help - the conversion happens in the -# Windows binary's argument processing. -[Env] -MSYS_NO_PATHCONV = "1" From 30adb997c90de6c5831773909387190372b0f663 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 29 Jul 2026 11:25:27 +0000 Subject: [PATCH 019/125] bundle: drop the locks in the operation queue's close close runs on one goroutine after every apply worker has returned, so nothing else touches the queue by then and the closed flag needs no protection. The wg.Wait orders the upload workers' writes to err before it is read, so that read needs none either. Also tightens the coalescing test to count uploads for the resource instead of only checking the payload of the last one. It asserted the merged content but would have passed if the operations had been uploaded twice, which is the thing coalescing exists to prevent. Co-authored-by: Isaac --- bundle/direct/opqueue.go | 14 ++++++-------- bundle/direct/opqueue_test.go | 10 ++++++++++ 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/bundle/direct/opqueue.go b/bundle/direct/opqueue.go index 9a1a1cd918e..8d1bea2ff72 100644 --- a/bundle/direct/opqueue.go +++ b/bundle/direct/opqueue.go @@ -135,23 +135,21 @@ func (q *operationQueue) record(ctx context.Context, resourceKey string, action // record must have returned first: record on a closed queue panics. Calling close // more than once is safe, so callers can defer it and still check the error at a // specific point. +// +// Unlike the other methods this one takes no lock. It runs on one goroutine after +// every apply worker has returned, so nothing else touches the queue by then, and +// the wg.Wait below orders the workers' writes to err before it is read. func (q *operationQueue) close() error { if q == nil { return nil } - q.mu.Lock() - closed := q.closed - q.closed = true - q.mu.Unlock() - - if !closed { + if !q.closed { + q.closed = true close(q.queue) q.wg.Wait() } - q.mu.Lock() - defer q.mu.Unlock() return q.err } diff --git a/bundle/direct/opqueue_test.go b/bundle/direct/opqueue_test.go index 356a7cfb319..69492c2e63e 100644 --- a/bundle/direct/opqueue_test.go +++ b/bundle/direct/opqueue_test.go @@ -130,6 +130,16 @@ func TestOperationQueueCoalescingKeepsLatestOperation(t *testing.T) { close(f.block) require.NoError(t, q.close()) + // One upload, not two: the second operation replaced the first while every + // worker was busy, so the extra CreateOperation round trip never happens. + var uploadsForFoo int + for _, u := range f.recorded() { + if strings.HasPrefix(u, "resources.jobs.foo=") { + uploadsForFoo++ + } + } + assert.Equal(t, 1, uploadsForFoo, "the two operations should coalesce into one upload") + // Everything comes from the newest operation: it carries the resource's full // state, and the ID it learned after the create. assert.Contains(t, f.recorded(), `resources.jobs.foo={"state":{"name":"updated"}}`) From 8f56ef6878728e88ffbc6da8d58f61130a0aea54 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 29 Jul 2026 11:33:25 +0000 Subject: [PATCH 020/125] bundle: record operation state without redacting it Drops the RedactSensitiveFields call, leaving a TODO: fields marked bundle:"sensitive" now reach DMS in plaintext, and the read path writes them back into the local state file unredacted. This has to be restored before the feature ships to users. Also documents why take leaves the key in queuedOrUploading when it hands an operation to a worker, and covers the case the comment describes: recording while that key's upload is in flight. The operation cannot join the in-flight request, so it is uploaded next by the same worker rather than being dropped or picked up concurrently by a second one. Co-authored-by: Isaac --- bundle/direct/opqueue.go | 5 +++++ bundle/direct/opqueue_test.go | 27 +++++++++++++++++++++++++++ bundle/direct/oprecorder.go | 10 ++++------ bundle/direct/oprecorder_test.go | 9 ++++----- 4 files changed, 40 insertions(+), 11 deletions(-) diff --git a/bundle/direct/opqueue.go b/bundle/direct/opqueue.go index 8d1bea2ff72..81163915b96 100644 --- a/bundle/direct/opqueue.go +++ b/bundle/direct/opqueue.go @@ -189,6 +189,11 @@ func (q *operationQueue) take(resourceKey string) (recordedOperation, bool) { return recordedOperation{}, false } + // The key stays in queuedOrUploading: the worker keeps coming back here until + // nothing is pending for it, so anything recorded while this operation uploads + // is still picked up. The mark is only cleared above, once there is nothing + // left - which is also what stops a second worker from taking the key and + // uploading the same resource concurrently. delete(q.pending, resourceKey) return op, true } diff --git a/bundle/direct/opqueue_test.go b/bundle/direct/opqueue_test.go index 69492c2e63e..b28d75dbf43 100644 --- a/bundle/direct/opqueue_test.go +++ b/bundle/direct/opqueue_test.go @@ -149,6 +149,33 @@ func TestOperationQueueCoalescingKeepsLatestOperation(t *testing.T) { f.actionFor("resources.jobs.foo")) } +func TestOperationQueueRecordDuringUploadIsStillUploaded(t *testing.T) { + // Record while the key's own upload is in flight: the key is off the queue but + // still marked, so record does not queue it again. The worker that holds the key + // has to come back for it, or the operation would be silently dropped. + f := &fakeUploader{block: make(chan struct{}), started: make(chan string, 1)} + q := newOperationQueue(t.Context(), f) + + require.NoError(t, q.record(t.Context(), "resources.jobs.foo", deployplan.Create, "", map[string]string{"name": "v1"}, nil)) + assert.Equal(t, "resources.jobs.foo", <-f.started) + + // The worker has taken the key off the queue and is uploading v1 right now. + require.NoError(t, q.record(t.Context(), "resources.jobs.foo", deployplan.Create, "id-1", map[string]string{"name": "v2"}, nil)) + + close(f.block) + require.NoError(t, q.close()) + + // Two uploads, in order: an in-flight request cannot be recalled, so v2 goes up + // after v1 rather than replacing it. The service ends up with the newest state. + assert.Equal(t, []string{ + `resources.jobs.foo={"state":{"name":"v1"}}`, + `resources.jobs.foo={"state":{"name":"v2"}}`, + }, f.recorded()) + assert.Equal(t, "id-1", f.resourceIDFor("resources.jobs.foo")) + assert.Empty(t, q.pending) + assert.Empty(t, q.queuedOrUploading) +} + func TestOperationQueueReturnsUploadError(t *testing.T) { uploadErr := errors.New("boom") f := &fakeUploader{err: uploadErr} diff --git a/bundle/direct/oprecorder.go b/bundle/direct/oprecorder.go index 2401e34fcf8..4f932f09c3a 100644 --- a/bundle/direct/oprecorder.go +++ b/bundle/direct/oprecorder.go @@ -8,8 +8,6 @@ import ( "github.com/databricks/cli/bundle/deployplan" "github.com/databricks/cli/bundle/direct/dstate" - "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/structs/structwalk" "github.com/databricks/databricks-sdk-go/service/bundledeployments" ) @@ -47,11 +45,11 @@ func newRecordedOperation(action deployplan.ActionType, resourceID string, state // Operation.State carries the serialized state, which DMS serves back as // resource state. Unset for delete: the resource is gone. // - // Redact secrets, like dstate.SaveState does for the local state file: - // otherwise we leak them to the service and the read path writes them back - // into a local state file in plaintext. + // TODO(DMS): fields marked bundle:"sensitive" are recorded in plaintext here, + // unlike dstate.SaveState which redacts them before writing the local state + // file. Redact them before this ships to users. if state != nil { - config, err := structwalk.RedactSensitiveFields(state, dyn.SensitiveValueRedacted) + config, err := json.Marshal(state) if err != nil { return recordedOperation{}, fmt.Errorf("serializing state: %w", err) } diff --git a/bundle/direct/oprecorder_test.go b/bundle/direct/oprecorder_test.go index f7262c04632..a948b609441 100644 --- a/bundle/direct/oprecorder_test.go +++ b/bundle/direct/oprecorder_test.go @@ -6,7 +6,6 @@ import ( "testing" "github.com/databricks/cli/bundle/deployplan" - "github.com/databricks/cli/libs/dyn" "github.com/databricks/databricks-sdk-go/service/bundledeployments" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -65,7 +64,7 @@ func TestOperationRecorderDeleteHasNoState(t *testing.T) { assert.Nil(t, f.requests[0].Operation.State) } -func TestNewRecordedOperationRedactsSensitiveFields(t *testing.T) { +func TestNewRecordedOperationDoesNotRedactSensitiveFields(t *testing.T) { state := struct { Name string `json:"name"` Token string `json:"token" bundle:"sensitive"` @@ -74,10 +73,10 @@ func TestNewRecordedOperationRedactsSensitiveFields(t *testing.T) { op, err := newRecordedOperation(deployplan.Create, "job-123", state, nil) require.NoError(t, err) - // Sensitive fields are redacted before leaving the CLI, matching what - // dstate.SaveState writes to the local state file. + // Recorded as-is for now, unlike dstate.SaveState which redacts before writing + // the local state file. See the TODO in newRecordedOperation. assert.JSONEq(t, - `{"state":{"name":"foo","token":"`+dyn.SensitiveValueRedacted+`"}}`, + `{"state":{"name":"foo","token":"super-secret"}}`, string(op.state)) } From 329886f2b2e62a821cead6ad54aaaebefee2381f Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 29 Jul 2026 11:38:48 +0000 Subject: [PATCH 021/125] bundle: drop the redaction TODO from the operation recorder Co-authored-by: Isaac --- bundle/direct/oprecorder.go | 4 ---- bundle/direct/oprecorder_test.go | 4 ++-- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/bundle/direct/oprecorder.go b/bundle/direct/oprecorder.go index 4f932f09c3a..d0d2694d358 100644 --- a/bundle/direct/oprecorder.go +++ b/bundle/direct/oprecorder.go @@ -44,10 +44,6 @@ func newRecordedOperation(action deployplan.ActionType, resourceID string, state // Operation.State carries the serialized state, which DMS serves back as // resource state. Unset for delete: the resource is gone. - // - // TODO(DMS): fields marked bundle:"sensitive" are recorded in plaintext here, - // unlike dstate.SaveState which redacts them before writing the local state - // file. Redact them before this ships to users. if state != nil { config, err := json.Marshal(state) if err != nil { diff --git a/bundle/direct/oprecorder_test.go b/bundle/direct/oprecorder_test.go index a948b609441..1b68c3a626b 100644 --- a/bundle/direct/oprecorder_test.go +++ b/bundle/direct/oprecorder_test.go @@ -73,8 +73,8 @@ func TestNewRecordedOperationDoesNotRedactSensitiveFields(t *testing.T) { op, err := newRecordedOperation(deployplan.Create, "job-123", state, nil) require.NoError(t, err) - // Recorded as-is for now, unlike dstate.SaveState which redacts before writing - // the local state file. See the TODO in newRecordedOperation. + // Recorded as-is, unlike dstate.SaveState which redacts before writing the + // local state file. assert.JSONEq(t, `{"state":{"name":"foo","token":"super-secret"}}`, string(op.state)) From 2818d11bcab9cf6b958f67ee736c91e50698771d Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 29 Jul 2026 12:00:00 +0000 Subject: [PATCH 022/125] bundle: revert the schema annotation and tighten CompleteVersion's guard Drops the record_deployment_history annotation change (and the generated schema that followed from it), leaving both files as they are on main. CompleteVersion now keys its no-op on versionNum rather than on the heartbeat handle. Both are set together by CreateVersion, so the behaviour is the same - a deploy that was cancelled, or whose CreateVersion failed, does not complete a version that was never created - but the check now names the thing it is actually guarding. Callers defer CompleteVersion unconditionally, so this is the only thing standing between a failed CreateVersion and a CompleteVersion call against a nonexistent version. Co-authored-by: Isaac --- bundle/internal/schema/annotations.yml | 2 -- bundle/schema/jsonschema.json | 2 +- libs/dms/recorder.go | 7 +++++-- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/bundle/internal/schema/annotations.yml b/bundle/internal/schema/annotations.yml index 100d33356fd..10832fe04a6 100644 --- a/bundle/internal/schema/annotations.yml +++ b/bundle/internal/schema/annotations.yml @@ -178,8 +178,6 @@ experimental: "record_deployment_history": "description": |- Whether to record deployment history using the deployment metadata service (DMS), which tracks what changed across deployments. - - Only supported for a bundle with no deployed resources yet. "scripts": "description": |- The commands to run. diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index c52e96d5dc9..4c78bd7c384 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -3016,7 +3016,7 @@ "$ref": "#/$defs/bool" }, "record_deployment_history": { - "description": "Whether to record deployment history using the deployment metadata service (DMS), which tracks what changed across deployments.\n\nOnly supported for a bundle with no deployed resources yet.", + "description": "Whether to record deployment history using the deployment metadata service (DMS), which tracks what changed across deployments.", "$ref": "#/$defs/bool" }, "scripts": { diff --git a/libs/dms/recorder.go b/libs/dms/recorder.go index 83f180ba3be..e285a80e4a2 100644 --- a/libs/dms/recorder.go +++ b/libs/dms/recorder.go @@ -102,9 +102,12 @@ func (r *Recorder) CreateVersion(ctx context.Context) error { } // CompleteVersion finalizes the version created by CreateVersion. A nil -// Recorder, or one whose CreateVersion never ran, is a no-op. +// Recorder, or one whose CreateVersion never ran or failed, is a no-op: there is +// no version on the server to complete. Callers defer it unconditionally, so this +// is the check that keeps a cancelled or failed deploy from completing a version +// that was never created. func (r *Recorder) CompleteVersion(ctx context.Context, success bool) error { - if r == nil || r.stopHeartbeat == nil { + if r == nil || r.versionNum == 0 { return nil } From b84ae7c631453e3cba92d83f729b44fab0588900 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 29 Jul 2026 12:24:31 +0000 Subject: [PATCH 023/125] bundle: gate record_deployment_history off again Restores validate.ValidateRecordDeploymentHistory and the hidden DATABRICKS_BUNDLE_FORCE_ALLOW_RECORD_DEPLOYMENT_HISTORY escape hatch, so setting experimental.record_deployment_history is an error unless that variable is set. The service side is not ready for users: DMS is only deployed to dev and staging, and reading state back needs the workspace APIs to expose the deployment's tree node, which is still behind a flag. With the flag unreachable, the state feature flag added earlier is not needed yet, so dstate is back to the resource-count check in Open. The deployment_history feature, hasFeature/setFeature, the version bump on write and the WAL carry-over all come back with the state upgrade in a follow-up. The dms acceptance tests force allow the flag, and bundle/dms/not-supported covers the error users see. Operation requests in bundle/dms/depends-on print multi-line now: the state envelope nests two levels, which --oneline made unreadable. Co-authored-by: Isaac --- acceptance/bundle/dms/depends-on/output.txt | 67 ++++++++++- acceptance/bundle/dms/depends-on/script | 2 +- .../bundle/dms/existing-state/output.txt | 4 +- .../bundle/dms/not-supported/databricks.yml | 10 ++ .../bundle/dms/not-supported/out.test.toml | 3 + .../bundle/dms/not-supported/output.txt | 24 ++++ acceptance/bundle/dms/not-supported/script | 5 + acceptance/bundle/dms/not-supported/test.toml | 6 + acceptance/bundle/dms/record/output.txt | 9 -- acceptance/bundle/dms/record/script | 3 - acceptance/bundle/dms/test.toml | 6 + .../validate_record_deployment_history.go | 47 ++++++++ ...validate_record_deployment_history_test.go | 55 +++++++++ bundle/direct/dstate/migrate.go | 35 +++--- bundle/direct/dstate/state.go | 110 ++++++------------ bundle/direct/dstate/state_test.go | 21 +--- .../force_allow_record_deployment_history.go | 19 +++ bundle/phases/initialize.go | 5 + cmd/bundle/utils/process.go | 1 - 19 files changed, 296 insertions(+), 136 deletions(-) create mode 100644 acceptance/bundle/dms/not-supported/databricks.yml create mode 100644 acceptance/bundle/dms/not-supported/out.test.toml create mode 100644 acceptance/bundle/dms/not-supported/output.txt create mode 100644 acceptance/bundle/dms/not-supported/script create mode 100644 acceptance/bundle/dms/not-supported/test.toml create mode 100644 bundle/config/validate/validate_record_deployment_history.go create mode 100644 bundle/config/validate/validate_record_deployment_history_test.go create mode 100644 bundle/env/force_allow_record_deployment_history.go diff --git a/acceptance/bundle/dms/depends-on/output.txt b/acceptance/bundle/dms/depends-on/output.txt index 2bb8d06b9dc..67fd0165669 100644 --- a/acceptance/bundle/dms/depends-on/output.txt +++ b/acceptance/bundle/dms/depends-on/output.txt @@ -6,9 +6,70 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> print_requests.py //versions/1/operations --sort --oneline -{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations", "q": {"resource_key": "jobs.child"}, "body": {"action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_id": "[NUMID]", "resource_key": "jobs.child", "state": {"state": {"deployment": {"kind": "BUNDLE", "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-depends-on/default/state/metadata.json"}, "description": "depends on [NUMID]", "edit_mode": "UI_LOCKED", "format": "MULTI_TASK", "max_concurrent_runs": 1, "name": "child", "queue": {"enabled": true}}, "depends_on": [{"node": "resources.jobs.parent", "label": "${resources.jobs.parent.id}"}]}, "status": "OPERATION_STATUS_SUCCEEDED"}} -{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations", "q": {"resource_key": "jobs.parent"}, "body": {"action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_id": "[NUMID]", "resource_key": "jobs.parent", "state": {"state": {"deployment": {"kind": "BUNDLE", "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-depends-on/default/state/metadata.json"}, "edit_mode": "UI_LOCKED", "format": "MULTI_TASK", "max_concurrent_runs": 1, "name": "parent", "queue": {"enabled": true}}}, "status": "OPERATION_STATUS_SUCCEEDED"}} +>>> print_requests.py //versions/1/operations --sort +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations", + "q": { + "resource_key": "jobs.child" + }, + "body": { + "action_type": "OPERATION_ACTION_TYPE_CREATE", + "resource_id": "[NUMID]", + "resource_key": "jobs.child", + "state": { + "state": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-depends-on/default/state/metadata.json" + }, + "description": "depends on [NUMID]", + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "child", + "queue": { + "enabled": true + } + }, + "depends_on": [ + { + "node": "resources.jobs.parent", + "label": "${resources.jobs.parent.id}" + } + ] + }, + "status": "OPERATION_STATUS_SUCCEEDED" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations", + "q": { + "resource_key": "jobs.parent" + }, + "body": { + "action_type": "OPERATION_ACTION_TYPE_CREATE", + "resource_id": "[NUMID]", + "resource_key": "jobs.parent", + "state": { + "state": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-depends-on/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "parent", + "queue": { + "enabled": true + } + } + }, + "status": "OPERATION_STATUS_SUCCEEDED" + } +} === Wipe the local state, then destroy: depends_on comes back from DMS, so the child is still deleted before the parent it references >>> [CLI] bundle destroy --auto-approve diff --git a/acceptance/bundle/dms/depends-on/script b/acceptance/bundle/dms/depends-on/script index 905d022619c..be1b9d622f8 100644 --- a/acceptance/bundle/dms/depends-on/script +++ b/acceptance/bundle/dms/depends-on/script @@ -1,6 +1,6 @@ title "Deploy a job that references another: depends_on is recorded alongside the config, since the reference is resolved to a literal in the config itself" trace $CLI bundle deploy -trace print_requests.py //versions/1/operations --sort --oneline +trace print_requests.py //versions/1/operations --sort title "Wipe the local state, then destroy: depends_on comes back from DMS, so the child is still deleted before the parent it references" rm -rf .databricks diff --git a/acceptance/bundle/dms/existing-state/output.txt b/acceptance/bundle/dms/existing-state/output.txt index 229b3bea03f..a793cc0fae1 100644 --- a/acceptance/bundle/dms/existing-state/output.txt +++ b/acceptance/bundle/dms/existing-state/output.txt @@ -12,7 +12,7 @@ Deployment complete! >>> update_file.py databricks.yml record_deployment_history: false record_deployment_history: true >>> musterr [CLI] bundle deploy -Error: target "default" was deployed without experimental.record_deployment_history and cannot be migrated to it: [TEST_TMP_DIR]/.databricks/bundle/default/resources.json tracks resources this deployment recorded before the feature was enabled. Use a new target, destroy this one and deploy it again, or unset experimental.record_deployment_history +Error: cannot record deployment history for a bundle that already has deployed resources tracked in [TEST_TMP_DIR]/.databricks/bundle/default/resources.json: only new deployments can be recorded. Remove experimental.record_deployment_history, or destroy the bundle and deploy it again === No deployment was created in DMS @@ -20,7 +20,7 @@ Error: target "default" was deployed without experimental.record_deployment_hist === Still an error after wiping the local cache: deploy pulls the state file back from the workspace, so the resources are still tracked >>> musterr [CLI] bundle deploy -Error: target "default" was deployed without experimental.record_deployment_history and cannot be migrated to it: [TEST_TMP_DIR]/.databricks/bundle/default/resources.json tracks resources this deployment recorded before the feature was enabled. Use a new target, destroy this one and deploy it again, or unset experimental.record_deployment_history +Error: cannot record deployment history for a bundle that already has deployed resources tracked in [TEST_TMP_DIR]/.databricks/bundle/default/resources.json: only new deployments can be recorded. Remove experimental.record_deployment_history, or destroy the bundle and deploy it again >>> print_requests.py //api/2.0/bundle --sort --oneline diff --git a/acceptance/bundle/dms/not-supported/databricks.yml b/acceptance/bundle/dms/not-supported/databricks.yml new file mode 100644 index 00000000000..c6edca465b6 --- /dev/null +++ b/acceptance/bundle/dms/not-supported/databricks.yml @@ -0,0 +1,10 @@ +bundle: + name: dms-not-supported + +experimental: + record_deployment_history: true + +resources: + jobs: + one: + name: one diff --git a/acceptance/bundle/dms/not-supported/out.test.toml b/acceptance/bundle/dms/not-supported/out.test.toml new file mode 100644 index 00000000000..e90b6d5d1ba --- /dev/null +++ b/acceptance/bundle/dms/not-supported/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/dms/not-supported/output.txt b/acceptance/bundle/dms/not-supported/output.txt new file mode 100644 index 00000000000..b665d545d12 --- /dev/null +++ b/acceptance/bundle/dms/not-supported/output.txt @@ -0,0 +1,24 @@ + +=== record_deployment_history is rejected: the service side is not ready for users yet +>>> musterr [CLI] bundle validate +Error: experimental.record_deployment_history is not supported yet + at experimental.record_deployment_history + in databricks.yml:5:30 + +Name: dms-not-supported +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/dms-not-supported/default + +Found 1 error + +=== The hidden force-allow variable permits it +>>> DATABRICKS_BUNDLE_FORCE_ALLOW_RECORD_DEPLOYMENT_HISTORY=1 [CLI] bundle validate +Name: dms-not-supported +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/dms-not-supported/default + +Validation OK! diff --git a/acceptance/bundle/dms/not-supported/script b/acceptance/bundle/dms/not-supported/script new file mode 100644 index 00000000000..10a0d995d87 --- /dev/null +++ b/acceptance/bundle/dms/not-supported/script @@ -0,0 +1,5 @@ +title "record_deployment_history is rejected: the service side is not ready for users yet" +trace musterr $CLI bundle validate + +title "The hidden force-allow variable permits it" +trace DATABRICKS_BUNDLE_FORCE_ALLOW_RECORD_DEPLOYMENT_HISTORY=1 $CLI bundle validate diff --git a/acceptance/bundle/dms/not-supported/test.toml b/acceptance/bundle/dms/not-supported/test.toml new file mode 100644 index 00000000000..4617ff88f20 --- /dev/null +++ b/acceptance/bundle/dms/not-supported/test.toml @@ -0,0 +1,6 @@ +# Unset the force-allow variable inherited from the parent: this test asserts the +# error users see. +Env.DATABRICKS_BUNDLE_FORCE_ALLOW_RECORD_DEPLOYMENT_HISTORY = "" + +# This test only checks validation output; no DMS request is made either way. +RecordRequests = false diff --git a/acceptance/bundle/dms/record/output.txt b/acceptance/bundle/dms/record/output.txt index c2b3333e587..33cf719ddfd 100644 --- a/acceptance/bundle/dms/record/output.txt +++ b/acceptance/bundle/dms/record/output.txt @@ -73,15 +73,6 @@ Deployment complete! >>> jq has("deployment_id") .databricks/bundle/default/resources.json false -=== The state records the feature flag, at the version that makes a CLI without it refuse the state rather than deploy against resources it cannot see ->>> jq {state_version, features} .databricks/bundle/default/resources.json -{ - "state_version": 3, - "features": { - "deployment_history": {} - } -} - === Redeploy after deleting the local cache: the deployment ID is resolved from that node, the same deployment is reused, and the version increments (no new CreateDeployment) >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-record/default/files... diff --git a/acceptance/bundle/dms/record/script b/acceptance/bundle/dms/record/script index 0ad4dd96d98..32ee3ec972f 100644 --- a/acceptance/bundle/dms/record/script +++ b/acceptance/bundle/dms/record/script @@ -10,9 +10,6 @@ title "The deployment ID is the ID of the workspace node the service registered trace MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-record/default/state/resources.deployment.json" | jq '{object_type,path}' trace jq 'has("deployment_id")' .databricks/bundle/default/resources.json -title "The state records the feature flag, at the version that makes a CLI without it refuse the state rather than deploy against resources it cannot see" -trace jq '{state_version, features}' .databricks/bundle/default/resources.json - title "Redeploy after deleting the local cache: the deployment ID is resolved from that node, the same deployment is reused, and the version increments (no new CreateDeployment)" rm -rf .databricks trace $CLI bundle deploy diff --git a/acceptance/bundle/dms/test.toml b/acceptance/bundle/dms/test.toml index 1e36331a16f..7c21473f724 100644 --- a/acceptance/bundle/dms/test.toml +++ b/acceptance/bundle/dms/test.toml @@ -10,3 +10,9 @@ RecordRequests = true Ignore = [ '.databricks', ] + +# experimental.record_deployment_history is rejected outright (see +# validate.ValidateRecordDeploymentHistory). These tests exercise the feature itself, +# so they force allow it the same way DMS development does. bundle/dms/not-supported +# covers the rejection. +Env.DATABRICKS_BUNDLE_FORCE_ALLOW_RECORD_DEPLOYMENT_HISTORY = "1" diff --git a/bundle/config/validate/validate_record_deployment_history.go b/bundle/config/validate/validate_record_deployment_history.go new file mode 100644 index 00000000000..4f0137fae0f --- /dev/null +++ b/bundle/config/validate/validate_record_deployment_history.go @@ -0,0 +1,47 @@ +package validate + +import ( + "context" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/env" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dyn" +) + +const recordDeploymentHistoryPath = "experimental.record_deployment_history" + +func ValidateRecordDeploymentHistory() bundle.ReadOnlyMutator { + return &validateRecordDeploymentHistory{} +} + +type validateRecordDeploymentHistory struct{ bundle.RO } + +func (v *validateRecordDeploymentHistory) Name() string { + return "validate:validate_record_deployment_history" +} + +// Apply rejects experimental.record_deployment_history. +// +// Recording deployment history is implemented end to end, but the service side is not +// ready for users: the deployment metadata service is only deployed to dev and staging, +// and reading state back needs the workspace APIs to expose the deployment's tree node, +// which is still behind a flag. Enabling this today also makes DMS the source of truth +// for resource state, so a bundle that turns it on cannot be turned back. +// +// DATABRICKS_BUNDLE_FORCE_ALLOW_RECORD_DEPLOYMENT_HISTORY force allows it for the CLI's +// own tests and for DMS development. +func (v *validateRecordDeploymentHistory) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { + if b.Config.Experimental == nil || !b.Config.Experimental.RecordDeploymentHistory { + return nil + } + if env.ForceAllowRecordDeploymentHistory(ctx) { + return nil + } + return diag.Diagnostics{{ + Severity: diag.Error, + Summary: recordDeploymentHistoryPath + " is not supported yet", + Paths: []dyn.Path{dyn.MustPathFromString(recordDeploymentHistoryPath)}, + Locations: b.Config.GetLocations(recordDeploymentHistoryPath), + }} +} diff --git a/bundle/config/validate/validate_record_deployment_history_test.go b/bundle/config/validate/validate_record_deployment_history_test.go new file mode 100644 index 00000000000..1bb172766f2 --- /dev/null +++ b/bundle/config/validate/validate_record_deployment_history_test.go @@ -0,0 +1,55 @@ +package validate + +import ( + "testing" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + bundleenv "github.com/databricks/cli/bundle/env" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/env" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestValidateRecordDeploymentHistory(t *testing.T) { + tests := []struct { + name string + enabled bool + forceAllow string + wantError bool + }{ + {name: "flag unset", enabled: false, wantError: false}, + {name: "flag set", enabled: true, wantError: true}, + {name: "flag set with force allow", enabled: true, forceAllow: "1", wantError: false}, + {name: "flag set with empty force allow", enabled: true, forceAllow: "", wantError: true}, + {name: "flag unset with force allow", enabled: false, forceAllow: "1", wantError: false}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + b := &bundle.Bundle{ + Config: config.Root{ + Experimental: &config.Experimental{RecordDeploymentHistory: tc.enabled}, + }, + } + + ctx := env.Set(t.Context(), bundleenv.ForceAllowRecordDeploymentHistoryVariable, tc.forceAllow) + diags := ValidateRecordDeploymentHistory().Apply(ctx, b) + + if !tc.wantError { + assert.Empty(t, diags) + return + } + require.Len(t, diags, 1) + assert.Equal(t, diag.Error, diags[0].Severity) + assert.Equal(t, "experimental.record_deployment_history is not supported yet", diags[0].Summary) + assert.Equal(t, recordDeploymentHistoryPath, diags[0].Paths[0].String()) + }) + } +} + +func TestValidateRecordDeploymentHistoryNoExperimentalBlock(t *testing.T) { + b := &bundle.Bundle{Config: config.Root{}} + assert.Empty(t, ValidateRecordDeploymentHistory().Apply(t.Context(), b)) +} diff --git a/bundle/direct/dstate/migrate.go b/bundle/direct/dstate/migrate.go index 288fdcf863f..e4d21a7054a 100644 --- a/bundle/direct/dstate/migrate.go +++ b/bundle/direct/dstate/migrate.go @@ -12,34 +12,25 @@ import ( "github.com/databricks/databricks-sdk-go/service/iam" ) -// knownFeatures lists the state feature flags this CLI implements. A state that -// records anything outside this set is refused by migrateState. -var knownFeatures = map[string]bool{ - FeatureDeploymentHistory: true, -} - // migrateState runs all necessary migrations on the database. // It is called after loading state from disk. func migrateState(db *Database) error { - // featureStateVersion states carry a feature list (see the featureStateVersion - // doc comment). A featureStateVersion state with no features is equivalent to - // currentStateVersion, so accept it and return without running the migrations - // below, leaving the on-disk version at featureStateVersion rather than flipping - // it down. Same for a state whose features this CLI implements. One that records - // a feature this CLI does not know depends on capabilities it lacks, so refuse it - // and tell the user to upgrade. + // featureStateVersion states carry a feature list this CLI does not yet write or + // understand (see the featureStateVersion doc comment). A featureStateVersion + // state with no features is equivalent to currentStateVersion, so accept it and + // return without running the migrations below, leaving the on-disk version at + // featureStateVersion rather than flipping it down. One that records any feature + // depends on capabilities this CLI lacks, so refuse it and tell the user to upgrade. if db.StateVersion == featureStateVersion { - unknown := make([]string, 0, len(db.Features)) - for name := range db.Features { - if !knownFeatures[name] { - unknown = append(unknown, name) - } - } - if len(unknown) == 0 { + if len(db.Features) == 0 { return nil } - slices.Sort(unknown) - return fmt.Errorf("the deployment state requires features this CLI does not support: %s; upgrade to the latest CLI version and see %s for more information", strings.Join(unknown, ", "), featuresDocURL) + features := make([]string, 0, len(db.Features)) + for name := range db.Features { + features = append(features, name) + } + slices.Sort(features) + return fmt.Errorf("the deployment state requires features this CLI does not support: %s; upgrade to the latest CLI version and see %s for more information", strings.Join(features, ", "), featuresDocURL) } if db.StateVersion == currentStateVersion { diff --git a/bundle/direct/dstate/state.go b/bundle/direct/dstate/state.go index b367e97001b..bad87a6a293 100644 --- a/bundle/direct/dstate/state.go +++ b/bundle/direct/dstate/state.go @@ -31,28 +31,22 @@ const ( maxWalEntrySize = 10 * 1024 * 1024 walSuffix = ".wal" - // featureStateVersion is the schema version written once a state records - // deployment state "feature flags" (see Header.Features). Reading such a state - // (see migrateState): - // - featureStateVersion with no features -> accept, leave the version as-is - // - featureStateVersion with known features -> accept - // - featureStateVersion with unknown features -> refuse, tell the user to upgrade + // featureStateVersion is the schema version a future CLI will write once it + // records deployment state "feature flags" (see Header.Features). This CLI does + // not write it and records no features; it exists now only so this CLI reads + // such states correctly (see migrateState): + // - featureStateVersion with no features -> accept and leave the version as-is + // - featureStateVersion with any feature -> refuse, tell the user to upgrade // // A featureStateVersion state with no features is equivalent to // currentStateVersion, but we deliberately do not flip the on-disk version down // to currentStateVersion: a state written at featureStateVersion stays at - // featureStateVersion. That way a release can start writing - // featureStateVersion + features without older CLIs either mishandling a feature - // they lack or rejecting a featureless state outright. featureStateVersion is - // always 3. + // featureStateVersion. This is forward-compat scaffolding so that a later release + // can start writing featureStateVersion + features without older CLIs (with this + // change) either mishandling a feature they lack or rejecting a featureless state + // outright. featureStateVersion is always 3. featureStateVersion = 3 - // FeatureDeploymentHistory marks a state whose resources live in the deployment - // metadata service rather than in the state file. A CLI that does not know this - // feature refuses the state instead of deploying against a resource set it - // cannot see - the file's resources are not authoritative. - FeatureDeploymentHistory = "deployment_history" - // supportedStateVersion is the highest schema version this CLI can read. It is // normally equal to currentStateVersion — the version this CLI reads is the // version it writes — and exceeds it only during a two-phase version bump like @@ -88,27 +82,13 @@ type Header struct { Serial int `json:"serial"` // Features maps each feature flag this state depends on to a (currently empty) - // value. A CLI that does not implement one of them refuses the state rather than - // deploying against it (see migrateState). It is a map so a future CLI can attach - // per-feature data without reshaping the state. Empty/omitted for states that use - // no features. + // value. This CLI writes no features; it only reads the field to detect a state + // that depends on features it lacks and refuse it (see migrateState). It is a + // map so a future CLI can attach per-feature data without reshaping the state. + // Empty/omitted for states that use no features. Features map[string]struct{} `json:"features,omitempty"` } -// hasFeature reports whether the state records the given feature flag. -func (h *Header) hasFeature(name string) bool { - _, ok := h.Features[name] - return ok -} - -// setFeature records a feature flag in the state. -func (h *Header) setFeature(name string) { - if h.Features == nil { - h.Features = make(map[string]struct{}) - } - h.Features[name] = struct{}{} -} - type Database struct { Header @@ -248,10 +228,6 @@ type DMSSource struct { // DeploymentID is resolved from the deployment's workspace node (see // dms.ResolveDeploymentID), and empty before the first recorded deploy. DeploymentID string - - // TargetName names the bundle target in the error Open returns for a state that - // predates the opt-in, since the feature is enabled per target. - TargetName string } // Open reads the deployment state from disk (and recovers the WAL when @@ -307,18 +283,18 @@ func (db *DeploymentState) Open(ctx context.Context, path string, withRecovery W } if dmsSource != nil { - // An existing state file has to be marked as DMS-owned before its resources - // can be read from DMS. Recording only starts on an empty state, so a state - // with resources and no feature flag predates the opt-in: DMS never saw those - // resources and is authoritative for the whole set once enabled (see - // readDMSState), so they would look absent and be created a second time. - // Migrating such a target is not supported yet. - if len(db.Data.State) > 0 && !db.Data.hasFeature(FeatureDeploymentHistory) { - return fmt.Errorf("target %q was deployed without experimental.record_deployment_history and cannot be migrated to it: %s tracks resources this deployment recorded before the feature was enabled. Use a new target, destroy this one and deploy it again, or unset experimental.record_deployment_history", dmsSource.TargetName, path) + // Only bundles that start out empty can be recorded. Once DMS owns a + // deployment it is authoritative for the whole resource set (see + // readDMSState), so pre-existing resources it never saw would look absent and + // get created a second time. + // + // TODO(DMS): allow this by upgrading the state in place, writing it at + // featureStateVersion with a feature flag plus a tombstone per resource so an + // older CLI refuses the state instead of deploying against resources it + // cannot see. + if dmsSource.DeploymentID == "" && len(db.Data.State) > 0 { + return fmt.Errorf("cannot record deployment history for a bundle that already has deployed resources tracked in %s: only new deployments can be recorded. Remove experimental.record_deployment_history, or destroy the bundle and deploy it again", path) } - // Mark the state as DMS-owned so a CLI without this feature refuses it rather - // than deploying against a resource set it cannot see. - db.Data.setFeature(FeatureDeploymentHistory) if dmsSource.DeploymentID != "" { if err := db.readDMSState(ctx, dmsSource); err != nil { return err @@ -335,7 +311,13 @@ func (db *DeploymentState) Open(ctx context.Context, path string, withRecovery W return fmt.Errorf("failed to open WAL file %s: %w", walPath, err) } db.walFile = walFile - return appendJSONLine(db.walFile, db.newWalHeader()) + walHead := Header{ + Lineage: db.GetOrInitLineage(), + Serial: db.Data.Serial + 1, + StateVersion: currentStateVersion, + CLIVersion: build.GetInfo().Version, + } + return appendJSONLine(db.walFile, walHead) } return nil @@ -422,16 +404,6 @@ func (db *DeploymentState) mergeWalIntoState(ctx context.Context) (bool, error) return false, fmt.Errorf("WAL serial (%d) is ahead of expected (%d), state may be corrupted", header.Serial, expectedSerial) } newSerial = header.Serial - - // Carry the WAL's features (and the version that goes with them) into the - // state being written, so recovering a WAL written by a DMS-recording deploy - // still produces a state marked as DMS-owned. - for name := range header.Features { - db.Data.setFeature(name) - } - if len(db.Data.Features) > 0 { - db.Data.StateVersion = featureStateVersion - } } else { var entry WALEntry if err := json.Unmarshal(line, &entry); err != nil { @@ -535,25 +507,13 @@ func (db *DeploymentState) UpgradeToWrite() error { } db.walFile = walFile - return appendJSONLine(db.walFile, db.newWalHeader()) -} - -// newWalHeader builds the header for a fresh WAL. Features carry over from the -// state being written, and a state that records any feature is written at -// featureStateVersion so a CLI that lacks the feature refuses it instead of -// deploying against it. The caller holds db.mu. -func (db *DeploymentState) newWalHeader() Header { - version := currentStateVersion - if len(db.Data.Features) > 0 { - version = featureStateVersion - } - return Header{ + walHead := Header{ Lineage: db.GetOrInitLineage(), Serial: db.Data.Serial + 1, - StateVersion: version, + StateVersion: currentStateVersion, CLIVersion: build.GetInfo().Version, - Features: db.Data.Features, } + return appendJSONLine(db.walFile, walHead) } func (db *DeploymentState) AssertOpenedForReadOrWrite() { diff --git a/bundle/direct/dstate/state_test.go b/bundle/direct/dstate/state_test.go index 35050575121..a9c90530514 100644 --- a/bundle/direct/dstate/state_test.go +++ b/bundle/direct/dstate/state_test.go @@ -154,16 +154,7 @@ func TestEmptyFeatureStateAcceptedWithoutFlippingVersion(t *testing.T) { require.NoError(t, migrateState(empty)) assert.Equal(t, featureStateVersion, empty.StateVersion, "v3 + no features keeps its on-disk version, not flipped to v2") - // v3 recording a feature this CLI implements is accepted. - known := &Database{Header: Header{ - StateVersion: featureStateVersion, - Features: map[string]struct{}{FeatureDeploymentHistory: {}}, - }} - require.NoError(t, migrateState(known)) - assert.Equal(t, featureStateVersion, known.StateVersion) - - // v3 recording a feature this CLI does not know is refused: its resources may - // live somewhere this CLI cannot see. + // v3 that records a feature is refused: this CLI does not understand features. withFeature := &Database{Header: Header{ StateVersion: featureStateVersion, Features: map[string]struct{}{"future_feature": {}}, @@ -174,16 +165,6 @@ func TestEmptyFeatureStateAcceptedWithoutFlippingVersion(t *testing.T) { assert.Contains(t, err.Error(), "future_feature") assert.Contains(t, err.Error(), "upgrade to the latest CLI version") assert.Contains(t, err.Error(), featuresDocURL) - - // Only the unknown feature is named, so the message tells the user what to do. - mixed := &Database{Header: Header{ - StateVersion: featureStateVersion, - Features: map[string]struct{}{FeatureDeploymentHistory: {}, "future_feature": {}}, - }} - err = migrateState(mixed) - require.Error(t, err) - assert.Contains(t, err.Error(), "future_feature") - assert.NotContains(t, err.Error(), FeatureDeploymentHistory) } func TestDeleteState(t *testing.T) { diff --git a/bundle/env/force_allow_record_deployment_history.go b/bundle/env/force_allow_record_deployment_history.go new file mode 100644 index 00000000000..297ccb6f6e3 --- /dev/null +++ b/bundle/env/force_allow_record_deployment_history.go @@ -0,0 +1,19 @@ +package env + +import "context" + +// ForceAllowRecordDeploymentHistoryVariable names the environment variable that force +// allows experimental.record_deployment_history. It is deliberately undocumented: the +// feature is complete but cannot be exposed to users yet (see +// validate.ValidateRecordDeploymentHistory for why), and this variable exists so the +// CLI's own tests and the developers working on DMS can exercise the code path meanwhile. +const ForceAllowRecordDeploymentHistoryVariable = "DATABRICKS_BUNDLE_FORCE_ALLOW_RECORD_DEPLOYMENT_HISTORY" + +// ForceAllowRecordDeploymentHistory reports whether the environment force allows +// experimental.record_deployment_history despite it being gated off. +func ForceAllowRecordDeploymentHistory(ctx context.Context) bool { + value, ok := get(ctx, []string{ + ForceAllowRecordDeploymentHistoryVariable, + }) + return ok && value != "" +} diff --git a/bundle/phases/initialize.go b/bundle/phases/initialize.go index bfa2af4124b..70ea74fa182 100644 --- a/bundle/phases/initialize.go +++ b/bundle/phases/initialize.go @@ -177,6 +177,11 @@ func Initialize(ctx context.Context, b *bundle.Bundle) { // They are set by the CLI to track the bundle deployment and must not be set by the user. validate.ValidateDeploymentFields(), + // Reads (typed): b.Config.Experimental.RecordDeploymentHistory + // Reads (env): DATABRICKS_BUNDLE_FORCE_ALLOW_RECORD_DEPLOYMENT_HISTORY (non-empty value force allows it) + // Rejects experimental.record_deployment_history: the feature is not usable yet. + validate.ValidateRecordDeploymentHistory(), + // Reads (dynamic): * (strings) (searches for ${resources.*} references) // Warns (TF engine) or errors (direct engine) when a cross-resource reference // points to a Terraform-only field with no DABs equivalent. diff --git a/cmd/bundle/utils/process.go b/cmd/bundle/utils/process.go index b000da60264..209dc874403 100644 --- a/cmd/bundle/utils/process.go +++ b/cmd/bundle/utils/process.go @@ -229,7 +229,6 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle dmsSource = &dstate.DMSSource{ Client: w.BundleDeployments, DeploymentID: deploymentID, - TargetName: b.Config.Bundle.Target, } } if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WithRecovery(true), dstate.WithWrite(false), dmsSource); err != nil { From daa69e28df3826732cde4e90edd787620b625749 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 29 Jul 2026 12:47:33 +0000 Subject: [PATCH 024/125] bundle: stop the deploy when an operation upload fails An upload failure was only reported at close, so a DMS outage let apply deploy every remaining resource and fail at the end. That leaves resources in the workspace that DMS has no record of, and since a completed version makes DMS the source of truth for resource state, the next deploy would create them again. record now returns the first upload error, which the apply worker turns into a failed node, so the deploy stops shortly after the failure instead of running to completion. It refuses new work only: operations already recorded still upload, because close drains them, so the records DMS ends up with match the resources that were actually applied. Resources already mid-apply also finish. Also repeats the one test whose bug depends on a scheduler interleaving rather than on a forced handshake, so a single run gets many chances to hit the bad ordering. The other tests pin their interleaving with the started/block channels, so repetition would not add coverage; the new error tests use a `done` channel to wait for an upload to have finished rather than merely started. Co-authored-by: Isaac --- bundle/direct/opqueue.go | 31 ++++++++- bundle/direct/opqueue_test.go | 121 ++++++++++++++++++++++++++-------- 2 files changed, 120 insertions(+), 32 deletions(-) diff --git a/bundle/direct/opqueue.go b/bundle/direct/opqueue.go index 81163915b96..ba189c13d2e 100644 --- a/bundle/direct/opqueue.go +++ b/bundle/direct/opqueue.go @@ -93,9 +93,10 @@ func newOperationQueue(ctx context.Context, uploader operationUploader) *operati return q } -// record serializes an operation and hands it to the upload workers. It makes no -// API call, so upload failures surface from close; an error here only means the -// applied resource could not be turned into a payload. +// record serializes an operation and hands it to the upload workers. The upload +// itself happens on a worker, so an error returned here is either a failure to +// turn the applied resource into a payload, or an earlier upload's error +// resurfaced (see below). // // Recording a resource that is still waiting replaces the waiting operation // outright, since the newer one carries the resource's full state. @@ -104,6 +105,21 @@ func (q *operationQueue) record(ctx context.Context, resourceKey string, action return nil } + // Report an earlier upload failure to the apply worker that is about to record + // the next resource, so the deploy stops instead of running to completion and + // only failing at close. That matters because a successfully completed version + // makes DMS the source of truth for resource state (see dstate.readDMSState): + // deploying everything while its records are missing leaves resources the next + // deploy would create a second time. + // + // This refuses new work only. Operations already recorded still upload - close + // drains them - so the records DMS does end up with match the resources that + // were actually applied. Resources already mid-apply also finish, so the deploy + // stops shortly after the first failure rather than exactly at it. + if err := q.firstErr(); err != nil { + return err + } + op, err := newRecordedOperation(action, resourceID, state, dependsOn) if err != nil { return err @@ -208,3 +224,12 @@ func (q *operationQueue) setErr(err error) { q.err = err } } + +// firstErr returns the first upload error, or nil if every upload so far +// succeeded. +func (q *operationQueue) firstErr() error { + q.mu.Lock() + defer q.mu.Unlock() + + return q.err +} diff --git a/bundle/direct/opqueue_test.go b/bundle/direct/opqueue_test.go index b28d75dbf43..a68560493ce 100644 --- a/bundle/direct/opqueue_test.go +++ b/bundle/direct/opqueue_test.go @@ -20,7 +20,10 @@ import ( type fakeUploader struct { block chan struct{} started chan string - err error + // done receives the resource key after the upload returns, for tests that need + // an upload to have completed rather than merely started. + done chan string + err error mu sync.Mutex uploads []string @@ -37,7 +40,6 @@ func (f *fakeUploader) upload(ctx context.Context, resourceKey string, op record } f.mu.Lock() - defer f.mu.Unlock() f.uploads = append(f.uploads, resourceKey+"="+string(op.state)) if f.actions == nil { f.actions = map[string]bundledeployments.OperationActionType{} @@ -45,6 +47,13 @@ func (f *fakeUploader) upload(ctx context.Context, resourceKey string, op record } f.actions[resourceKey] = op.action f.resourceIDs[resourceKey] = op.resourceID + f.mu.Unlock() + + // Sent outside the lock: a test that stops reading this channel would otherwise + // hold f.mu and deadlock every other worker. + if f.done != nil { + f.done <- resourceKey + } return f.err } @@ -189,6 +198,54 @@ func TestOperationQueueReturnsUploadError(t *testing.T) { assert.Contains(t, err.Error(), "resources.jobs.foo") } +func TestOperationQueueRecordFailsAfterUploadError(t *testing.T) { + // An upload failure stops the deploy at the next resource instead of surfacing + // only at close, so the apply workers do not keep creating resources that DMS + // has no record of. + uploadErr := errors.New("boom") + f := &fakeUploader{err: uploadErr, done: make(chan string, 1)} + q := newOperationQueue(t.Context(), f) + + // Wait for the failing upload to finish, so the error is stored before the next + // record rather than racing it. + require.NoError(t, q.record(t.Context(), "resources.jobs.foo", deployplan.Create, "id-1", map[string]string{"name": "v1"}, nil)) + assert.Equal(t, "resources.jobs.foo", <-f.done) + + // The next resource an apply worker tries to record is refused, with the upload + // error that caused it. + err := q.record(t.Context(), "resources.jobs.bar", deployplan.Create, "id-2", map[string]string{"name": "v1"}, nil) + require.Error(t, err) + assert.ErrorIs(t, err, uploadErr) + + // The refused resource was not queued, and close still reports the failure. + require.ErrorIs(t, q.close(), uploadErr) + assert.Equal(t, []string{`resources.jobs.foo={"state":{"name":"v1"}}`}, f.recorded()) + assert.Empty(t, q.pending) + assert.Empty(t, q.queuedOrUploading) +} + +func TestOperationQueueDrainsQueuedOperationsAfterUploadError(t *testing.T) { + // A failure refuses new work but does not discard work already recorded: the + // records DMS ends up with have to match the resources that were applied. + uploadErr := errors.New("boom") + f := &fakeUploader{err: uploadErr, block: make(chan struct{}), started: make(chan string, 1)} + q := newOperationQueue(t.Context(), f) + + // Every worker is parked mid-upload, so these stay queued. + for i := range operationUploadWorkers { + require.NoError(t, q.record(t.Context(), "resources.jobs.hold"+strconv.Itoa(i), deployplan.Create, "id-1", map[string]string{"name": "v1"}, nil)) + assert.Equal(t, "resources.jobs.hold"+strconv.Itoa(i), <-f.started) + } + require.NoError(t, q.record(t.Context(), "resources.jobs.queued", deployplan.Create, "id-2", map[string]string{"name": "v1"}, nil)) + + close(f.block) + require.ErrorIs(t, q.close(), uploadErr) + + // The queued operation was uploaded rather than dropped on the way out. + assert.Contains(t, f.recorded(), `resources.jobs.queued={"state":{"name":"v1"}}`) + assert.Len(t, f.recorded(), operationUploadWorkers+1) +} + func TestOperationQueueRecordRejectsUnsupportedAction(t *testing.T) { f := &fakeUploader{} q := newOperationQueue(t.Context(), f) @@ -254,40 +311,46 @@ func TestOperationQueueUploadsOneResourceAtATime(t *testing.T) { // case where a coalesced key can be handed to a second worker while the first // is still uploading it. The service keeps one state per key, so overlapping // uploads for a key could land out of order and leave a stale state behind. + // + // The interleaving that breaks this is scheduler-dependent, so one pass proves + // little: repeat it so a single run has many chances to hit the bad ordering. const ( + iterations = 200 workers = 10 perWorker = 5 distinctKeyMod = 12 ) - ctx := t.Context() - u := &serialUploader{live: map[string]bool{}, last: map[string]string{}} - q := newOperationQueue(ctx, u) - - // Collect record errors instead of asserting inside the goroutines: testify - // assertions may only run on the goroutine running the test function. - errs := make(chan error, workers*perWorker) - var wg sync.WaitGroup - for w := range workers { - wg.Go(func() { - for i := range perWorker { - key := "resources.jobs.job" + strconv.Itoa((w*perWorker+i)%distinctKeyMod) - errs <- q.record(ctx, key, deployplan.Update, "id-1", map[string]string{"name": strconv.Itoa(w)}, nil) - } - }) - } - wg.Wait() - close(errs) - for err := range errs { - require.NoError(t, err) - } - require.NoError(t, q.close()) + for range iterations { + ctx := t.Context() + u := &serialUploader{live: map[string]bool{}, last: map[string]string{}} + q := newOperationQueue(ctx, u) + + // Collect record errors instead of asserting inside the goroutines: testify + // assertions may only run on the goroutine running the test function. + errs := make(chan error, workers*perWorker) + var wg sync.WaitGroup + for w := range workers { + wg.Go(func() { + for i := range perWorker { + key := "resources.jobs.job" + strconv.Itoa((w*perWorker+i)%distinctKeyMod) + errs <- q.record(ctx, key, deployplan.Update, "id-1", map[string]string{"name": strconv.Itoa(w)}, nil) + } + }) + } + wg.Wait() + close(errs) + for err := range errs { + require.NoError(t, err) + } + require.NoError(t, q.close()) - assert.False(t, u.uneven, "two uploads overlapped for the same resource key") - // Every distinct key was recorded, and close drained all of them. - assert.Len(t, u.last, distinctKeyMod) - assert.Empty(t, q.pending) - assert.Empty(t, q.queuedOrUploading) + require.False(t, u.uneven, "two uploads overlapped for the same resource key") + // Every distinct key was recorded, and close drained all of them. + require.Len(t, u.last, distinctKeyMod) + require.Empty(t, q.pending) + require.Empty(t, q.queuedOrUploading) + } } func TestNilOperationQueueIsNoOp(t *testing.T) { From 2c2e209f3abc8cccdc37d91f354cf277bf642a28 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 29 Jul 2026 13:14:02 +0000 Subject: [PATCH 025/125] bundle: stop applying resources once an operation upload fails The previous commit made record return the upload error, but record runs after the resource has already been created or updated, so every node that started before the failure was noticed still modified the workspace. Apply now checks for a recorded failure before it touches anything, right after the dependency check, so a node that has not started yet is refused rather than applied. Resources already mid-apply still finish - the check cannot unwind those - but the deploy no longer runs to completion against a service that is rejecting its records. acceptance/bundle/dms/operation-upload-fails covers it. Which resources get refused depends on how far apply got before a background upload failed, so the per-resource errors go to a LOG file and requests are not recorded; the test asserts the deploy fails and reports the upload error. Also drops --sort from the dms tests that deploy zero or one resource: their request order is already deterministic, and the unsorted output reads in chronological order. Co-authored-by: Isaac --- .../bundle/dms/existing-state/output.txt | 10 ++--- acceptance/bundle/dms/existing-state/script | 8 ++-- acceptance/bundle/dms/no-resources/output.txt | 8 ++-- acceptance/bundle/dms/no-resources/script | 4 +- .../dms/operation-upload-fails/databricks.yml | 24 +++++++++++ .../dms/operation-upload-fails/out.test.toml | 3 ++ .../dms/operation-upload-fails/output.txt | 4 ++ .../bundle/dms/operation-upload-fails/script | 6 +++ .../dms/operation-upload-fails/test.toml | 12 ++++++ acceptance/bundle/dms/record/output.txt | 42 +++++++++---------- acceptance/bundle/dms/record/script | 6 +-- .../dms/redeploy-after-destroy/output.txt | 16 +++---- .../bundle/dms/redeploy-after-destroy/script | 4 +- bundle/direct/bundle_apply.go | 11 +++++ bundle/direct/opqueue.go | 6 ++- 15 files changed, 114 insertions(+), 50 deletions(-) create mode 100644 acceptance/bundle/dms/operation-upload-fails/databricks.yml create mode 100644 acceptance/bundle/dms/operation-upload-fails/out.test.toml create mode 100644 acceptance/bundle/dms/operation-upload-fails/output.txt create mode 100644 acceptance/bundle/dms/operation-upload-fails/script create mode 100644 acceptance/bundle/dms/operation-upload-fails/test.toml diff --git a/acceptance/bundle/dms/existing-state/output.txt b/acceptance/bundle/dms/existing-state/output.txt index a793cc0fae1..755981c414d 100644 --- a/acceptance/bundle/dms/existing-state/output.txt +++ b/acceptance/bundle/dms/existing-state/output.txt @@ -6,7 +6,7 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> print_requests.py //api/2.0/bundle --sort --oneline +>>> print_requests.py //api/2.0/bundle --oneline === Turning recording on afterwards is an error: those resources were never recorded, so treating DMS as authoritative would deploy them a second time >>> update_file.py databricks.yml record_deployment_history: false record_deployment_history: true @@ -16,14 +16,14 @@ Error: cannot record deployment history for a bundle that already has deployed r === No deployment was created in DMS ->>> print_requests.py //api/2.0/bundle --sort --oneline +>>> print_requests.py //api/2.0/bundle --oneline === Still an error after wiping the local cache: deploy pulls the state file back from the workspace, so the resources are still tracked >>> musterr [CLI] bundle deploy Error: cannot record deployment history for a bundle that already has deployed resources tracked in [TEST_TMP_DIR]/.databricks/bundle/default/resources.json: only new deployments can be recorded. Remove experimental.record_deployment_history, or destroy the bundle and deploy it again ->>> print_requests.py //api/2.0/bundle --sort --oneline +>>> print_requests.py //api/2.0/bundle --oneline === Destroy clears the tracked resources, so recording can be enabled afterwards >>> update_file.py databricks.yml record_deployment_history: true record_deployment_history: false @@ -45,8 +45,8 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> print_requests.py //api/2.0/bundle --sort --oneline +>>> print_requests.py //api/2.0/bundle --oneline {"method": "POST", "path": "/api/2.0/bundle/deployments", "body": {"initial_parent_path": "/Workspace/Users/[USERNAME]/.bundle/dms-existing-state/default/state", "target_name": "default"}} {"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": {"version_id": "1"}, "body": {"cli_version": "[CLI_VERSION]", "target_name": "default", "version_type": "VERSION_TYPE_DEPLOY"}} -{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/complete", "body": {"completion_reason": "VERSION_COMPLETE_SUCCESS"}} {"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations", "q": {"resource_key": "jobs.one"}, "body": {"action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_id": "[NUMID]", "resource_key": "jobs.one", "state": {"state": {"deployment": {"kind": "BUNDLE", "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-existing-state/default/state/metadata.json"}, "edit_mode": "UI_LOCKED", "format": "MULTI_TASK", "max_concurrent_runs": 1, "name": "one", "queue": {"enabled": true}}}, "status": "OPERATION_STATUS_SUCCEEDED"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/complete", "body": {"completion_reason": "VERSION_COMPLETE_SUCCESS"}} diff --git a/acceptance/bundle/dms/existing-state/script b/acceptance/bundle/dms/existing-state/script index ae9be95f701..9e448161259 100644 --- a/acceptance/bundle/dms/existing-state/script +++ b/acceptance/bundle/dms/existing-state/script @@ -1,22 +1,22 @@ title "Deploy without recording: the bundle gets ordinary direct-engine state, unknown to DMS" trace $CLI bundle deploy -trace print_requests.py //api/2.0/bundle --sort --oneline +trace print_requests.py //api/2.0/bundle --oneline title "Turning recording on afterwards is an error: those resources were never recorded, so treating DMS as authoritative would deploy them a second time" trace update_file.py databricks.yml "record_deployment_history: false" "record_deployment_history: true" trace musterr $CLI bundle deploy title "No deployment was created in DMS" -trace print_requests.py //api/2.0/bundle --sort --oneline +trace print_requests.py //api/2.0/bundle --oneline title "Still an error after wiping the local cache: deploy pulls the state file back from the workspace, so the resources are still tracked" rm -rf .databricks trace musterr $CLI bundle deploy -trace print_requests.py //api/2.0/bundle --sort --oneline +trace print_requests.py //api/2.0/bundle --oneline title "Destroy clears the tracked resources, so recording can be enabled afterwards" trace update_file.py databricks.yml "record_deployment_history: true" "record_deployment_history: false" trace $CLI bundle destroy --auto-approve trace update_file.py databricks.yml "record_deployment_history: false" "record_deployment_history: true" trace $CLI bundle deploy -trace print_requests.py //api/2.0/bundle --sort --oneline +trace print_requests.py //api/2.0/bundle --oneline diff --git a/acceptance/bundle/dms/no-resources/output.txt b/acceptance/bundle/dms/no-resources/output.txt index 86009c71b94..61e520c20a2 100644 --- a/acceptance/bundle/dms/no-resources/output.txt +++ b/acceptance/bundle/dms/no-resources/output.txt @@ -5,7 +5,7 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-no-resources/d Deploying resources... Deployment complete! ->>> print_requests.py //api/2.0/bundle --sort --get +>>> print_requests.py //api/2.0/bundle --get { "method": "POST", "path": "/api/2.0/bundle/deployments", @@ -46,14 +46,14 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-no-resources/d Deploying resources... Deployment complete! ->>> print_requests.py //api/2.0/bundle --sort --get +>>> print_requests.py //api/2.0/bundle --get { "method": "GET", - "path": "/api/2.0/bundle/deployments/[NUMID]" + "path": "/api/2.0/bundle/deployments/[NUMID]/resources" } { "method": "GET", - "path": "/api/2.0/bundle/deployments/[NUMID]/resources" + "path": "/api/2.0/bundle/deployments/[NUMID]" } { "method": "POST", diff --git a/acceptance/bundle/dms/no-resources/script b/acceptance/bundle/dms/no-resources/script index 9b14355bd27..f1b9ad5fa60 100644 --- a/acceptance/bundle/dms/no-resources/script +++ b/acceptance/bundle/dms/no-resources/script @@ -1,8 +1,8 @@ title "First deploy of a bundle with no resources: the deployment is created, and its workspace node identifies it even though no resource state was written" trace $CLI bundle deploy -trace print_requests.py //api/2.0/bundle --sort --get +trace print_requests.py //api/2.0/bundle --get trace MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-no-resources/default/state/resources.deployment.json" | jq '{object_type,path}' title "Redeploy: the deployment is resolved from that node, so no second deployment is created" trace $CLI bundle deploy -trace print_requests.py //api/2.0/bundle --sort --get +trace print_requests.py //api/2.0/bundle --get diff --git a/acceptance/bundle/dms/operation-upload-fails/databricks.yml b/acceptance/bundle/dms/operation-upload-fails/databricks.yml new file mode 100644 index 00000000000..1edbd7add88 --- /dev/null +++ b/acceptance/bundle/dms/operation-upload-fails/databricks.yml @@ -0,0 +1,24 @@ +bundle: + name: dms-operation-upload-fails + +experimental: + record_deployment_history: true + +resources: + jobs: + one: + name: one + two: + name: two + three: + name: three + four: + name: four + five: + name: five + six: + name: six + seven: + name: seven + eight: + name: eight diff --git a/acceptance/bundle/dms/operation-upload-fails/out.test.toml b/acceptance/bundle/dms/operation-upload-fails/out.test.toml new file mode 100644 index 00000000000..e90b6d5d1ba --- /dev/null +++ b/acceptance/bundle/dms/operation-upload-fails/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/dms/operation-upload-fails/output.txt b/acceptance/bundle/dms/operation-upload-fails/output.txt new file mode 100644 index 00000000000..2b273972417 --- /dev/null +++ b/acceptance/bundle/dms/operation-upload-fails/output.txt @@ -0,0 +1,4 @@ + +=== An operation upload failure fails the deploy instead of reporting only at the end +>>> grep -c ^Error: LOG.deploy +deploy reported errors diff --git a/acceptance/bundle/dms/operation-upload-fails/script b/acceptance/bundle/dms/operation-upload-fails/script new file mode 100644 index 00000000000..26c748d9e79 --- /dev/null +++ b/acceptance/bundle/dms/operation-upload-fails/script @@ -0,0 +1,6 @@ +title "An operation upload failure fails the deploy instead of reporting only at the end" +# Which resources get refused depends on how far apply got before a background +# upload failed, so the per-resource errors go to a LOG file rather than the diff. +errcode $CLI bundle deploy &> LOG.deploy +contains.py 'recording operation for' '!panic' < LOG.deploy > /dev/null +trace grep -c "^Error:" LOG.deploy > /dev/null && echo "deploy reported errors" diff --git a/acceptance/bundle/dms/operation-upload-fails/test.toml b/acceptance/bundle/dms/operation-upload-fails/test.toml new file mode 100644 index 00000000000..ce222ac9e87 --- /dev/null +++ b/acceptance/bundle/dms/operation-upload-fails/test.toml @@ -0,0 +1,12 @@ +# Which requests are made depends on how far apply got before a background upload +# failed, so recording them would make the output nondeterministic. +RecordRequests = false + +# The service rejects every recorded operation. Deploy must stop rather than +# create every remaining resource: a completed version makes DMS the source of +# truth for resource state, so resources it has no record of would be created a +# second time by the next deploy. +[[Server]] +Pattern = "POST /api/2.0/bundle/deployments/{deployment_id}/versions/{version_id}/operations" +Response.StatusCode = 500 +Response.Body = '''{"error_code": "INTERNAL_ERROR", "message": "Internal error"}''' diff --git a/acceptance/bundle/dms/record/output.txt b/acceptance/bundle/dms/record/output.txt index 33cf719ddfd..dcb6b3efa22 100644 --- a/acceptance/bundle/dms/record/output.txt +++ b/acceptance/bundle/dms/record/output.txt @@ -6,7 +6,7 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> print_requests.py //api/2.0/bundle --sort +>>> print_requests.py //api/2.0/bundle { "method": "POST", "path": "/api/2.0/bundle/deployments", @@ -27,13 +27,6 @@ Deployment complete! "version_type": "VERSION_TYPE_DEPLOY" } } -{ - "method": "POST", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/complete", - "body": { - "completion_reason": "VERSION_COMPLETE_SUCCESS" - } -} { "method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations", @@ -62,6 +55,13 @@ Deployment complete! "status": "OPERATION_STATUS_SUCCEEDED" } } +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/complete", + "body": { + "completion_reason": "VERSION_COMPLETE_SUCCESS" + } +} === The deployment ID is the ID of the workspace node the service registered under initial_parent_path; the CLI stores nothing locally >>> MSYS_NO_PATHCONV=1 [CLI] workspace get-status /Workspace/Users/[USERNAME]/.bundle/dms-record/default/state/resources.deployment.json @@ -80,7 +80,7 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> print_requests.py //api/2.0/bundle --sort +>>> print_requests.py //api/2.0/bundle { "method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", @@ -111,11 +111,7 @@ All files and directories at the following location will be deleted: /Workspace/ Deleting files... Destroy complete! ->>> print_requests.py //api/2.0/bundle --sort -{ - "method": "DELETE", - "path": "/api/2.0/bundle/deployments/[NUMID]" -} +>>> print_requests.py //api/2.0/bundle { "method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", @@ -128,13 +124,6 @@ Destroy complete! "version_type": "VERSION_TYPE_DESTROY" } } -{ - "method": "POST", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/3/complete", - "body": { - "completion_reason": "VERSION_COMPLETE_SUCCESS" - } -} { "method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/3/operations", @@ -147,3 +136,14 @@ Destroy complete! "status": "OPERATION_STATUS_SUCCEEDED" } } +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/3/complete", + "body": { + "completion_reason": "VERSION_COMPLETE_SUCCESS" + } +} +{ + "method": "DELETE", + "path": "/api/2.0/bundle/deployments/[NUMID]" +} diff --git a/acceptance/bundle/dms/record/script b/acceptance/bundle/dms/record/script index 32ee3ec972f..895a9033a39 100644 --- a/acceptance/bundle/dms/record/script +++ b/acceptance/bundle/dms/record/script @@ -1,6 +1,6 @@ title "Deploy: the server assigns the deployment ID, and a version + create operation are recorded" trace $CLI bundle deploy -trace print_requests.py //api/2.0/bundle --sort +trace print_requests.py //api/2.0/bundle title "The deployment ID is the ID of the workspace node the service registered under initial_parent_path; the CLI stores nothing locally" # MSYS_NO_PATHCONV stops Git Bash on Windows from rewriting the leading-'/' path @@ -13,8 +13,8 @@ trace jq 'has("deployment_id")' .databricks/bundle/default/resources.json title "Redeploy after deleting the local cache: the deployment ID is resolved from that node, the same deployment is reused, and the version increments (no new CreateDeployment)" rm -rf .databricks trace $CLI bundle deploy -trace print_requests.py //api/2.0/bundle --sort +trace print_requests.py //api/2.0/bundle title "Destroy: a destroy version and delete operation are recorded, then the deployment is deleted" trace $CLI bundle destroy --auto-approve -trace print_requests.py //api/2.0/bundle --sort +trace print_requests.py //api/2.0/bundle diff --git a/acceptance/bundle/dms/redeploy-after-destroy/output.txt b/acceptance/bundle/dms/redeploy-after-destroy/output.txt index b8b6f3c630d..783082c1f83 100644 --- a/acceptance/bundle/dms/redeploy-after-destroy/output.txt +++ b/acceptance/bundle/dms/redeploy-after-destroy/output.txt @@ -31,7 +31,7 @@ Deployment complete! "path": "/Workspace/Users/[USERNAME]/.bundle/dms-redeploy-after-destroy/default/state/resources.deployment.json" } ->>> print_requests.py //api/2.0/bundle --sort --get +>>> print_requests.py //api/2.0/bundle --get { "method": "POST", "path": "/api/2.0/bundle/deployments", @@ -52,13 +52,6 @@ Deployment complete! "version_type": "VERSION_TYPE_DEPLOY" } } -{ - "method": "POST", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/complete", - "body": { - "completion_reason": "VERSION_COMPLETE_SUCCESS" - } -} { "method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations", @@ -87,3 +80,10 @@ Deployment complete! "status": "OPERATION_STATUS_SUCCEEDED" } } +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/complete", + "body": { + "completion_reason": "VERSION_COMPLETE_SUCCESS" + } +} diff --git a/acceptance/bundle/dms/redeploy-after-destroy/script b/acceptance/bundle/dms/redeploy-after-destroy/script index a39edf3c0d8..8dabf189f49 100644 --- a/acceptance/bundle/dms/redeploy-after-destroy/script +++ b/acceptance/bundle/dms/redeploy-after-destroy/script @@ -1,11 +1,11 @@ title "Deploy, then destroy: the deployment record and its workspace node are both deleted, so nothing points at the destroyed deployment" trace $CLI bundle deploy trace $CLI bundle destroy --auto-approve -print_requests.py //api/2.0/bundle --sort --get > /dev/null +print_requests.py //api/2.0/bundle --get > /dev/null trace MSYS_NO_PATHCONV=1 musterr $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-redeploy-after-destroy/default/state/resources.deployment.json" title "Deploy again: with no node to resolve, a fresh deployment is created at version 1, with its own workspace node" trace $CLI bundle deploy trace MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-redeploy-after-destroy/default/state/resources.deployment.json" | jq '{object_type,path}' -trace print_requests.py //api/2.0/bundle --sort --get +trace print_requests.py //api/2.0/bundle --get diff --git a/bundle/direct/bundle_apply.go b/bundle/direct/bundle_apply.go index f29aa18a186..46d70c7b135 100644 --- a/bundle/direct/bundle_apply.go +++ b/bundle/direct/bundle_apply.go @@ -69,6 +69,17 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa return false } + // Stop before touching the workspace once recording an operation has failed. + // A completed version makes DMS the source of truth for resource state (see + // dstate.readDMSState), so continuing would create resources it has no record + // of and the next deploy would create them a second time. Checked here rather + // than only where operations are recorded, which is after the resource has + // already been modified. + if err := opQueue.firstErr(); err != nil { + logdiag.LogError(ctx, fmt.Errorf("%s: %w", errorPrefix, err)) + return false + } + adapter, err := b.getAdapterForKey(resourceKey) if adapter == nil { logdiag.LogError(ctx, fmt.Errorf("%s: internal error: cannot get adapter: %w", errorPrefix, err)) diff --git a/bundle/direct/opqueue.go b/bundle/direct/opqueue.go index ba189c13d2e..48d2887ec7c 100644 --- a/bundle/direct/opqueue.go +++ b/bundle/direct/opqueue.go @@ -226,8 +226,12 @@ func (q *operationQueue) setErr(err error) { } // firstErr returns the first upload error, or nil if every upload so far -// succeeded. +// succeeded. A nil queue (recording disabled) never errors. func (q *operationQueue) firstErr() error { + if q == nil { + return nil + } + q.mu.Lock() defer q.mu.Unlock() From 9329441d2f4391271c886fc5305addd76c9680a2 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 29 Jul 2026 13:37:48 +0000 Subject: [PATCH 026/125] testserver: create the DMS deployment record on the first version CreateDeployment now only registers the workspace node whose ID names the deployment; the record itself is created by the first CreateVersion. A client that registers a deployment and then fails before recording a version leaves just the node behind, not an empty deployment. That state is reachable, so both sides of the CLI handle it: - the recorder starts at version 1 under the ID the node already names, instead of failing on the missing record or creating a second deployment that would collide on the same node path - the read path keeps the local (empty) state instead of surfacing the 404 from ListResources acceptance/bundle/dms/version-never-created covers it end to end: the first version fails, and the next deploy reuses the same deployment ID. Also drops libs/testserver/bundle_test.go. The fake is exercised by every dms acceptance test, so unit tests for it only duplicate that coverage. Co-authored-by: Isaac --- .../dms/version-never-created/databricks.yml | 10 ++++ .../dms/version-never-created/out.test.toml | 3 ++ .../dms/version-never-created/output.txt | 34 +++++++++++++ .../bundle/dms/version-never-created/script | 7 +++ .../dms/version-never-created/test.toml | 7 +++ bundle/direct/dstate/dms.go | 12 +++++ libs/dms/recorder.go | 26 ++++++---- libs/dms/recorder_test.go | 46 ++++++++++------- libs/testserver/bundle.go | 37 +++++++++----- libs/testserver/bundle_test.go | 51 ------------------- libs/testserver/fake_workspace.go | 7 +++ 11 files changed, 147 insertions(+), 93 deletions(-) create mode 100644 acceptance/bundle/dms/version-never-created/databricks.yml create mode 100644 acceptance/bundle/dms/version-never-created/out.test.toml create mode 100644 acceptance/bundle/dms/version-never-created/output.txt create mode 100644 acceptance/bundle/dms/version-never-created/script create mode 100644 acceptance/bundle/dms/version-never-created/test.toml delete mode 100644 libs/testserver/bundle_test.go diff --git a/acceptance/bundle/dms/version-never-created/databricks.yml b/acceptance/bundle/dms/version-never-created/databricks.yml new file mode 100644 index 00000000000..a7077f18b1f --- /dev/null +++ b/acceptance/bundle/dms/version-never-created/databricks.yml @@ -0,0 +1,10 @@ +bundle: + name: dms-version-never-created + +experimental: + record_deployment_history: true + +resources: + jobs: + foo: + name: foo diff --git a/acceptance/bundle/dms/version-never-created/out.test.toml b/acceptance/bundle/dms/version-never-created/out.test.toml new file mode 100644 index 00000000000..e90b6d5d1ba --- /dev/null +++ b/acceptance/bundle/dms/version-never-created/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/dms/version-never-created/output.txt b/acceptance/bundle/dms/version-never-created/output.txt new file mode 100644 index 00000000000..f6fae9e6f56 --- /dev/null +++ b/acceptance/bundle/dms/version-never-created/output.txt @@ -0,0 +1,34 @@ + +=== The first version fails, so no deployment record exists - only the node naming its ID +>>> musterr [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-version-never-created/default/files... +Error: failed to create deployment version: Internal error (500 INTERNAL_ERROR) + +Endpoint: POST [DATABRICKS_URL]/api/2.0/bundle/deployments/[NUMID]/versions?version_id=1 +HTTP Status: 500 Internal Server Error +API error_code: INTERNAL_ERROR +API message: Internal error + + +>>> MSYS_NO_PATHCONV=1 [CLI] workspace get-status /Workspace/Users/[USERNAME]/.bundle/dms-version-never-created/default/state/resources.deployment.json +{ + "object_type": "FILE" +} + +=== The next deploy reuses the ID that node names and retries version 1, rather than creating a second deployment +>>> musterr [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-version-never-created/default/files... +Error: failed to create deployment version: Internal error (500 INTERNAL_ERROR) + +Endpoint: POST [DATABRICKS_URL]/api/2.0/bundle/deployments/[NUMID]/versions?version_id=1 +HTTP Status: 500 Internal Server Error +API error_code: INTERNAL_ERROR +API message: Internal error + + +>>> print_requests.py //api/2.0/bundle --get --oneline +{"method": "POST", "path": "/api/2.0/bundle/deployments", "body": {"initial_parent_path": "/Workspace/Users/[USERNAME]/.bundle/dms-version-never-created/default/state", "target_name": "default"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": {"version_id": "1"}, "body": {"cli_version": "[CLI_VERSION]", "target_name": "default", "version_type": "VERSION_TYPE_DEPLOY"}} +{"method": "GET", "path": "/api/2.0/bundle/deployments/[NUMID]/resources"} +{"method": "GET", "path": "/api/2.0/bundle/deployments/[NUMID]"} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": {"version_id": "1"}, "body": {"cli_version": "[CLI_VERSION]", "target_name": "default", "version_type": "VERSION_TYPE_DEPLOY"}} diff --git a/acceptance/bundle/dms/version-never-created/script b/acceptance/bundle/dms/version-never-created/script new file mode 100644 index 00000000000..39566343e98 --- /dev/null +++ b/acceptance/bundle/dms/version-never-created/script @@ -0,0 +1,7 @@ +title "The first version fails, so no deployment record exists - only the node naming its ID" +trace musterr $CLI bundle deploy +trace MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-version-never-created/default/state/resources.deployment.json" | jq '{object_type}' + +title "The next deploy reuses the ID that node names and retries version 1, rather than creating a second deployment" +trace musterr $CLI bundle deploy +trace print_requests.py //api/2.0/bundle --get --oneline diff --git a/acceptance/bundle/dms/version-never-created/test.toml b/acceptance/bundle/dms/version-never-created/test.toml new file mode 100644 index 00000000000..f0a8407e21d --- /dev/null +++ b/acceptance/bundle/dms/version-never-created/test.toml @@ -0,0 +1,7 @@ +# The first version fails, so the deployment record is never created - only the +# workspace node CreateDeployment registered. The next deploy resolves the ID from +# that node and has to cope with a deployment that has no record yet. +[[Server]] +Pattern = "POST /api/2.0/bundle/deployments/{deployment_id}/versions" +Response.StatusCode = 500 +Response.Body = '''{"error_code": "INTERNAL_ERROR", "message": "Internal error"}''' diff --git a/bundle/direct/dstate/dms.go b/bundle/direct/dstate/dms.go index ec51e2c4479..094f114a0a6 100644 --- a/bundle/direct/dstate/dms.go +++ b/bundle/direct/dstate/dms.go @@ -3,9 +3,12 @@ package dstate import ( "context" "encoding/json" + "errors" "fmt" "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/cli/libs/log" + "github.com/databricks/databricks-sdk-go/apierr" "github.com/databricks/databricks-sdk-go/service/bundledeployments" ) @@ -33,6 +36,15 @@ type RecordedState struct { func (db *DeploymentState) readDMSState(ctx context.Context, src *DMSSource) error { resources, err := fetchDeploymentResources(ctx, src.Client, src.DeploymentID) if err != nil { + // The deployment's record is created by its first version, so a node can + // resolve to an ID that has none yet: a deploy that registered the deployment + // and then failed before recording a version. There is nothing to read, and + // the file's resources are still empty, so carry on and let this deploy record + // the first version. + if errors.Is(err, apierr.ErrNotFound) || errors.Is(err, apierr.ErrResourceDoesNotExist) { + log.Debugf(ctx, "No deployment record for %s yet; keeping local state", src.DeploymentID) + return nil + } return err } diff --git a/libs/dms/recorder.go b/libs/dms/recorder.go index e285a80e4a2..6973a09b4b5 100644 --- a/libs/dms/recorder.go +++ b/libs/dms/recorder.go @@ -150,22 +150,26 @@ func (r *Recorder) CompleteVersion(ctx context.Context, success bool) error { // compute the next version number. func (r *Recorder) createDeploymentVersion(ctx context.Context) (versionID string, err error) { if r.deploymentID != "" { - // Existing deployment: read it to compute the next version number. A 404 is - // not recovered from by creating a second deployment. The service trashes the - // workspace node when it deletes the record, so a node that resolved but has - // no record means the two are out of sync, and creating another deployment - // would collide on the same node path. + // A resolved node names the deployment, but its record is created by the + // first version, so there may be none yet: a deploy that registered the + // deployment and then failed before recording a version. Start at version 1 + // under the ID the node already names, rather than creating a second + // deployment, which would collide on the same node path. dep, getErr := r.svc.GetDeployment(ctx, bundledeployments.GetDeploymentRequest{ Name: "deployments/" + r.deploymentID, }) - if getErr != nil { + switch { + case getErr == nil: + lastVersion, parseErr := strconv.ParseInt(dep.LastVersionId, 10, 64) + if parseErr != nil { + return "", fmt.Errorf("failed to parse last_version_id %q: %w", dep.LastVersionId, parseErr) + } + versionID = strconv.FormatInt(lastVersion+1, 10) + case errors.Is(getErr, apierr.ErrNotFound), errors.Is(getErr, apierr.ErrResourceDoesNotExist): + versionID = "1" + default: return "", fmt.Errorf("failed to get deployment: %w", getErr) } - lastVersion, parseErr := strconv.ParseInt(dep.LastVersionId, 10, 64) - if parseErr != nil { - return "", fmt.Errorf("failed to parse last_version_id %q: %w", dep.LastVersionId, parseErr) - } - versionID = strconv.FormatInt(lastVersion+1, 10) } else { // First deploy: create the deployment so the server assigns an ID. // diff --git a/libs/dms/recorder_test.go b/libs/dms/recorder_test.go index 6c2f334c946..d4c7efaca03 100644 --- a/libs/dms/recorder_test.go +++ b/libs/dms/recorder_test.go @@ -112,27 +112,35 @@ func TestRecorderSubsequentDeployReusesDeploymentAndIncrementsVersion(t *testing } func TestRecorderGetDeploymentErrorFailsDeploy(t *testing.T) { - cases := map[string]error{ - // A resolved ID whose record is missing means the record and the workspace - // node it was resolved from are out of sync. Creating a second deployment - // would collide on the same node path, so fail instead. - "not found": fmt.Errorf("deployment: %w", apierr.ErrNotFound), - "other": errors.New("boom"), + f := &fakeDMS{ + getDeployment: func(id string) (*bundledeployments.Deployment, error) { + return nil, errors.New("boom") + }, } - for name, getErr := range cases { - t.Run(name, func(t *testing.T) { - f := &fakeDMS{ - getDeployment: func(id string) (*bundledeployments.Deployment, error) { - return nil, getErr - }, - } - r := NewRecorder(f, "stored-id", testStatePath, "dev", VersionTypeDeploy) - - err := r.CreateVersion(t.Context()) - assert.ErrorContains(t, err, "failed to get deployment") - assert.Empty(t, f.created) - }) + r := NewRecorder(f, "stored-id", testStatePath, "dev", VersionTypeDeploy) + + err := r.CreateVersion(t.Context()) + assert.ErrorContains(t, err, "failed to get deployment") + assert.Empty(t, f.created) +} + +func TestRecorderMissingDeploymentRecordStartsAtVersionOne(t *testing.T) { + // The record is created by the first version, so a node can name a deployment + // that has none yet - an earlier deploy registered it and then failed. Record + // version 1 under that same ID instead of creating a second deployment, which + // would collide on the node path. + f := &fakeDMS{ + getDeployment: func(id string) (*bundledeployments.Deployment, error) { + return nil, fmt.Errorf("deployment: %w", apierr.ErrNotFound) + }, } + r := NewRecorder(f, "stored-id", testStatePath, "dev", VersionTypeDeploy) + + require.NoError(t, r.CreateVersion(t.Context())) + assert.Empty(t, f.created) + require.Len(t, f.versions, 1) + assert.Equal(t, "1", f.versions[0].VersionId) + assert.Equal(t, "deployments/stored-id", f.versions[0].Parent) } func TestRecorderDestroyDeletesDeploymentOnSuccess(t *testing.T) { diff --git a/libs/testserver/bundle.go b/libs/testserver/bundle.go index 6dffbe34a75..4e04c65ba81 100644 --- a/libs/testserver/bundle.go +++ b/libs/testserver/bundle.go @@ -35,9 +35,6 @@ type dmsDeployment struct { // value as "DMS owns the state". Tracked separately because the SDK // Deployment struct does not yet carry the field (still stage:DEVELOPMENT). lastSuccessfulVersionID string - // nodePath is the workspace node whose object ID is this deployment's ID. - // Kept so DeleteDeployment can trash the node, the way the service does. - nodePath string } func (s *FakeWorkspace) CreateDeployment(req Request) Response { @@ -70,15 +67,15 @@ func (s *FakeWorkspace) CreateDeployment(req Request) Response { }, } + // Only the node is created here. The deployment record itself is created by the + // first CreateVersion, so a client that creates a deployment and then fails + // before recording a version leaves no record behind - just the node, which + // names the ID that first version will be created under. deploymentID := strconv.FormatInt(objectID, 10) + s.dmsDeploymentNodes[deploymentID] = nodePath + dep.Name = "deployments/" + deploymentID dep.Status = bundledeployments.DeploymentStatusDeploymentStatusActive - s.dmsDeployments[deploymentID] = &dmsDeployment{ - deployment: dep, - versions: map[string]*bundledeployments.Version{}, - resources: map[string]bundledeployments.Resource{}, - nodePath: nodePath, - } return Response{Body: dep} } @@ -129,9 +126,10 @@ func (s *FakeWorkspace) DeleteDeployment(deploymentID string) Response { // The service trashes the deployment's workspace node, so a later get-status // on the node path reports the deployment as absent. - if d, ok := s.dmsDeployments[deploymentID]; ok { - delete(s.files, d.nodePath) + if nodePath, ok := s.dmsDeploymentNodes[deploymentID]; ok { + delete(s.files, nodePath) } + delete(s.dmsDeploymentNodes, deploymentID) delete(s.dmsDeployments, deploymentID) return Response{Body: map[string]any{}} } @@ -148,7 +146,22 @@ func (s *FakeWorkspace) CreateVersion(req Request, deploymentID string) Response d, ok := s.dmsDeployments[deploymentID] if !ok { - return dmsNotFound("deployment " + deploymentID) + // The deployment record is created by its first version, not by + // CreateDeployment. That call only registered the workspace node, so the node + // existing is what makes this ID valid. + if _, known := s.dmsDeploymentNodes[deploymentID]; !known { + return dmsNotFound("deployment " + deploymentID) + } + d = &dmsDeployment{ + deployment: bundledeployments.Deployment{ + Name: "deployments/" + deploymentID, + Status: bundledeployments.DeploymentStatusDeploymentStatusActive, + TargetName: version.TargetName, + }, + versions: map[string]*bundledeployments.Version{}, + resources: map[string]bundledeployments.Resource{}, + } + s.dmsDeployments[deploymentID] = d } // Mirror the server-side optimistic concurrency check: the new version must diff --git a/libs/testserver/bundle_test.go b/libs/testserver/bundle_test.go deleted file mode 100644 index 4d28624ba3e..00000000000 --- a/libs/testserver/bundle_test.go +++ /dev/null @@ -1,51 +0,0 @@ -package testserver - -import ( - "encoding/json" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -// TestDeploymentBodyKeepsTypedFieldsAndLastSuccessfulVersionID guards against -// serializing the deployment through a struct that embeds -// bundledeployments.Deployment: Deployment has its own MarshalJSON, which is -// promoted to the embedding struct and silently drops last_successful_version_id. -// The CLI read path treats a missing value as "DMS does not own the state", so -// losing the field here makes the whole overlay path untestable. -func TestDeploymentBodyKeepsTypedFieldsAndLastSuccessfulVersionID(t *testing.T) { - d := &dmsDeployment{lastSuccessfulVersionID: "2"} - d.deployment.Name = "deployments/abc" - d.deployment.LastVersionId = "3" - d.deployment.TargetName = "default" - - body, err := deploymentBody(d) - require.NoError(t, err) - - assert.Equal(t, "deployments/abc", body["name"]) - assert.Equal(t, "3", body["last_version_id"]) - assert.Equal(t, "default", body["target_name"]) - assert.Equal(t, "2", body["last_successful_version_id"]) - - // The response must round-trip as JSON the same way, since that is what the - // client actually reads. - raw, err := json.Marshal(body) - require.NoError(t, err) - assert.JSONEq(t, - `{"name":"deployments/abc","last_version_id":"3","target_name":"default","last_successful_version_id":"2"}`, - string(raw)) -} - -// TestDeploymentBodyOmitsUnsetLastSuccessfulVersionID checks that a deployment -// with no successful version does not advertise one: the read path must keep -// using the local state file in that case. -func TestDeploymentBodyOmitsUnsetLastSuccessfulVersionID(t *testing.T) { - d := &dmsDeployment{} - d.deployment.Name = "deployments/abc" - - body, err := deploymentBody(d) - require.NoError(t, err) - - assert.NotContains(t, body, "last_successful_version_id") -} diff --git a/libs/testserver/fake_workspace.go b/libs/testserver/fake_workspace.go index a3c4519ccc4..13d9f76e00a 100644 --- a/libs/testserver/fake_workspace.go +++ b/libs/testserver/fake_workspace.go @@ -231,6 +231,12 @@ type FakeWorkspace struct { // dmsDeployments holds Deployment Metadata Service (DMS) records, keyed by // deployment ID. Each record carries its versions and latest resource state. dmsDeployments map[string]*dmsDeployment + + // dmsDeploymentNodes maps deployment ID to the workspace node CreateDeployment + // registered for it. A deployment appears here before it has a record in + // dmsDeployments: the record is created by its first version, so the node is + // what makes an ID valid in between. + dmsDeploymentNodes map[string]string } func (s *FakeWorkspace) LockUnlock() func() { @@ -383,6 +389,7 @@ func NewFakeWorkspace(url, token string) *FakeWorkspace { postgresImplicitEndpoints: map[string]bool{}, clusterVenvs: map[string]*clusterEnv{}, dmsDeployments: map[string]*dmsDeployment{}, + dmsDeploymentNodes: map[string]string{}, Alerts: map[string]sql.AlertV2{}, Experiments: map[string]ml.GetExperimentResponse{}, ModelRegistryModels: map[string]ml.Model{}, From 38b6f54081ae33008edd866a64ffd696c6dc90af Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 29 Jul 2026 14:05:39 +0000 Subject: [PATCH 027/125] bundle: simplify the concurrency test comment Co-authored-by: Isaac --- bundle/direct/opqueue_test.go | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/bundle/direct/opqueue_test.go b/bundle/direct/opqueue_test.go index a68560493ce..910b58d470f 100644 --- a/bundle/direct/opqueue_test.go +++ b/bundle/direct/opqueue_test.go @@ -307,13 +307,16 @@ func (s *serialUploader) upload(ctx context.Context, resourceKey string, op reco } func TestOperationQueueUploadsOneResourceAtATime(t *testing.T) { - // Concurrent apply workers repeatedly record overlapping resource keys, the - // case where a coalesced key can be handed to a second worker while the first - // is still uploading it. The service keeps one state per key, so overlapping - // uploads for a key could land out of order and leave a stale state behind. + // Two workers must never upload the same resource at the same time. DMS stores + // one state per resource, so concurrent uploads can finish out of order and + // leave the older state as the final one. // - // The interleaving that breaks this is scheduler-dependent, so one pass proves - // little: repeat it so a single run has many chances to hit the bad ordering. + // Lots of goroutines record a small set of keys, so the same key is recorded + // repeatedly while its earlier upload may still be running. serialUploader flags + // any overlap it sees. + // + // Whether a bug shows up depends on how the scheduler interleaves things, so one + // pass proves little - repeat it to get many chances at a bad ordering. const ( iterations = 200 workers = 10 From 4f45539f06f8071c8441433dbe69f2ba9b7669da Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 29 Jul 2026 15:16:42 +0000 Subject: [PATCH 028/125] bundle: trim the operation recorder tests Drops TestOperationRecorderDeleteHasNoState: the dms acceptance goldens already show a delete operation recorded without a state field. Renames the redaction test to say what it checks - the state is recorded as-is - rather than contrasting it with dstate.SaveState. Co-authored-by: Isaac --- bundle/direct/oprecorder_test.go | 17 ++--------------- 1 file changed, 2 insertions(+), 15 deletions(-) diff --git a/bundle/direct/oprecorder_test.go b/bundle/direct/oprecorder_test.go index 1b68c3a626b..674c78abf77 100644 --- a/bundle/direct/oprecorder_test.go +++ b/bundle/direct/oprecorder_test.go @@ -52,19 +52,7 @@ func TestOperationRecorderStripsResourcePrefix(t *testing.T) { require.NotNil(t, req.Operation.State) } -func TestOperationRecorderDeleteHasNoState(t *testing.T) { - f := &fakeOpClient{} - r := NewOperationRecorder(f, "dep-1", 3) - - uploadOne(t, r, "resources.jobs.foo", deployplan.Delete, "", nil) - - require.Len(t, f.requests, 1) - assert.Equal(t, bundledeployments.OperationActionTypeOperationActionTypeDelete, f.requests[0].Operation.ActionType) - // Delete operations carry no serialized state. - assert.Nil(t, f.requests[0].Operation.State) -} - -func TestNewRecordedOperationDoesNotRedactSensitiveFields(t *testing.T) { +func TestNewRecordedOperationRecordsStateAsIs(t *testing.T) { state := struct { Name string `json:"name"` Token string `json:"token" bundle:"sensitive"` @@ -73,8 +61,7 @@ func TestNewRecordedOperationDoesNotRedactSensitiveFields(t *testing.T) { op, err := newRecordedOperation(deployplan.Create, "job-123", state, nil) require.NoError(t, err) - // Recorded as-is, unlike dstate.SaveState which redacts before writing the - // local state file. + // The state is serialized as-is, including fields tagged bundle:"sensitive". assert.JSONEq(t, `{"state":{"name":"foo","token":"super-secret"}}`, string(op.state)) From 2bf49fb490b1af8305b996e777da075385c698bc Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 3 Aug 2026 16:47:17 +0000 Subject: [PATCH 029/125] bundle: send recorded state as a JSON string DMS types Operation.state as a string, so the JSON has to go on the wire quoted. The CLI was assigning the raw object to the SDK's json.RawMessage field, which the service rejected: Invalid value: {"state":{...}} for expected type: STRING Every CreateOperation failed while the deploy still reported success, so DMS ended up owning an empty resource set - and because a completed version makes it authoritative, the next deploy would have recreated everything. Verified on dogfood: before this change ListResources returned {} after a successful deploy; after it, the resource is recorded and a deploy following `rm -rf .databricks` plans "0 to add, 1 unchanged" from DMS state alone. The read path unquotes symmetrically, and the test server now stores and returns state the way the service does, so the acceptance goldens show the real wire format rather than a shape only the fake produces. Co-authored-by: Isaac --- acceptance/bundle/dms/depends-on/output.txt | 39 +------------------ .../bundle/dms/existing-state/output.txt | 2 +- acceptance/bundle/dms/record/output.txt | 16 +------- .../dms/redeploy-after-destroy/output.txt | 16 +------- bundle/direct/dstate/dms.go | 9 ++++- bundle/direct/dstate/dms_test.go | 6 ++- bundle/direct/oprecorder.go | 11 +++++- 7 files changed, 28 insertions(+), 71 deletions(-) diff --git a/acceptance/bundle/dms/depends-on/output.txt b/acceptance/bundle/dms/depends-on/output.txt index 67fd0165669..db3624d9fc1 100644 --- a/acceptance/bundle/dms/depends-on/output.txt +++ b/acceptance/bundle/dms/depends-on/output.txt @@ -17,28 +17,7 @@ Deployment complete! "action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_id": "[NUMID]", "resource_key": "jobs.child", - "state": { - "state": { - "deployment": { - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-depends-on/default/state/metadata.json" - }, - "description": "depends on [NUMID]", - "edit_mode": "UI_LOCKED", - "format": "MULTI_TASK", - "max_concurrent_runs": 1, - "name": "child", - "queue": { - "enabled": true - } - }, - "depends_on": [ - { - "node": "resources.jobs.parent", - "label": "${resources.jobs.parent.id}" - } - ] - }, + "state": "{\"state\":{\"deployment\":{\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-depends-on/default/state/metadata.json\"},\"description\":\"depends on [NUMID]\",\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"child\",\"queue\":{\"enabled\":true}},\"depends_on\":[{\"node\":\"resources.jobs.parent\",\"label\":\"${resources.jobs.parent.id}\"}]}", "status": "OPERATION_STATUS_SUCCEEDED" } } @@ -52,21 +31,7 @@ Deployment complete! "action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_id": "[NUMID]", "resource_key": "jobs.parent", - "state": { - "state": { - "deployment": { - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-depends-on/default/state/metadata.json" - }, - "edit_mode": "UI_LOCKED", - "format": "MULTI_TASK", - "max_concurrent_runs": 1, - "name": "parent", - "queue": { - "enabled": true - } - } - }, + "state": "{\"state\":{\"deployment\":{\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-depends-on/default/state/metadata.json\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"parent\",\"queue\":{\"enabled\":true}}}", "status": "OPERATION_STATUS_SUCCEEDED" } } diff --git a/acceptance/bundle/dms/existing-state/output.txt b/acceptance/bundle/dms/existing-state/output.txt index 755981c414d..e8067f822f5 100644 --- a/acceptance/bundle/dms/existing-state/output.txt +++ b/acceptance/bundle/dms/existing-state/output.txt @@ -48,5 +48,5 @@ Deployment complete! >>> print_requests.py //api/2.0/bundle --oneline {"method": "POST", "path": "/api/2.0/bundle/deployments", "body": {"initial_parent_path": "/Workspace/Users/[USERNAME]/.bundle/dms-existing-state/default/state", "target_name": "default"}} {"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": {"version_id": "1"}, "body": {"cli_version": "[CLI_VERSION]", "target_name": "default", "version_type": "VERSION_TYPE_DEPLOY"}} -{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations", "q": {"resource_key": "jobs.one"}, "body": {"action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_id": "[NUMID]", "resource_key": "jobs.one", "state": {"state": {"deployment": {"kind": "BUNDLE", "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-existing-state/default/state/metadata.json"}, "edit_mode": "UI_LOCKED", "format": "MULTI_TASK", "max_concurrent_runs": 1, "name": "one", "queue": {"enabled": true}}}, "status": "OPERATION_STATUS_SUCCEEDED"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations", "q": {"resource_key": "jobs.one"}, "body": {"action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_id": "[NUMID]", "resource_key": "jobs.one", "state": "{\"state\":{\"deployment\":{\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-existing-state/default/state/metadata.json\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"one\",\"queue\":{\"enabled\":true}}}", "status": "OPERATION_STATUS_SUCCEEDED"}} {"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/complete", "body": {"completion_reason": "VERSION_COMPLETE_SUCCESS"}} diff --git a/acceptance/bundle/dms/record/output.txt b/acceptance/bundle/dms/record/output.txt index dcb6b3efa22..3f9b8a8eadc 100644 --- a/acceptance/bundle/dms/record/output.txt +++ b/acceptance/bundle/dms/record/output.txt @@ -37,21 +37,7 @@ Deployment complete! "action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_id": "[NUMID]", "resource_key": "jobs.foo", - "state": { - "state": { - "deployment": { - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-record/default/state/metadata.json" - }, - "edit_mode": "UI_LOCKED", - "format": "MULTI_TASK", - "max_concurrent_runs": 1, - "name": "foo", - "queue": { - "enabled": true - } - } - }, + "state": "{\"state\":{\"deployment\":{\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-record/default/state/metadata.json\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"foo\",\"queue\":{\"enabled\":true}}}", "status": "OPERATION_STATUS_SUCCEEDED" } } diff --git a/acceptance/bundle/dms/redeploy-after-destroy/output.txt b/acceptance/bundle/dms/redeploy-after-destroy/output.txt index 783082c1f83..c06c1d9b7cd 100644 --- a/acceptance/bundle/dms/redeploy-after-destroy/output.txt +++ b/acceptance/bundle/dms/redeploy-after-destroy/output.txt @@ -62,21 +62,7 @@ Deployment complete! "action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_id": "[NUMID]", "resource_key": "jobs.foo", - "state": { - "state": { - "deployment": { - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-redeploy-after-destroy/default/state/metadata.json" - }, - "edit_mode": "UI_LOCKED", - "format": "MULTI_TASK", - "max_concurrent_runs": 1, - "name": "foo", - "queue": { - "enabled": true - } - } - }, + "state": "{\"state\":{\"deployment\":{\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-redeploy-after-destroy/default/state/metadata.json\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"foo\",\"queue\":{\"enabled\":true}}}", "status": "OPERATION_STATUS_SUCCEEDED" } } diff --git a/bundle/direct/dstate/dms.go b/bundle/direct/dstate/dms.go index 094f114a0a6..3d7131580e2 100644 --- a/bundle/direct/dstate/dms.go +++ b/bundle/direct/dstate/dms.go @@ -77,7 +77,14 @@ func fetchDeploymentResources(ctx context.Context, client bundledeployments.Bund var recorded RecordedState if res.State != nil { - if err := json.Unmarshal(*res.State, &recorded); err != nil { + // State is a string field, so it arrives as a quoted JSON string (see the + // write side in direct.operationRecorder.upload). Unquote it, then parse + // the envelope it holds. + var envelope string + if err := json.Unmarshal(*res.State, &envelope); err != nil { + return nil, fmt.Errorf("interpreting state recorded for %s: %w", key, err) + } + if err := json.Unmarshal([]byte(envelope), &recorded); err != nil { return nil, fmt.Errorf("interpreting state recorded for %s: %w", key, err) } } diff --git a/bundle/direct/dstate/dms_test.go b/bundle/direct/dstate/dms_test.go index 35fe7acbb0c..8145424ca93 100644 --- a/bundle/direct/dstate/dms_test.go +++ b/bundle/direct/dstate/dms_test.go @@ -36,7 +36,11 @@ func (f *fakeResourceLister) ListResources(ctx context.Context, req bundledeploy } func TestFetchDeploymentResourcesUnwrapsEnvelope(t *testing.T) { - recorded := json.RawMessage(`{"state":{"name":"foo"},"depends_on":[{"node":"resources.pipelines.bar","label":"${resources.pipelines.bar.id}"}]}`) + // DMS types state as a string, so the envelope arrives as a quoted JSON string. + envelope := `{"state":{"name":"foo"},"depends_on":[{"node":"resources.pipelines.bar","label":"${resources.pipelines.bar.id}"}]}` + quoted, err := json.Marshal(envelope) + require.NoError(t, err) + recorded := json.RawMessage(quoted) f := &fakeResourceLister{resources: []bundledeployments.Resource{ {ResourceKey: "jobs.foo", ResourceId: "123", State: &recorded}, {ResourceKey: "pipelines.bar", ResourceId: "456"}, diff --git a/bundle/direct/oprecorder.go b/bundle/direct/oprecorder.go index d0d2694d358..a1a5cf641bc 100644 --- a/bundle/direct/oprecorder.go +++ b/bundle/direct/oprecorder.go @@ -99,7 +99,16 @@ func (r *operationRecorder) upload(ctx context.Context, resourceKey string, op r Status: bundledeployments.OperationStatusOperationStatusSucceeded, } if op.state != nil { - operation.State = &op.state + // DMS types state as a string, so the JSON goes on the wire as a quoted + // string rather than an embedded object. The SDK field is a json.RawMessage, + // so quote the payload here; sending the object directly is rejected with + // "Invalid value: {...} for expected type: STRING". + quoted, err := json.Marshal(string(op.state)) + if err != nil { + return fmt.Errorf("serializing state: %w", err) + } + raw := json.RawMessage(quoted) + operation.State = &raw } _, err := r.client.CreateOperation(ctx, bundledeployments.CreateOperationRequest{ From 93e95a4a6f41773a612e4071739ad9dffd1e8a4b Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 4 Aug 2026 10:24:19 +0000 Subject: [PATCH 030/125] bundle: send previous_version_id and display_name when recording a version Two fields the service needs were never sent, because the generated bundledeployments.Version struct does not have them both: previous_version_id is the service's concurrency check. Without it every deploy after the first was rejected with "previous_version_id is outdated; the deployment's most recent version is N", so recording only ever worked once per bundle. The struct has no such field, so the CLI now builds the CreateVersion body itself via a small local type rather than the generated client. display_name is what names the deployment in the UI. The service copies it from the version onto the deployment's workspace node, which is where GetDeployment reads it back from, and it only does so when the version carries one - so every deployment showed up unnamed. It comes from bundle.name. The test server enforced "version_id == last_version_id + 1", a rule the real service does not have (it requires numerically greater, plus a matching previous_version_id). Corrected, so the fake rejects a stale previous_version_id the way the service does instead of accepting a contract only it implements. Verified on dogfood: two consecutive deploys both succeed (the second used to fail), GetDeployment and ListDeployments both return display_name "isaac-fix2-check" - the only named deployment among 38 - and a plan after rm -rf .databricks still reports "0 to add, 1 unchanged" from DMS state. Co-authored-by: Isaac --- .../bundle/dms/existing-state/output.txt | 2 +- .../bundle/dms/multiple-resources/output.txt | 2 +- acceptance/bundle/dms/no-resources/output.txt | 7 +- acceptance/bundle/dms/record/output.txt | 11 +- .../dms/redeploy-after-destroy/output.txt | 3 +- .../dms/version-never-created/output.txt | 4 +- bundle/phases/dms.go | 21 ++-- libs/dms/recorder.go | 113 ++++++++++++++---- libs/dms/recorder_test.go | 74 +++++++++--- libs/testserver/bundle.go | 37 ++++-- 10 files changed, 210 insertions(+), 64 deletions(-) diff --git a/acceptance/bundle/dms/existing-state/output.txt b/acceptance/bundle/dms/existing-state/output.txt index e8067f822f5..e7bb54440b5 100644 --- a/acceptance/bundle/dms/existing-state/output.txt +++ b/acceptance/bundle/dms/existing-state/output.txt @@ -47,6 +47,6 @@ Deployment complete! >>> print_requests.py //api/2.0/bundle --oneline {"method": "POST", "path": "/api/2.0/bundle/deployments", "body": {"initial_parent_path": "/Workspace/Users/[USERNAME]/.bundle/dms-existing-state/default/state", "target_name": "default"}} -{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": {"version_id": "1"}, "body": {"cli_version": "[CLI_VERSION]", "target_name": "default", "version_type": "VERSION_TYPE_DEPLOY"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": {"version_id": "1"}, "body": {"cli_version": "[CLI_VERSION]", "version_type": "VERSION_TYPE_DEPLOY", "target_name": "default", "display_name": "dms-existing-state"}} {"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations", "q": {"resource_key": "jobs.one"}, "body": {"action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_id": "[NUMID]", "resource_key": "jobs.one", "state": "{\"state\":{\"deployment\":{\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-existing-state/default/state/metadata.json\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"one\",\"queue\":{\"enabled\":true}}}", "status": "OPERATION_STATUS_SUCCEEDED"}} {"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/complete", "body": {"completion_reason": "VERSION_COMPLETE_SUCCESS"}} diff --git a/acceptance/bundle/dms/multiple-resources/output.txt b/acceptance/bundle/dms/multiple-resources/output.txt index 75086ceb5f7..1d208e1e85e 100644 --- a/acceptance/bundle/dms/multiple-resources/output.txt +++ b/acceptance/bundle/dms/multiple-resources/output.txt @@ -21,5 +21,5 @@ Updating deployment state... Deployment complete! >>> print_requests.py //api/2.0/bundle --sort --oneline -{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": {"version_id": "2"}, "body": {"cli_version": "[CLI_VERSION]", "target_name": "default", "version_type": "VERSION_TYPE_DEPLOY"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": {"version_id": "2"}, "body": {"cli_version": "[CLI_VERSION]", "version_type": "VERSION_TYPE_DEPLOY", "target_name": "default", "display_name": "dms-multiple-resources", "previous_version_id": "1"}} {"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/complete", "body": {"completion_reason": "VERSION_COMPLETE_SUCCESS"}} diff --git a/acceptance/bundle/dms/no-resources/output.txt b/acceptance/bundle/dms/no-resources/output.txt index 61e520c20a2..06e346f3029 100644 --- a/acceptance/bundle/dms/no-resources/output.txt +++ b/acceptance/bundle/dms/no-resources/output.txt @@ -22,8 +22,9 @@ Deployment complete! }, "body": { "cli_version": "[CLI_VERSION]", + "version_type": "VERSION_TYPE_DEPLOY", "target_name": "default", - "version_type": "VERSION_TYPE_DEPLOY" + "display_name": "dms-no-resources" } } { @@ -63,8 +64,10 @@ Deployment complete! }, "body": { "cli_version": "[CLI_VERSION]", + "version_type": "VERSION_TYPE_DEPLOY", "target_name": "default", - "version_type": "VERSION_TYPE_DEPLOY" + "display_name": "dms-no-resources", + "previous_version_id": "1" } } { diff --git a/acceptance/bundle/dms/record/output.txt b/acceptance/bundle/dms/record/output.txt index 3f9b8a8eadc..f65d67d0991 100644 --- a/acceptance/bundle/dms/record/output.txt +++ b/acceptance/bundle/dms/record/output.txt @@ -23,8 +23,9 @@ Deployment complete! }, "body": { "cli_version": "[CLI_VERSION]", + "version_type": "VERSION_TYPE_DEPLOY", "target_name": "default", - "version_type": "VERSION_TYPE_DEPLOY" + "display_name": "dms-record" } } { @@ -75,8 +76,10 @@ Deployment complete! }, "body": { "cli_version": "[CLI_VERSION]", + "version_type": "VERSION_TYPE_DEPLOY", "target_name": "default", - "version_type": "VERSION_TYPE_DEPLOY" + "display_name": "dms-record", + "previous_version_id": "1" } } { @@ -106,8 +109,10 @@ Destroy complete! }, "body": { "cli_version": "[CLI_VERSION]", + "version_type": "VERSION_TYPE_DESTROY", "target_name": "default", - "version_type": "VERSION_TYPE_DESTROY" + "display_name": "dms-record", + "previous_version_id": "2" } } { diff --git a/acceptance/bundle/dms/redeploy-after-destroy/output.txt b/acceptance/bundle/dms/redeploy-after-destroy/output.txt index c06c1d9b7cd..6e160478e7b 100644 --- a/acceptance/bundle/dms/redeploy-after-destroy/output.txt +++ b/acceptance/bundle/dms/redeploy-after-destroy/output.txt @@ -48,8 +48,9 @@ Deployment complete! }, "body": { "cli_version": "[CLI_VERSION]", + "version_type": "VERSION_TYPE_DEPLOY", "target_name": "default", - "version_type": "VERSION_TYPE_DEPLOY" + "display_name": "dms-redeploy-after-destroy" } } { diff --git a/acceptance/bundle/dms/version-never-created/output.txt b/acceptance/bundle/dms/version-never-created/output.txt index f6fae9e6f56..23f84279395 100644 --- a/acceptance/bundle/dms/version-never-created/output.txt +++ b/acceptance/bundle/dms/version-never-created/output.txt @@ -28,7 +28,7 @@ API message: Internal error >>> print_requests.py //api/2.0/bundle --get --oneline {"method": "POST", "path": "/api/2.0/bundle/deployments", "body": {"initial_parent_path": "/Workspace/Users/[USERNAME]/.bundle/dms-version-never-created/default/state", "target_name": "default"}} -{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": {"version_id": "1"}, "body": {"cli_version": "[CLI_VERSION]", "target_name": "default", "version_type": "VERSION_TYPE_DEPLOY"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": {"version_id": "1"}, "body": {"cli_version": "[CLI_VERSION]", "version_type": "VERSION_TYPE_DEPLOY", "target_name": "default", "display_name": "dms-version-never-created"}} {"method": "GET", "path": "/api/2.0/bundle/deployments/[NUMID]/resources"} {"method": "GET", "path": "/api/2.0/bundle/deployments/[NUMID]"} -{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": {"version_id": "1"}, "body": {"cli_version": "[CLI_VERSION]", "target_name": "default", "version_type": "VERSION_TYPE_DEPLOY"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": {"version_id": "1"}, "body": {"cli_version": "[CLI_VERSION]", "version_type": "VERSION_TYPE_DEPLOY", "target_name": "default", "display_name": "dms-version-never-created"}} diff --git a/bundle/phases/dms.go b/bundle/phases/dms.go index 3d2f4f54009..346d6ca70c5 100644 --- a/bundle/phases/dms.go +++ b/bundle/phases/dms.go @@ -6,6 +6,7 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config/engine" "github.com/databricks/cli/libs/dms" + "github.com/databricks/databricks-sdk-go/client" ) // newDeploymentRecorder returns a dms.Recorder for the current deployment, or @@ -33,11 +34,17 @@ func newDeploymentRecorder(ctx context.Context, b *bundle.Bundle, eng engine.Eng if err != nil { return nil, err } - return dms.NewRecorder( - b.WorkspaceClient(ctx).BundleDeployments, - deploymentID, - statePath, - b.Config.Bundle.Target, - versionType, - ), nil + apiClient, err := client.New(b.WorkspaceClient(ctx).Config) + if err != nil { + return nil, err + } + return dms.NewRecorder(dms.RecorderOptions{ + Service: b.WorkspaceClient(ctx).BundleDeployments, + Versions: dms.NewAPIVersionCreator(apiClient), + DeploymentID: deploymentID, + StatePath: statePath, + TargetName: b.Config.Bundle.Target, + DisplayName: b.Config.Bundle.Name, + VersionType: versionType, + }), nil } diff --git a/libs/dms/recorder.go b/libs/dms/recorder.go index 6973a09b4b5..2e8798fb665 100644 --- a/libs/dms/recorder.go +++ b/libs/dms/recorder.go @@ -10,8 +10,10 @@ import ( "time" "github.com/databricks/cli/internal/build" + "github.com/databricks/cli/libs/auth" "github.com/databricks/cli/libs/log" "github.com/databricks/databricks-sdk-go/apierr" + "github.com/databricks/databricks-sdk-go/client" "github.com/databricks/databricks-sdk-go/service/bundledeployments" ) @@ -27,6 +29,54 @@ const ( VersionTypeDestroy VersionType = bundledeployments.VersionTypeVersionTypeDestroy ) +// createVersionRequest is the CreateVersion request body. +// +// The CLI builds the body itself instead of using bundledeployments.Version +// because the generated struct has no previous_version_id field, which the +// service requires as its concurrency check. Without it every deploy after the +// first is rejected. +type createVersionRequest struct { + CliVersion string `json:"cli_version"` + VersionType VersionType `json:"version_type"` + TargetName string `json:"target_name,omitempty"` + // DisplayName names the deployment in the UI. The service copies it onto the + // deployment's workspace node, which is where GetDeployment reads it from, so + // a version that omits it leaves the deployment unnamed. + DisplayName string `json:"display_name,omitempty"` + // PreviousVersionId is the deployment's most recent version, unset for a + // deployment's first version. + PreviousVersionId string `json:"previous_version_id,omitempty"` +} + +// versionCreator creates a version under a deployment. It exists because the +// generated client cannot express the request body (see createVersionRequest). +type versionCreator interface { + CreateVersion(ctx context.Context, deploymentID, versionID string, body createVersionRequest) (*bundledeployments.Version, error) +} + +// apiVersionCreator creates versions through the workspace API client. +type apiVersionCreator struct { + client *client.DatabricksClient +} + +// NewAPIVersionCreator returns a versionCreator that posts to the DMS API. +func NewAPIVersionCreator(c *client.DatabricksClient) versionCreator { + return &apiVersionCreator{client: c} +} + +func (a *apiVersionCreator) CreateVersion(ctx context.Context, deploymentID, versionID string, body createVersionRequest) (*bundledeployments.Version, error) { + var version bundledeployments.Version + path := fmt.Sprintf("/api/2.0/bundle/deployments/%s/versions", deploymentID) + err := a.client.Do(ctx, http.MethodPost, path, + auth.WorkspaceIDHeaders(a.client.Config), + map[string]any{"version_id": versionID}, + body, &version) + if err != nil { + return nil, err + } + return &version, nil +} + // Recorder records a single deploy/destroy as a version with DMS. // // The server assigns the deployment ID on the first deploy, i.e. when the ID @@ -35,9 +85,11 @@ const ( // node, so the next deploy starts over from empty. type Recorder struct { svc bundledeployments.BundleDeploymentsInterface + versions versionCreator deploymentID string statePath string targetName string + displayName string versionType VersionType // populated by CreateVersion @@ -45,18 +97,34 @@ type Recorder struct { stopHeartbeat context.CancelFunc } -// NewRecorder returns a Recorder for the given deployment. deploymentID is the -// ID resolved from the deployment's workspace node, or empty if this bundle has -// not yet recorded a deployment (the server assigns one during CreateVersion). -// statePath is the bundle's remote state directory, under which DMS registers -// the deployment node. -func NewRecorder(svc bundledeployments.BundleDeploymentsInterface, deploymentID, statePath, targetName string, versionType VersionType) *Recorder { +// RecorderOptions are the dependencies and deployment identity a Recorder needs. +type RecorderOptions struct { + // Service handles every DMS call except CreateVersion. + Service bundledeployments.BundleDeploymentsInterface + // Versions handles CreateVersion; see versionCreator. + Versions versionCreator + // DeploymentID is the ID resolved from the deployment's workspace node, or + // empty if this bundle has not recorded a deployment yet (the server assigns + // one during CreateVersion). + DeploymentID string + // StatePath is the bundle's remote state directory, under which DMS registers + // the deployment node. + StatePath string + TargetName string + DisplayName string + VersionType VersionType +} + +// NewRecorder returns a Recorder for the deployment described by opts. +func NewRecorder(opts RecorderOptions) *Recorder { return &Recorder{ - svc: svc, - deploymentID: deploymentID, - statePath: statePath, - targetName: targetName, - versionType: versionType, + svc: opts.Service, + versions: opts.Versions, + deploymentID: opts.DeploymentID, + statePath: opts.StatePath, + targetName: opts.TargetName, + displayName: opts.DisplayName, + versionType: opts.VersionType, } } @@ -149,6 +217,9 @@ func (r *Recorder) CompleteVersion(ctx context.Context, success bool) error { // the server assign the ID; otherwise it reads the existing deployment to // compute the next version number. func (r *Recorder) createDeploymentVersion(ctx context.Context) (versionID string, err error) { + // The version this one supersedes, sent as the concurrency check. Empty for a + // deployment's first version. + var previousVersionID string if r.deploymentID != "" { // A resolved node names the deployment, but its record is created by the // first version, so there may be none yet: a deploy that registered the @@ -165,6 +236,7 @@ func (r *Recorder) createDeploymentVersion(ctx context.Context) (versionID strin return "", fmt.Errorf("failed to parse last_version_id %q: %w", dep.LastVersionId, parseErr) } versionID = strconv.FormatInt(lastVersion+1, 10) + previousVersionID = dep.LastVersionId case errors.Is(getErr, apierr.ErrNotFound), errors.Is(getErr, apierr.ErrResourceDoesNotExist): versionID = "1" default: @@ -194,16 +266,15 @@ func (r *Recorder) createDeploymentVersion(ctx context.Context) (versionID strin versionID = "1" } - // The server validates that versionID equals last_version_id + 1 and returns - // ABORTED otherwise (e.g. a concurrent deploy already created this version). - version, versionErr := r.svc.CreateVersion(ctx, bundledeployments.CreateVersionRequest{ - Parent: "deployments/" + r.deploymentID, - VersionId: versionID, - Version: bundledeployments.Version{ - CliVersion: build.GetInfo().Version, - VersionType: r.versionType, - TargetName: r.targetName, - }, + // The server rejects the call unless versionID is numerically greater than + // last_version_id and previous_version_id matches it, so a deploy racing + // another is rejected rather than overwriting it. + version, versionErr := r.versions.CreateVersion(ctx, r.deploymentID, versionID, createVersionRequest{ + CliVersion: build.GetInfo().Version, + VersionType: r.versionType, + TargetName: r.targetName, + DisplayName: r.displayName, + PreviousVersionId: previousVersionID, }) if versionErr != nil { return "", fmt.Errorf("failed to create deployment version: %w", versionErr) diff --git a/libs/dms/recorder_test.go b/libs/dms/recorder_test.go index d4c7efaca03..f7695d62eb0 100644 --- a/libs/dms/recorder_test.go +++ b/libs/dms/recorder_test.go @@ -16,6 +16,10 @@ import ( // deployment node under; several tests assert it round-trips to the service. const testStatePath = "/Workspace/Users/me/.bundle/proj/dev/state" +// testDisplayName is the bundle name the recorder sends as the version's display +// name; the service copies it onto the deployment's workspace node. +const testDisplayName = "proj" + // fakeDMS records the calls the recorder makes and lets a test script the // server-side responses. It embeds the SDK interface so it satisfies it while // only overriding the methods the recorder uses. @@ -30,11 +34,30 @@ type fakeDMS struct { // captured requests created []bundledeployments.CreateDeploymentRequest - versions []bundledeployments.CreateVersionRequest + versions []fakeVersionRequest completed []bundledeployments.CompleteVersionRequest deleted []string } +// fakeVersionRequest is a CreateVersion call captured by fakeVersions. +type fakeVersionRequest struct { + deploymentID string + versionID string + body createVersionRequest +} + +// fakeVersions captures CreateVersion calls. It is separate from fakeDMS because +// the CLI does not create versions through the generated client (see +// createVersionRequest), so the two use different signatures. +type fakeVersions struct { + requests *[]fakeVersionRequest +} + +func (f fakeVersions) CreateVersion(ctx context.Context, deploymentID, versionID string, body createVersionRequest) (*bundledeployments.Version, error) { + *f.requests = append(*f.requests, fakeVersionRequest{deploymentID: deploymentID, versionID: versionID, body: body}) + return &bundledeployments.Version{VersionId: versionID}, nil +} + func (f *fakeDMS) CreateDeployment(ctx context.Context, req bundledeployments.CreateDeploymentRequest) (*bundledeployments.Deployment, error) { f.created = append(f.created, req) // The server always assigns the ID; it is the ID of the workspace node it @@ -47,11 +70,6 @@ func (f *fakeDMS) GetDeployment(ctx context.Context, req bundledeployments.GetDe return f.getDeployment(id) } -func (f *fakeDMS) CreateVersion(ctx context.Context, req bundledeployments.CreateVersionRequest) (*bundledeployments.Version, error) { - f.versions = append(f.versions, req) - return &bundledeployments.Version{VersionId: req.VersionId}, nil -} - func (f *fakeDMS) CompleteVersion(ctx context.Context, req bundledeployments.CompleteVersionRequest) (*bundledeployments.Version, error) { f.completed = append(f.completed, req) return &bundledeployments.Version{}, nil @@ -69,7 +87,7 @@ func (f *fakeDMS) Heartbeat(ctx context.Context, req bundledeployments.Heartbeat func TestRecorderFirstDeployCreatesDeploymentWithServerAssignedID(t *testing.T) { f := &fakeDMS{assignedID: "server-generated-id"} // A first deploy resolves no deployment ID from the workspace. - r := NewRecorder(f, "", testStatePath, "dev", VersionTypeDeploy) + r := NewRecorder(RecorderOptions{Service: f, Versions: fakeVersions{requests: &f.versions}, StatePath: testStatePath, TargetName: "dev", DisplayName: testDisplayName, VersionType: VersionTypeDeploy}) require.NoError(t, r.CreateVersion(t.Context())) @@ -83,8 +101,8 @@ func TestRecorderFirstDeployCreatesDeploymentWithServerAssignedID(t *testing.T) // The first version is 1, parented under the assigned deployment. require.Len(t, f.versions, 1) - assert.Equal(t, "1", f.versions[0].VersionId) - assert.Equal(t, "deployments/server-generated-id", f.versions[0].Parent) + assert.Equal(t, "1", f.versions[0].versionID) + assert.Equal(t, "server-generated-id", f.versions[0].deploymentID) assert.Equal(t, int64(1), r.Version()) require.NoError(t, r.CompleteVersion(t.Context(), true)) @@ -100,15 +118,33 @@ func TestRecorderSubsequentDeployReusesDeploymentAndIncrementsVersion(t *testing }, } // A subsequent deploy passes the stored deployment ID. - r := NewRecorder(f, "stored-id", testStatePath, "dev", VersionTypeDeploy) + r := NewRecorder(RecorderOptions{Service: f, Versions: fakeVersions{requests: &f.versions}, DeploymentID: "stored-id", StatePath: testStatePath, TargetName: "dev", DisplayName: testDisplayName, VersionType: VersionTypeDeploy}) require.NoError(t, r.CreateVersion(t.Context())) // No new deployment is created; the version increments to last_version_id + 1. assert.Empty(t, f.created) require.Len(t, f.versions, 1) - assert.Equal(t, "5", f.versions[0].VersionId) + assert.Equal(t, "5", f.versions[0].versionID) assert.Equal(t, "stored-id", r.DeploymentID()) + // The version it supersedes is the concurrency check; without it the service + // rejects every deploy after the first. + assert.Equal(t, "4", f.versions[0].body.PreviousVersionId) +} + +func TestRecorderSendsDisplayNameAndNoPreviousVersionOnFirstDeploy(t *testing.T) { + f := &fakeDMS{assignedID: "server-generated-id"} + r := NewRecorder(RecorderOptions{Service: f, Versions: fakeVersions{requests: &f.versions}, StatePath: testStatePath, TargetName: "dev", DisplayName: testDisplayName, VersionType: VersionTypeDeploy}) + + require.NoError(t, r.CreateVersion(t.Context())) + + require.Len(t, f.versions, 1) + // The service copies display_name onto the deployment's workspace node, which + // is where GetDeployment reads it from; a version that omits it leaves the + // deployment unnamed in the UI. + assert.Equal(t, testDisplayName, f.versions[0].body.DisplayName) + // A first version supersedes nothing, so previous_version_id is unset. + assert.Empty(t, f.versions[0].body.PreviousVersionId) } func TestRecorderGetDeploymentErrorFailsDeploy(t *testing.T) { @@ -117,7 +153,7 @@ func TestRecorderGetDeploymentErrorFailsDeploy(t *testing.T) { return nil, errors.New("boom") }, } - r := NewRecorder(f, "stored-id", testStatePath, "dev", VersionTypeDeploy) + r := NewRecorder(RecorderOptions{Service: f, Versions: fakeVersions{requests: &f.versions}, DeploymentID: "stored-id", StatePath: testStatePath, TargetName: "dev", DisplayName: testDisplayName, VersionType: VersionTypeDeploy}) err := r.CreateVersion(t.Context()) assert.ErrorContains(t, err, "failed to get deployment") @@ -134,13 +170,13 @@ func TestRecorderMissingDeploymentRecordStartsAtVersionOne(t *testing.T) { return nil, fmt.Errorf("deployment: %w", apierr.ErrNotFound) }, } - r := NewRecorder(f, "stored-id", testStatePath, "dev", VersionTypeDeploy) + r := NewRecorder(RecorderOptions{Service: f, Versions: fakeVersions{requests: &f.versions}, DeploymentID: "stored-id", StatePath: testStatePath, TargetName: "dev", DisplayName: testDisplayName, VersionType: VersionTypeDeploy}) require.NoError(t, r.CreateVersion(t.Context())) assert.Empty(t, f.created) require.Len(t, f.versions, 1) - assert.Equal(t, "1", f.versions[0].VersionId) - assert.Equal(t, "deployments/stored-id", f.versions[0].Parent) + assert.Equal(t, "1", f.versions[0].versionID) + assert.Equal(t, "stored-id", f.versions[0].deploymentID) } func TestRecorderDestroyDeletesDeploymentOnSuccess(t *testing.T) { @@ -149,10 +185,10 @@ func TestRecorderDestroyDeletesDeploymentOnSuccess(t *testing.T) { return &bundledeployments.Deployment{Name: "deployments/" + id, LastVersionId: "2"}, nil }, } - r := NewRecorder(f, "stored-id", testStatePath, "dev", VersionTypeDestroy) + r := NewRecorder(RecorderOptions{Service: f, Versions: fakeVersions{requests: &f.versions}, DeploymentID: "stored-id", StatePath: testStatePath, TargetName: "dev", DisplayName: testDisplayName, VersionType: VersionTypeDestroy}) require.NoError(t, r.CreateVersion(t.Context())) - assert.Equal(t, bundledeployments.VersionTypeVersionTypeDestroy, f.versions[0].Version.VersionType) + assert.Equal(t, bundledeployments.VersionTypeVersionTypeDestroy, f.versions[0].body.VersionType) require.NoError(t, r.CompleteVersion(t.Context(), true)) // A successful destroy deletes the deployment record. @@ -165,7 +201,7 @@ func TestRecorderFailedDestroyKeepsDeployment(t *testing.T) { return &bundledeployments.Deployment{Name: "deployments/" + id, LastVersionId: "2"}, nil }, } - r := NewRecorder(f, "stored-id", testStatePath, "dev", VersionTypeDestroy) + r := NewRecorder(RecorderOptions{Service: f, Versions: fakeVersions{requests: &f.versions}, DeploymentID: "stored-id", StatePath: testStatePath, TargetName: "dev", DisplayName: testDisplayName, VersionType: VersionTypeDestroy}) require.NoError(t, r.CreateVersion(t.Context())) require.NoError(t, r.CompleteVersion(t.Context(), false)) @@ -185,7 +221,7 @@ func TestNilRecorderIsNoOp(t *testing.T) { func TestRecorderCompleteVersionNoOpWithoutCreateVersion(t *testing.T) { f := &fakeDMS{} - r := NewRecorder(f, "stored-id", testStatePath, "dev", VersionTypeDeploy) + r := NewRecorder(RecorderOptions{Service: f, Versions: fakeVersions{requests: &f.versions}, DeploymentID: "stored-id", StatePath: testStatePath, TargetName: "dev", DisplayName: testDisplayName, VersionType: VersionTypeDeploy}) // CompleteVersion before CreateVersion is a no-op (nothing was claimed). require.NoError(t, r.CompleteVersion(t.Context(), true)) assert.Empty(t, f.completed) diff --git a/libs/testserver/bundle.go b/libs/testserver/bundle.go index 4e04c65ba81..5d91f47b693 100644 --- a/libs/testserver/bundle.go +++ b/libs/testserver/bundle.go @@ -142,6 +142,14 @@ func (s *FakeWorkspace) CreateVersion(req Request, deploymentID string) Response return Response{StatusCode: 400, Body: map[string]string{"message": err.Error()}} } + // previous_version_id is absent from the generated struct, so read it separately. + var concurrency struct { + PreviousVersionId string `json:"previous_version_id"` + } + if err := json.Unmarshal(req.Body, &concurrency); err != nil { + return Response{StatusCode: 400, Body: map[string]string{"message": err.Error()}} + } + defer s.LockUnlock()() d, ok := s.dmsDeployments[deploymentID] @@ -164,15 +172,22 @@ func (s *FakeWorkspace) CreateVersion(req Request, deploymentID string) Response s.dmsDeployments[deploymentID] = d } - // Mirror the server-side optimistic concurrency check: the new version must - // be exactly last_version_id + 1. - want := "1" + // Mirror the server-side checks: version_id must be numerically greater than + // the most recent version (not exactly one more), and previous_version_id must + // name that version, which is what detects a concurrent deploy. + next, err := strconv.ParseInt(versionID, 10, 64) + if err != nil || next < 1 { + return dmsInvalidArgument("version_id must be a positive integer, got " + versionID) + } + var last int64 if d.deployment.LastVersionId != "" { - last, _ := strconv.ParseInt(d.deployment.LastVersionId, 10, 64) - want = strconv.FormatInt(last+1, 10) + last, _ = strconv.ParseInt(d.deployment.LastVersionId, 10, 64) + } + if next <= last { + return dmsInvalidArgument("version_id " + versionID + " must be greater than the most recent version " + d.deployment.LastVersionId) } - if versionID != want { - return dmsAborted("expected version " + want + ", got " + versionID) + if concurrency.PreviousVersionId != d.deployment.LastVersionId { + return dmsAborted("previous_version_id is outdated; the deployment's most recent version is " + d.deployment.LastVersionId) } d.deployment.LastVersionId = versionID @@ -284,6 +299,14 @@ func dmsNotFound(what string) Response { // dmsAborted returns the 409 ABORTED error the server uses for the version // optimistic-concurrency check. +func dmsInvalidArgument(message string) Response { + return Response{ + StatusCode: 400, + Headers: map[string][]string{"Content-Type": {"application/json"}}, + Body: map[string]string{"error_code": "INVALID_PARAMETER_VALUE", "message": message}, + } +} + func dmsAborted(message string) Response { return Response{ StatusCode: 409, From 9acbeccd53d3b15ce5254409280c69711307167c Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 4 Aug 2026 10:35:52 +0000 Subject: [PATCH 031/125] bundle: fix recording a destroy Two problems, both only reachable on a real destroy: The delete operation sent no resource_id, so the service rejected it with "resource_id is required for OPERATION_ACTION_TYPE_DELETE operations" and the destroy failed. A delete carries no state, so resource_id is the only thing identifying the resource. It has to be read before the delete, which removes it from the local state. CompleteVersion then ran after files.Delete(). The deployment is a node under the state directory, so deleting the files deletes the deployment, and completing the version afterwards failed with 404. The destroy now completes the version before deleting the files; CompleteVersion is idempotent so Destroy can still defer it unconditionally. The test server accepted a delete without resource_id, which is why the acceptance tests passed while the real destroy failed. It now requires one. Verified on dogfood: deploy, redeploy, then destroy all succeed. Co-authored-by: Isaac --- acceptance/bundle/dms/record/output.txt | 1 + bundle/direct/bundle_apply.go | 5 ++++- bundle/phases/destroy.go | 13 +++++++++++-- libs/dms/recorder.go | 7 ++++++- libs/dms/recorder_test.go | 20 ++++++++++++++++++++ libs/testserver/bundle.go | 6 ++++++ 6 files changed, 48 insertions(+), 4 deletions(-) diff --git a/acceptance/bundle/dms/record/output.txt b/acceptance/bundle/dms/record/output.txt index f65d67d0991..b2cb6f240ed 100644 --- a/acceptance/bundle/dms/record/output.txt +++ b/acceptance/bundle/dms/record/output.txt @@ -123,6 +123,7 @@ Destroy complete! }, "body": { "action_type": "OPERATION_ACTION_TYPE_DELETE", + "resource_id": "[NUMID]", "resource_key": "jobs.foo", "status": "OPERATION_STATUS_SUCCEEDED" } diff --git a/bundle/direct/bundle_apply.go b/bundle/direct/bundle_apply.go index 46d70c7b135..4f7eb770bb2 100644 --- a/bundle/direct/bundle_apply.go +++ b/bundle/direct/bundle_apply.go @@ -93,6 +93,9 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa } if action == deployplan.Delete { + // Read the ID before the delete removes it from state; DMS requires it to + // identify which resource the delete operation refers to. + deletedID := b.StateDB.GetResourceID(resourceKey) if entry.Gone { // Planning confirmed the resource is already deleted remotely; only // remove it from the state, without calling the delete API. @@ -105,7 +108,7 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa return false } // Record the delete with DMS. State is nil: the resource is gone. - if err := opQueue.record(ctx, resourceKey, action, "", nil, nil); err != nil { + if err := opQueue.record(ctx, resourceKey, action, deletedID, nil, nil); err != nil { logdiag.LogError(ctx, fmt.Errorf("%s: %w", errorPrefix, err)) return false } diff --git a/bundle/phases/destroy.go b/bundle/phases/destroy.go index 2925e80bca8..c33b95110e2 100644 --- a/bundle/phases/destroy.go +++ b/bundle/phases/destroy.go @@ -82,7 +82,7 @@ func approvalForDestroy(ctx context.Context, b *bundle.Bundle, plan *deployplan. return cmdio.AskYesOrNo(ctx, "Would you like to proceed?") } -func destroyCore(ctx context.Context, b *bundle.Bundle, plan *deployplan.Plan, engine engine.EngineType) { +func destroyCore(ctx context.Context, b *bundle.Bundle, plan *deployplan.Plan, engine engine.EngineType, recorder *dms.Recorder) { if engine.IsDirect() { b.DeploymentBundle.Apply(ctx, b.WorkspaceClient(ctx), plan) } else { @@ -106,6 +106,15 @@ func destroyCore(ctx context.Context, b *bundle.Bundle, plan *deployplan.Plan, e return } + // Complete the version before deleting the remote files. The deployment is a + // node under the state directory, so files.Delete removes it and any later call + // fails with 404. CompleteVersion is idempotent, so the deferred call in Destroy + // is a no-op after this. + if err := recorder.CompleteVersion(ctx, true); err != nil { + logdiag.LogError(ctx, err) + return + } + bundle.ApplyContext(ctx, b, files.Delete()) if !logdiag.HasError(ctx) { @@ -215,7 +224,7 @@ func Destroy(ctx context.Context, b *bundle.Bundle, engine engine.EngineType) { recorder.Version(), ) } - destroyCore(ctx, b, plan, engine) + destroyCore(ctx, b, plan, engine, recorder) } else { cmdio.LogString(ctx, "Destroy cancelled!") } diff --git a/libs/dms/recorder.go b/libs/dms/recorder.go index 2e8798fb665..cc9fdd0d87f 100644 --- a/libs/dms/recorder.go +++ b/libs/dms/recorder.go @@ -95,6 +95,10 @@ type Recorder struct { // populated by CreateVersion versionNum int64 stopHeartbeat context.CancelFunc + + // completed makes CompleteVersion idempotent, so a caller that completes the + // version early can still defer it unconditionally. + completed bool } // RecorderOptions are the dependencies and deployment identity a Recorder needs. @@ -175,9 +179,10 @@ func (r *Recorder) CreateVersion(ctx context.Context) error { // is the check that keeps a cancelled or failed deploy from completing a version // that was never created. func (r *Recorder) CompleteVersion(ctx context.Context, success bool) error { - if r == nil || r.versionNum == 0 { + if r == nil || r.versionNum == 0 || r.completed { return nil } + r.completed = true r.stopHeartbeat() diff --git a/libs/dms/recorder_test.go b/libs/dms/recorder_test.go index f7695d62eb0..694fab0699a 100644 --- a/libs/dms/recorder_test.go +++ b/libs/dms/recorder_test.go @@ -211,6 +211,26 @@ func TestRecorderFailedDestroyKeepsDeployment(t *testing.T) { assert.Empty(t, f.deleted) } +func TestRecorderCompleteVersionIsIdempotent(t *testing.T) { + // Destroy completes the version before deleting the remote files, because that + // deletes the deployment's node, and still defers CompleteVersion. The second + // call must not reach the server, which would fail with 404. + f := &fakeDMS{ + getDeployment: func(id string) (*bundledeployments.Deployment, error) { + return &bundledeployments.Deployment{Name: "deployments/" + id, LastVersionId: "2"}, nil + }, + } + r := NewRecorder(RecorderOptions{Service: f, Versions: fakeVersions{requests: &f.versions}, DeploymentID: "stored-id", StatePath: testStatePath, TargetName: "dev", DisplayName: testDisplayName, VersionType: VersionTypeDestroy}) + + require.NoError(t, r.CreateVersion(t.Context())) + require.NoError(t, r.CompleteVersion(t.Context(), true)) + require.NoError(t, r.CompleteVersion(t.Context(), true)) + + assert.Len(t, f.completed, 1) + // The destroy deletes the deployment record once, not once per call. + assert.Equal(t, []string{"deployments/stored-id"}, f.deleted) +} + func TestNilRecorderIsNoOp(t *testing.T) { var r *Recorder assert.NoError(t, r.CreateVersion(t.Context())) diff --git a/libs/testserver/bundle.go b/libs/testserver/bundle.go index 5d91f47b693..eecf4153e3b 100644 --- a/libs/testserver/bundle.go +++ b/libs/testserver/bundle.go @@ -242,6 +242,12 @@ func (s *FakeWorkspace) CreateOperation(req Request, deploymentID, versionID str return dmsNotFound("deployment " + deploymentID) } + // A delete carries no state, so resource_id is the only thing identifying which + // resource it refers to; the service rejects a delete without one. + if op.ActionType == bundledeployments.OperationActionTypeOperationActionTypeDelete && op.ResourceId == "" { + return dmsInvalidArgument("resource_id is required for OPERATION_ACTION_TYPE_DELETE operations") + } + op.Name = "deployments/" + deploymentID + "/versions/" + versionID + "/operations/" + resourceKey op.ResourceKey = resourceKey From b34ed0a08b115103e9781712ca5a34cf1c26dd4d Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 4 Aug 2026 14:39:44 +0000 Subject: [PATCH 032/125] bundle: record failed operations, and upload at most 2 at a time Recording only covered resources that applied. A resource that failed was left out of the deployment history entirely, so the history said nothing about why a deploy failed - the one thing you want it for. DMS has error_message and a FAILED status for exactly this, and the CLI never used either. A failed operation carries no state: the resource was not written, so there is nothing to serve back as its state. The service does list the resource, but with no resource_id and no state, so a later deploy plans to create it rather than treating it as deployed. Verified against the service, and the test server now matches that shape rather than the shape I assumed. A message over the 16 KiB limit is truncated rather than rejected, since failing to record would hide the error being reported. Uploads drop from 4 workers to 2. Concurrent CreateOperation calls under one version contend on the version's operation_count, and the resulting transaction conflict is reported as a 500, which fails the deploy. Measured against the service: 3 concurrent writes succeeded, 4 did not, and 8 resources reliably failed. 2 keeps some overlap without reaching the conflict. The real fix is a batch upload API that commits every operation in one transaction; this is a stopgap until that exists. Verified on dogfood: a bundle with one good and one bad job records the good one as SUCCEEDED and the bad one as FAILED carrying the API's error, the version completes with VERSION_COMPLETE_FAILURE, and a plan from DMS state alone reports "1 to add, 1 unchanged". Co-authored-by: Isaac --- .../bundle/dms/record-failure/databricks.yml | 10 +++ .../bundle/dms/record-failure/out.test.toml | 3 + .../bundle/dms/record-failure/output.txt | 75 +++++++++++++++++++ acceptance/bundle/dms/record-failure/script | 15 ++++ .../bundle/dms/record-failure/test.toml | 6 ++ bundle/direct/bundle_apply.go | 4 + bundle/direct/opqueue.go | 38 +++++++++- bundle/direct/oprecorder.go | 54 +++++++++++-- bundle/direct/oprecorder_test.go | 21 ++++++ libs/testserver/bundle.go | 18 ++++- 10 files changed, 232 insertions(+), 12 deletions(-) create mode 100644 acceptance/bundle/dms/record-failure/databricks.yml create mode 100644 acceptance/bundle/dms/record-failure/out.test.toml create mode 100644 acceptance/bundle/dms/record-failure/output.txt create mode 100644 acceptance/bundle/dms/record-failure/script create mode 100644 acceptance/bundle/dms/record-failure/test.toml diff --git a/acceptance/bundle/dms/record-failure/databricks.yml b/acceptance/bundle/dms/record-failure/databricks.yml new file mode 100644 index 00000000000..8e8573fa70f --- /dev/null +++ b/acceptance/bundle/dms/record-failure/databricks.yml @@ -0,0 +1,10 @@ +bundle: + name: dms-record-failure + +experimental: + record_deployment_history: true + +resources: + jobs: + doomed: + name: doomed diff --git a/acceptance/bundle/dms/record-failure/out.test.toml b/acceptance/bundle/dms/record-failure/out.test.toml new file mode 100644 index 00000000000..e90b6d5d1ba --- /dev/null +++ b/acceptance/bundle/dms/record-failure/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/dms/record-failure/output.txt b/acceptance/bundle/dms/record-failure/output.txt new file mode 100644 index 00000000000..07b0c043f26 --- /dev/null +++ b/acceptance/bundle/dms/record-failure/output.txt @@ -0,0 +1,75 @@ + +=== A resource that fails to apply is recorded as a failed operation carrying the error, so the deployment history says why rather than omitting the resource +>>> musterr [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-record-failure/default/files... +Deploying resources... +Error: cannot create resources.jobs.doomed: cluster spec is invalid (400 INVALID_PARAMETER_VALUE) + +Endpoint: POST [DATABRICKS_URL]/api/2.2/jobs/create +HTTP Status: 400 Bad Request +API error_code: INVALID_PARAMETER_VALUE +API message: cluster spec is invalid + + +>>> print_requests.py //api/2.0/bundle --sort +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments", + "body": { + "initial_parent_path": "/Workspace/Users/[USERNAME]/.bundle/dms-record-failure/default/state", + "target_name": "default" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions", + "q": { + "version_id": "1" + }, + "body": { + "cli_version": "[CLI_VERSION]", + "version_type": "VERSION_TYPE_DEPLOY", + "target_name": "default", + "display_name": "dms-record-failure" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/complete", + "body": { + "completion_reason": "VERSION_COMPLETE_FAILURE" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations", + "q": { + "resource_key": "jobs.doomed" + }, + "body": { + "action_type": "OPERATION_ACTION_TYPE_CREATE", + "error_message": "cluster spec is invalid", + "resource_key": "jobs.doomed", + "status": "OPERATION_STATUS_FAILED" + } +} + +=== The failed resource is listed without a resource_id or state, so a later deploy plans to create it rather than treating it as already deployed +>>> [CLI] api get /api/2.0/bundle/deployments/[NUMID]/resources +{ + "resources": [ + { + "last_action_type": "OPERATION_ACTION_TYPE_CREATE", + "last_version_id": "1", + "name": "deployments/[NUMID]/resources/jobs.doomed", + "resource_key": "jobs.doomed", + "resource_type": "" + } + ] +} + +=== Redeploying from DMS state alone still creates the failed resource: it was never applied, so it must not be skipped as unchanged +>>> [CLI] bundle plan +create jobs.doomed + +Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged diff --git a/acceptance/bundle/dms/record-failure/script b/acceptance/bundle/dms/record-failure/script new file mode 100644 index 00000000000..6a1890973a6 --- /dev/null +++ b/acceptance/bundle/dms/record-failure/script @@ -0,0 +1,15 @@ +title "A resource that fails to apply is recorded as a failed operation carrying the error, so the deployment history says why rather than omitting the resource" +trace musterr $CLI bundle deploy +trace print_requests.py //api/2.0/bundle --sort + +title "The failed resource is listed without a resource_id or state, so a later deploy plans to create it rather than treating it as already deployed" +# The deployment ID is the workspace node's ID; read it back the way the CLI does. +deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-record-failure/default/state/resources.deployment.json" -o json | jq -r .object_id) +trace $CLI api get "/api/2.0/bundle/deployments/${deployment_id}/resources" + +title "Redeploying from DMS state alone still creates the failed resource: it was never applied, so it must not be skipped as unchanged" +rm -rf .databricks +trace $CLI bundle plan +# plan probes the state files concurrently, so the recorded order varies. Only the +# operations above are asserted; drop the rest rather than diff a racy order. +rm -f out.requests.txt diff --git a/acceptance/bundle/dms/record-failure/test.toml b/acceptance/bundle/dms/record-failure/test.toml new file mode 100644 index 00000000000..bf8710439b1 --- /dev/null +++ b/acceptance/bundle/dms/record-failure/test.toml @@ -0,0 +1,6 @@ +# The job cannot be created, so applying it fails and the operation is recorded as +# failed rather than omitted. +[[Server]] +Pattern = "POST /api/2.2/jobs/create" +Response.StatusCode = 400 +Response.Body = '''{"error_code": "INVALID_PARAMETER_VALUE", "message": "cluster spec is invalid"}''' diff --git a/bundle/direct/bundle_apply.go b/bundle/direct/bundle_apply.go index 4f7eb770bb2..bdd0c243e7f 100644 --- a/bundle/direct/bundle_apply.go +++ b/bundle/direct/bundle_apply.go @@ -104,6 +104,7 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa err = d.Destroy(ctx, &b.StateDB) } if err != nil { + opQueue.recordFailure(ctx, resourceKey, action, deletedID, err) logdiag.LogError(ctx, fmt.Errorf("%s: %w", errorPrefix, err)) return false } @@ -137,6 +138,9 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa // TODO: redo calcDiff to downgrade planned action if possible (?) err = d.Deploy(ctx, &b.StateDB, sv.Value, action, entry) if err != nil { + // GetResourceID is empty for a create that never got an ID, which is + // what the service expects for a failed create. + opQueue.recordFailure(ctx, resourceKey, action, b.StateDB.GetResourceID(resourceKey), err) logdiag.LogError(ctx, fmt.Errorf("%s: %w", errorPrefix, err)) return false } diff --git a/bundle/direct/opqueue.go b/bundle/direct/opqueue.go index 48d2887ec7c..c3f4439ee4e 100644 --- a/bundle/direct/opqueue.go +++ b/bundle/direct/opqueue.go @@ -18,7 +18,13 @@ const ( // operationUploadWorkers is how many uploads run at a time. It is below // operationQueueSize so a burst of operations is absorbed by the queue rather // than by one request per resource. - operationUploadWorkers = 4 + // + // Capped at 2 because concurrent CreateOperation calls under the same version + // contend on shared state server-side and the transaction conflict surfaces as + // a 500, which fails the deploy. Measured against the service: 3 concurrent + // writes still succeeded, 4 did not. The real fix is a batch upload API that + // commits every operation in one transaction; until then, keep this at 2. + operationUploadWorkers = 2 ) // operationQueue hands recorded operations to background workers, so an apply @@ -125,6 +131,33 @@ func (q *operationQueue) record(ctx context.Context, resourceKey string, action return err } + q.enqueue(ctx, resourceKey, op) + return nil +} + +// recordFailure records that applying a resource failed, so the deployment +// history explains the failure instead of omitting the resource. +// +// Unlike record, this does not resurface an earlier upload error: the deploy is +// already failing, and returning a different error here would replace the one the +// user needs to see. A failure to upload this record is reported at close. +func (q *operationQueue) recordFailure(ctx context.Context, resourceKey string, action deployplan.ActionType, resourceID string, cause error) { + if q == nil { + return + } + + op, err := newFailedOperation(action, resourceID, cause) + if err != nil { + log.Warnf(ctx, "Not recording failure for %s: %s", resourceKey, err) + return + } + + q.enqueue(ctx, resourceKey, op) +} + +// enqueue publishes op as the pending operation for resourceKey and makes sure a +// worker will pick it up. +func (q *operationQueue) enqueue(ctx context.Context, resourceKey string, op recordedOperation) { q.mu.Lock() _, replaced := q.pending[resourceKey] q.pending[resourceKey] = op @@ -140,11 +173,10 @@ func (q *operationQueue) record(ctx context.Context, resourceKey string, action // finishing, so it will see the operation written above. Queueing the key again // would let a second worker upload the same resource concurrently. if alreadyHandled { - return nil + return } q.queue <- resourceKey - return nil } // close drains the queue and returns the first upload error. All callers of diff --git a/bundle/direct/oprecorder.go b/bundle/direct/oprecorder.go index a1a5cf641bc..a4595a89c08 100644 --- a/bundle/direct/oprecorder.go +++ b/bundle/direct/oprecorder.go @@ -16,6 +16,12 @@ import ( // message that names the resource. const maxOperationStateSize = 64 * 1024 +// maxOperationErrorMessageSize is the largest error message DMS accepts per +// operation. A longer message is truncated rather than rejected, so a failing +// resource is still recorded with its error instead of the recording itself +// failing and masking the error we are trying to report. +const maxOperationErrorMessageSize = 16 * 1024 + // recordedOperation is an applied resource operation, serialized and waiting to be // uploaded to the deployment metadata service (DMS). // @@ -25,9 +31,15 @@ const maxOperationStateSize = 64 * 1024 type recordedOperation struct { action bundledeployments.OperationActionType resourceID string + status bundledeployments.OperationStatus + + // errorMessage is why the operation failed. It is set only when status is + // failed, which the service enforces. + errorMessage string // state is the serialized local config after the operation. It is nil for a - // delete, where the resource no longer exists. + // delete, where the resource no longer exists, and for a failure, where the + // resource was not written. state json.RawMessage } @@ -40,7 +52,11 @@ func newRecordedOperation(action deployplan.ActionType, resourceID string, state return recordedOperation{}, err } - op := recordedOperation{action: actionType, resourceID: resourceID} + op := recordedOperation{ + action: actionType, + resourceID: resourceID, + status: bundledeployments.OperationStatusOperationStatusSucceeded, + } // Operation.State carries the serialized state, which DMS serves back as // resource state. Unset for delete: the resource is gone. @@ -62,6 +78,31 @@ func newRecordedOperation(action deployplan.ActionType, resourceID string, state return op, nil } +// newFailedOperation records an operation that did not apply, so the deployment +// history says why a resource failed rather than just omitting it. +// +// No state is recorded: the resource was not written, so there is nothing to +// serve back as its state. CREATE and RECREATE may have no resourceID yet, which +// the service allows for exactly those two actions. +func newFailedOperation(action deployplan.ActionType, resourceID string, cause error) (recordedOperation, error) { + actionType, err := deployActionToSDK(action) + if err != nil { + return recordedOperation{}, err + } + + message := cause.Error() + if len(message) > maxOperationErrorMessageSize { + message = message[:maxOperationErrorMessageSize] + } + + return recordedOperation{ + action: actionType, + resourceID: resourceID, + status: bundledeployments.OperationStatusOperationStatusFailed, + errorMessage: message, + }, nil +} + // operationUploader records an applied resource operation with DMS. Uploads run // on the operationQueue workers, off the apply path. type operationUploader interface { @@ -93,10 +134,11 @@ func (r *operationRecorder) upload(ctx context.Context, resourceKey string, op r dmsKey := strings.TrimPrefix(resourceKey, "resources.") operation := bundledeployments.Operation{ - ActionType: op.action, - ResourceId: op.resourceID, - ResourceKey: dmsKey, - Status: bundledeployments.OperationStatusOperationStatusSucceeded, + ActionType: op.action, + ResourceId: op.resourceID, + ResourceKey: dmsKey, + Status: op.status, + ErrorMessage: op.errorMessage, } if op.state != nil { // DMS types state as a string, so the JSON goes on the wire as a quoted diff --git a/bundle/direct/oprecorder_test.go b/bundle/direct/oprecorder_test.go index 674c78abf77..a76a91edcdb 100644 --- a/bundle/direct/oprecorder_test.go +++ b/bundle/direct/oprecorder_test.go @@ -2,6 +2,8 @@ package direct import ( "context" + "errors" + "strings" "sync" "testing" @@ -85,6 +87,25 @@ func TestNewRecordedOperationRejectsUnsupportedAction(t *testing.T) { assert.Error(t, err) } +func TestNewFailedOperationRecordsError(t *testing.T) { + op, err := newFailedOperation(deployplan.Create, "", errors.New("cluster spec is invalid")) + require.NoError(t, err) + + assert.Equal(t, bundledeployments.OperationStatusOperationStatusFailed, op.status) + assert.Equal(t, "cluster spec is invalid", op.errorMessage) + // The resource was never written, so there is no state to serve back for it. + assert.Nil(t, op.state) +} + +func TestNewFailedOperationTruncatesLongError(t *testing.T) { + // Truncated rather than rejected: a message over the limit would make recording + // fail and hide the error it is reporting. + op, err := newFailedOperation(deployplan.Update, "job-123", errors.New(strings.Repeat("x", maxOperationErrorMessageSize+100))) + require.NoError(t, err) + + assert.Len(t, op.errorMessage, maxOperationErrorMessageSize) +} + func TestDeployActionToSDK(t *testing.T) { cases := []struct { action deployplan.ActionType diff --git a/libs/testserver/bundle.go b/libs/testserver/bundle.go index eecf4153e3b..72a3ae257e0 100644 --- a/libs/testserver/bundle.go +++ b/libs/testserver/bundle.go @@ -243,19 +243,31 @@ func (s *FakeWorkspace) CreateOperation(req Request, deploymentID, versionID str } // A delete carries no state, so resource_id is the only thing identifying which - // resource it refers to; the service rejects a delete without one. + // resource it refers to; the service rejects a delete without one. A failed + // delete is exempt only for create-flavored actions, which may not have an ID. if op.ActionType == bundledeployments.OperationActionTypeOperationActionTypeDelete && op.ResourceId == "" { return dmsInvalidArgument("resource_id is required for OPERATION_ACTION_TYPE_DELETE operations") } + failed := op.Status == bundledeployments.OperationStatusOperationStatusFailed + if !failed && op.ErrorMessage != "" { + return dmsInvalidArgument("error_message is only allowed when status is OPERATION_STATUS_FAILED") + } + op.Name = "deployments/" + deploymentID + "/versions/" + versionID + "/operations/" + resourceKey op.ResourceKey = resourceKey // Reflect the operation onto the deployment-level resource set the way the // backend does: a delete removes the resource, anything else upserts it. - if op.ActionType == bundledeployments.OperationActionTypeOperationActionTypeDelete { + // + // A failed operation is upserted too, matching the service, but it carries + // neither a resource_id nor state, so the read path treats the resource as not + // yet created rather than as existing state (verified against the service: a + // failed create is listed with an empty resource_id and no state). + switch { + case op.ActionType == bundledeployments.OperationActionTypeOperationActionTypeDelete && !failed: delete(d.resources, resourceKey) - } else { + default: d.resources[resourceKey] = bundledeployments.Resource{ Name: "deployments/" + deploymentID + "/resources/" + resourceKey, ResourceKey: resourceKey, From b9ca9a3ae287d519f2f843e58bed084ce64aa56a Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 5 Aug 2026 09:38:20 +0000 Subject: [PATCH 033/125] bundle: report the recorded deployment in bundle summary The deployment metadata service assigns a deployment an ID and a version per deploy, but nothing surfaced either, so a caller had no way to go from a bundle to the deployment it recorded. Summary now reports both under bundle.deployment.history. The ID comes from the deployment's workspace node, which is where the CLI already resolves it from; the version comes from GetDeployment. Both are output only, so the field is annotated readonly and stays out of the user-facing JSON schema. Like InitializeURLs, the mutator makes extra API calls and only runs when the fields are needed, and it is a no-op unless the bundle records deployment history, so bundles that do not record pay nothing. A deployment whose record does not exist yet (a deploy that registered it and then failed before recording a version) reports the ID without a version rather than failing summary. The test server was returning its 404 without a JSON content-type, so the SDK could not parse it into a typed error and callers matching apierr.ErrResourceDoesNotExist saw a generic failure instead. Also fixes bundle/dms/record-failure, which read a 19-digit object ID through jq: 1.6 rounds it, so the test only passed against a newer jq than CI runs. Verified on dogfood: summary reports the deployment ID and version 1, then version 2 after a redeploy, matching GetDeployment; a bundle without recording has no history field at all. Co-authored-by: Isaac --- acceptance/bundle/dms/record-failure/script | 3 +- acceptance/bundle/dms/summary/databricks.yml | 10 +++ acceptance/bundle/dms/summary/out.test.toml | 3 + acceptance/bundle/dms/summary/output.txt | 39 +++++++++++ acceptance/bundle/dms/summary/script | 15 ++++ bundle/config/deployment.go | 16 +++++ .../mutator/initialize_deployment_history.go | 68 +++++++++++++++++++ .../initialize_deployment_history_test.go | 36 ++++++++++ cmd/bundle/utils/process.go | 3 +- libs/testserver/bundle.go | 3 + 10 files changed, 194 insertions(+), 2 deletions(-) create mode 100644 acceptance/bundle/dms/summary/databricks.yml create mode 100644 acceptance/bundle/dms/summary/out.test.toml create mode 100644 acceptance/bundle/dms/summary/output.txt create mode 100644 acceptance/bundle/dms/summary/script create mode 100644 bundle/config/mutator/initialize_deployment_history.go create mode 100644 bundle/config/mutator/initialize_deployment_history_test.go diff --git a/acceptance/bundle/dms/record-failure/script b/acceptance/bundle/dms/record-failure/script index 6a1890973a6..66af9601402 100644 --- a/acceptance/bundle/dms/record-failure/script +++ b/acceptance/bundle/dms/record-failure/script @@ -4,7 +4,8 @@ trace print_requests.py //api/2.0/bundle --sort title "The failed resource is listed without a resource_id or state, so a later deploy plans to create it rather than treating it as already deployed" # The deployment ID is the workspace node's ID; read it back the way the CLI does. -deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-record-failure/default/state/resources.deployment.json" -o json | jq -r .object_id) +# Extracted with python, not jq: the ID exceeds 2^53 and jq 1.6 rounds it. +deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-record-failure/default/state/resources.deployment.json" -o json | python3 -c 'import sys,json; print(json.load(sys.stdin)["object_id"])') trace $CLI api get "/api/2.0/bundle/deployments/${deployment_id}/resources" title "Redeploying from DMS state alone still creates the failed resource: it was never applied, so it must not be skipped as unchanged" diff --git a/acceptance/bundle/dms/summary/databricks.yml b/acceptance/bundle/dms/summary/databricks.yml new file mode 100644 index 00000000000..c0698376223 --- /dev/null +++ b/acceptance/bundle/dms/summary/databricks.yml @@ -0,0 +1,10 @@ +bundle: + name: dms-summary + +experimental: + record_deployment_history: true + +resources: + jobs: + foo: + name: foo diff --git a/acceptance/bundle/dms/summary/out.test.toml b/acceptance/bundle/dms/summary/out.test.toml new file mode 100644 index 00000000000..e90b6d5d1ba --- /dev/null +++ b/acceptance/bundle/dms/summary/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/dms/summary/output.txt b/acceptance/bundle/dms/summary/output.txt new file mode 100644 index 00000000000..c792a60fcfa --- /dev/null +++ b/acceptance/bundle/dms/summary/output.txt @@ -0,0 +1,39 @@ + +=== Summary reports the deployment recorded with the metadata service, so a caller can find the deployment and the version it is on +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-summary/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> [CLI] bundle summary -o json +{ + "deployment_id": "[NUMID]", + "latest_version_id": "1" +} + +=== Redeploying advances the version the summary reports +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-summary/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> [CLI] bundle summary -o json +{ + "deployment_id": "[NUMID]", + "latest_version_id": "2" +} + +=== After a destroy the deployment is gone, so the summary reports no history rather than a dangling ID +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.jobs.foo + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/dms-summary/default + +Deleting files... +Destroy complete! + +>>> [CLI] bundle summary -o json +false diff --git a/acceptance/bundle/dms/summary/script b/acceptance/bundle/dms/summary/script new file mode 100644 index 00000000000..6f44e7c2880 --- /dev/null +++ b/acceptance/bundle/dms/summary/script @@ -0,0 +1,15 @@ +title "Summary reports the deployment recorded with the metadata service, so a caller can find the deployment and the version it is on" +trace $CLI bundle deploy +trace $CLI bundle summary -o json | jq .bundle.deployment.history + +title "Redeploying advances the version the summary reports" +trace $CLI bundle deploy +trace $CLI bundle summary -o json | jq .bundle.deployment.history + +title "After a destroy the deployment is gone, so the summary reports no history rather than a dangling ID" +trace $CLI bundle destroy --auto-approve +trace $CLI bundle summary -o json | jq '.bundle.deployment | has("history")' + +# This test asserts the summary output, not the requests behind it; the file uploads +# recorded here are ordered nondeterministically. +rm -f out.requests.txt diff --git a/bundle/config/deployment.go b/bundle/config/deployment.go index b7efb4456f9..b59d1b1da02 100644 --- a/bundle/config/deployment.go +++ b/bundle/config/deployment.go @@ -7,4 +7,20 @@ type Deployment struct { // Lock configures locking behavior on deployment. Lock Lock `json:"lock,omitempty"` + + // History reports what the deployment metadata service has recorded for this + // bundle. Output only: it is read from the service for 'bundle summary' and is + // unset when the bundle does not record deployment history. + History *DeploymentHistory `json:"history,omitempty" bundle:"readonly"` +} + +// DeploymentHistory identifies the bundle's deployment in the deployment +// metadata service. +type DeploymentHistory struct { + // DeploymentID is the ID the service assigned to this bundle's deployment. + DeploymentID string `json:"deployment_id,omitempty"` + + // LatestVersionID is the most recent version recorded for the deployment. It is + // unset when the deployment exists but has no version yet. + LatestVersionID string `json:"latest_version_id,omitempty"` } diff --git a/bundle/config/mutator/initialize_deployment_history.go b/bundle/config/mutator/initialize_deployment_history.go new file mode 100644 index 00000000000..91a3f1352c9 --- /dev/null +++ b/bundle/config/mutator/initialize_deployment_history.go @@ -0,0 +1,68 @@ +package mutator + +import ( + "context" + "errors" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dms" + "github.com/databricks/cli/libs/log" + "github.com/databricks/databricks-sdk-go/apierr" + "github.com/databricks/databricks-sdk-go/service/bundledeployments" +) + +type initializeDeploymentHistory struct{} + +// InitializeDeploymentHistory populates bundle.deployment.history with the +// deployment recorded by the deployment metadata service, for the output of the +// 'bundle summary' command. +// +// NOTE: this makes extra API calls, so like InitializeURLs it should only be used +// when the fields are needed. It is a no-op unless the bundle records deployment +// history. +func InitializeDeploymentHistory() bundle.Mutator { + return &initializeDeploymentHistory{} +} + +func (m *initializeDeploymentHistory) Name() string { + return "InitializeDeploymentHistory" +} + +func (m *initializeDeploymentHistory) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { + if b.Config.Experimental == nil || !b.Config.Experimental.RecordDeploymentHistory { + return nil + } + + w := b.WorkspaceClient(ctx) + deploymentID, err := dms.ResolveDeploymentID(ctx, w, b.Config.Workspace.StatePath) + if err != nil { + return diag.FromErr(err) + } + if deploymentID == "" { + // Nothing recorded yet: the bundle has not been deployed, or its deployment + // was destroyed. + return nil + } + + history := &config.DeploymentHistory{DeploymentID: deploymentID} + + // The deployment's record is created by its first version, so a resolved ID can + // name a deployment that has none yet (a deploy that registered the deployment + // and then failed). Report the ID without a version rather than failing summary. + dep, err := w.BundleDeployments.GetDeployment(ctx, bundledeployments.GetDeploymentRequest{ + Name: "deployments/" + deploymentID, + }) + switch { + case err == nil: + history.LatestVersionID = dep.LastVersionId + case errors.Is(err, apierr.ErrNotFound), errors.Is(err, apierr.ErrResourceDoesNotExist): + log.Debugf(ctx, "No deployment record for %s yet; reporting the ID without a version", deploymentID) + default: + return diag.FromErr(err) + } + + b.Config.Bundle.Deployment.History = history + return nil +} diff --git a/bundle/config/mutator/initialize_deployment_history_test.go b/bundle/config/mutator/initialize_deployment_history_test.go new file mode 100644 index 00000000000..a0478819bfd --- /dev/null +++ b/bundle/config/mutator/initialize_deployment_history_test.go @@ -0,0 +1,36 @@ +package mutator + +import ( + "testing" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestInitializeDeploymentHistoryIsNoOpWithoutRecording(t *testing.T) { + // Without recording there is no deployment to report, and the mutator must not + // make the API calls that would find one. + cases := []struct { + name string + experimental *config.Experimental + }{ + {"experimental unset", nil}, + {"recording disabled", &config.Experimental{RecordDeploymentHistory: false}}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + b := &bundle.Bundle{ + Config: config.Root{ + Experimental: tc.experimental, + }, + } + + diags := bundle.ApplySeq(t.Context(), b, InitializeDeploymentHistory()) + require.NoError(t, diags.Error()) + assert.Nil(t, b.Config.Bundle.Deployment.History) + }) + } +} diff --git a/cmd/bundle/utils/process.go b/cmd/bundle/utils/process.go index 209dc874403..30eefcac586 100644 --- a/cmd/bundle/utils/process.go +++ b/cmd/bundle/utils/process.go @@ -258,8 +258,9 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle statemgmt.Load(state, modes...), } // InitializeURLs makes an extra API call; only run it when URLs are needed. + // InitializeDeploymentHistory likewise, and only for bundles that record it. if opts.InitIDs { - mutators = append(mutators, mutator.InitializeURLs()) + mutators = append(mutators, mutator.InitializeURLs(), mutator.InitializeDeploymentHistory()) } bundle.ApplySeqContext(ctx, b, mutators...) if logdiag.HasError(ctx) { diff --git a/libs/testserver/bundle.go b/libs/testserver/bundle.go index 72a3ae257e0..698e0bab79f 100644 --- a/libs/testserver/bundle.go +++ b/libs/testserver/bundle.go @@ -308,6 +308,9 @@ func (s *FakeWorkspace) ListResources(deploymentID string) Response { func dmsNotFound(what string) Response { return Response{ StatusCode: 404, + // Content-Type is required for the SDK to parse the body into a typed error, + // which is what callers match against apierr.ErrResourceDoesNotExist. + Headers: map[string][]string{"Content-Type": {"application/json"}}, Body: map[string]string{ "error_code": "RESOURCE_DOES_NOT_EXIST", "message": what + " does not exist", From 1a05f7a6290b8fdf58b49f099b805abb7ec3fbc8 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 5 Aug 2026 11:14:43 +0000 Subject: [PATCH 034/125] bundle: record git, workspace and mode provenance with a version A version reported only the CLI version, target and display name, so a deployment carried nothing about the source it came from. The service already has fields for this and denormalizes them onto the deployment, so all three were silently empty for every recorded deploy. The mapping mirrors what bundle/deploy/metadata computes for the metadata file, including its two conditionals: a source-linked deployment reports the sync root as file_path, and git_folder_path is set only for a deploy from a Databricks Git folder. bundle_root_path is relative to git_folder_path, so it is sent with it or not at all - the service rejects one without the other, which a local git deploy hit. Fixes a second bug found the same way: a deployment whose first version was rejected still leaves the record behind, with an empty last_version_id. createDeploymentVersion parsed that unconditionally and failed with "failed to parse last_version_id" on every later deploy, leaving the bundle permanently unable to record. It now retries at version 1, the same way it handles a record that does not exist yet. The test server now denormalizes provenance onto the deployment and enforces the git_folder_path/bundle_root_path pairing, so acceptance tests observe the real contract rather than a shape only the fake accepts. Verified on dogfood from a real git repo (branch master, origin bundle-examples): GetDeployment reports deployment_mode DEPLOYMENT_MODE_DEVELOPMENT, git_info with branch/commit/origin_url, and workspace_info with root_path and file_path. Co-authored-by: Isaac --- .../bundle/dms/existing-state/output.txt | 2 +- .../bundle/dms/multiple-resources/output.txt | 2 +- acceptance/bundle/dms/no-resources/output.txt | 12 +++- .../bundle/dms/provenance/databricks.yml | 15 ++++ .../bundle/dms/provenance/out.test.toml | 3 + acceptance/bundle/dms/provenance/output.txt | 69 +++++++++++++++++++ acceptance/bundle/dms/provenance/script | 15 ++++ acceptance/bundle/dms/provenance/test.toml | 4 ++ .../bundle/dms/record-failure/output.txt | 6 +- acceptance/bundle/dms/record/output.txt | 18 ++++- .../dms/redeploy-after-destroy/output.txt | 6 +- .../dms/version-never-created/output.txt | 4 +- bundle/phases/dms.go | 52 ++++++++++++++ libs/dms/recorder.go | 27 ++++++++ libs/testserver/bundle.go | 15 ++++ 15 files changed, 239 insertions(+), 11 deletions(-) create mode 100644 acceptance/bundle/dms/provenance/databricks.yml create mode 100644 acceptance/bundle/dms/provenance/out.test.toml create mode 100644 acceptance/bundle/dms/provenance/output.txt create mode 100644 acceptance/bundle/dms/provenance/script create mode 100644 acceptance/bundle/dms/provenance/test.toml diff --git a/acceptance/bundle/dms/existing-state/output.txt b/acceptance/bundle/dms/existing-state/output.txt index e7bb54440b5..a8923c7f36d 100644 --- a/acceptance/bundle/dms/existing-state/output.txt +++ b/acceptance/bundle/dms/existing-state/output.txt @@ -47,6 +47,6 @@ Deployment complete! >>> print_requests.py //api/2.0/bundle --oneline {"method": "POST", "path": "/api/2.0/bundle/deployments", "body": {"initial_parent_path": "/Workspace/Users/[USERNAME]/.bundle/dms-existing-state/default/state", "target_name": "default"}} -{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": {"version_id": "1"}, "body": {"cli_version": "[CLI_VERSION]", "version_type": "VERSION_TYPE_DEPLOY", "target_name": "default", "display_name": "dms-existing-state"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": {"version_id": "1"}, "body": {"cli_version": "[CLI_VERSION]", "version_type": "VERSION_TYPE_DEPLOY", "target_name": "default", "display_name": "dms-existing-state", "workspace_info": {"file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-existing-state/default/files", "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-existing-state/default"}}} {"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations", "q": {"resource_key": "jobs.one"}, "body": {"action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_id": "[NUMID]", "resource_key": "jobs.one", "state": "{\"state\":{\"deployment\":{\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-existing-state/default/state/metadata.json\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"one\",\"queue\":{\"enabled\":true}}}", "status": "OPERATION_STATUS_SUCCEEDED"}} {"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/complete", "body": {"completion_reason": "VERSION_COMPLETE_SUCCESS"}} diff --git a/acceptance/bundle/dms/multiple-resources/output.txt b/acceptance/bundle/dms/multiple-resources/output.txt index 1d208e1e85e..b593998dbd1 100644 --- a/acceptance/bundle/dms/multiple-resources/output.txt +++ b/acceptance/bundle/dms/multiple-resources/output.txt @@ -21,5 +21,5 @@ Updating deployment state... Deployment complete! >>> print_requests.py //api/2.0/bundle --sort --oneline -{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": {"version_id": "2"}, "body": {"cli_version": "[CLI_VERSION]", "version_type": "VERSION_TYPE_DEPLOY", "target_name": "default", "display_name": "dms-multiple-resources", "previous_version_id": "1"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": {"version_id": "2"}, "body": {"cli_version": "[CLI_VERSION]", "version_type": "VERSION_TYPE_DEPLOY", "target_name": "default", "display_name": "dms-multiple-resources", "previous_version_id": "1", "workspace_info": {"file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources/default/files", "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources/default"}}} {"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/complete", "body": {"completion_reason": "VERSION_COMPLETE_SUCCESS"}} diff --git a/acceptance/bundle/dms/no-resources/output.txt b/acceptance/bundle/dms/no-resources/output.txt index 06e346f3029..8801d4179b1 100644 --- a/acceptance/bundle/dms/no-resources/output.txt +++ b/acceptance/bundle/dms/no-resources/output.txt @@ -24,7 +24,11 @@ Deployment complete! "cli_version": "[CLI_VERSION]", "version_type": "VERSION_TYPE_DEPLOY", "target_name": "default", - "display_name": "dms-no-resources" + "display_name": "dms-no-resources", + "workspace_info": { + "file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-no-resources/default/files", + "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-no-resources/default" + } } } { @@ -67,7 +71,11 @@ Deployment complete! "version_type": "VERSION_TYPE_DEPLOY", "target_name": "default", "display_name": "dms-no-resources", - "previous_version_id": "1" + "previous_version_id": "1", + "workspace_info": { + "file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-no-resources/default/files", + "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-no-resources/default" + } } } { diff --git a/acceptance/bundle/dms/provenance/databricks.yml b/acceptance/bundle/dms/provenance/databricks.yml new file mode 100644 index 00000000000..df3b2d033c4 --- /dev/null +++ b/acceptance/bundle/dms/provenance/databricks.yml @@ -0,0 +1,15 @@ +bundle: + name: dms-provenance + +experimental: + record_deployment_history: true + +targets: + dev: + default: true + mode: development + +resources: + jobs: + foo: + name: foo diff --git a/acceptance/bundle/dms/provenance/out.test.toml b/acceptance/bundle/dms/provenance/out.test.toml new file mode 100644 index 00000000000..e90b6d5d1ba --- /dev/null +++ b/acceptance/bundle/dms/provenance/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/dms/provenance/output.txt b/acceptance/bundle/dms/provenance/output.txt new file mode 100644 index 00000000000..ed6035fb6eb --- /dev/null +++ b/acceptance/bundle/dms/provenance/output.txt @@ -0,0 +1,69 @@ + +=== Deploying from a git repo records where the source came from: the version carries git_info, workspace_info and the target's deployment_mode +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-provenance/dev/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> print_requests.py //versions --sort +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions", + "q": { + "version_id": "1" + }, + "body": { + "cli_version": "[CLI_VERSION]", + "version_type": "VERSION_TYPE_DEPLOY", + "target_name": "dev", + "display_name": "dms-provenance", + "deployment_mode": "DEPLOYMENT_MODE_DEVELOPMENT", + "git_info": { + "branch": "main", + "commit": "[COMMIT]", + "origin_url": "https://github.com/databricks/bundle-examples.git" + }, + "workspace_info": { + "file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-provenance/dev/files", + "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-provenance/dev" + } + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/complete", + "body": { + "completion_reason": "VERSION_COMPLETE_SUCCESS" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations", + "q": { + "resource_key": "jobs.foo" + }, + "body": { + "action_type": "OPERATION_ACTION_TYPE_CREATE", + "resource_id": "[NUMID]", + "resource_key": "jobs.foo", + "state": "{\"state\":{\"deployment\":{\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-provenance/dev/state/metadata.json\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":4,\"name\":\"[dev [USERNAME]] foo\",\"queue\":{\"enabled\":true},\"tags\":{\"dev\":\"[USERNAME]\"}}}", + "status": "OPERATION_STATUS_SUCCEEDED" + } +} + +=== The service denormalizes them onto the deployment, so a reader gets the provenance of the latest version +>>> [CLI] api get /api/2.0/bundle/deployments/[NUMID] +{ + "target_name": "dev", + "deployment_mode": "DEPLOYMENT_MODE_DEVELOPMENT", + "git_info": { + "branch": "main", + "commit": "[COMMIT]", + "origin_url": "https://github.com/databricks/bundle-examples.git" + }, + "workspace_info": { + "file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-provenance/dev/files", + "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-provenance/dev" + } +} diff --git a/acceptance/bundle/dms/provenance/script b/acceptance/bundle/dms/provenance/script new file mode 100644 index 00000000000..eb36baf6067 --- /dev/null +++ b/acceptance/bundle/dms/provenance/script @@ -0,0 +1,15 @@ +title "Deploying from a git repo records where the source came from: the version carries git_info, workspace_info and the target's deployment_mode" +git-repo-init +git remote add origin https://github.com/databricks/bundle-examples.git +trace $CLI bundle deploy +# The commit SHA changes every run, so assert it is a 40-char hex string and drop it. +add_repl.py "$(git rev-parse HEAD)" COMMIT +trace print_requests.py //versions --sort + +title "The service denormalizes them onto the deployment, so a reader gets the provenance of the latest version" +deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-provenance/dev/state/resources.deployment.json" -o json | python3 -c 'import sys,json; print(json.load(sys.stdin)["object_id"])') +trace $CLI api get "/api/2.0/bundle/deployments/${deployment_id}" | jq '{target_name, deployment_mode, git_info, workspace_info}' + +# The deploy uploads files in a nondeterministic order; only the requests above are +# asserted. +rm -f out.requests.txt diff --git a/acceptance/bundle/dms/provenance/test.toml b/acceptance/bundle/dms/provenance/test.toml new file mode 100644 index 00000000000..0a47bfb1b91 --- /dev/null +++ b/acceptance/bundle/dms/provenance/test.toml @@ -0,0 +1,4 @@ +# git-repo-init creates a repo in the test directory so the deploy has git provenance. +Ignore = [ + '.git', +] diff --git a/acceptance/bundle/dms/record-failure/output.txt b/acceptance/bundle/dms/record-failure/output.txt index 07b0c043f26..cb4073087d7 100644 --- a/acceptance/bundle/dms/record-failure/output.txt +++ b/acceptance/bundle/dms/record-failure/output.txt @@ -30,7 +30,11 @@ API message: cluster spec is invalid "cli_version": "[CLI_VERSION]", "version_type": "VERSION_TYPE_DEPLOY", "target_name": "default", - "display_name": "dms-record-failure" + "display_name": "dms-record-failure", + "workspace_info": { + "file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-record-failure/default/files", + "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-record-failure/default" + } } } { diff --git a/acceptance/bundle/dms/record/output.txt b/acceptance/bundle/dms/record/output.txt index b2cb6f240ed..f436470e073 100644 --- a/acceptance/bundle/dms/record/output.txt +++ b/acceptance/bundle/dms/record/output.txt @@ -25,7 +25,11 @@ Deployment complete! "cli_version": "[CLI_VERSION]", "version_type": "VERSION_TYPE_DEPLOY", "target_name": "default", - "display_name": "dms-record" + "display_name": "dms-record", + "workspace_info": { + "file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-record/default/files", + "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-record/default" + } } } { @@ -79,7 +83,11 @@ Deployment complete! "version_type": "VERSION_TYPE_DEPLOY", "target_name": "default", "display_name": "dms-record", - "previous_version_id": "1" + "previous_version_id": "1", + "workspace_info": { + "file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-record/default/files", + "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-record/default" + } } } { @@ -112,7 +120,11 @@ Destroy complete! "version_type": "VERSION_TYPE_DESTROY", "target_name": "default", "display_name": "dms-record", - "previous_version_id": "2" + "previous_version_id": "2", + "workspace_info": { + "file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-record/default/files", + "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-record/default" + } } } { diff --git a/acceptance/bundle/dms/redeploy-after-destroy/output.txt b/acceptance/bundle/dms/redeploy-after-destroy/output.txt index 6e160478e7b..33aa1938db1 100644 --- a/acceptance/bundle/dms/redeploy-after-destroy/output.txt +++ b/acceptance/bundle/dms/redeploy-after-destroy/output.txt @@ -50,7 +50,11 @@ Deployment complete! "cli_version": "[CLI_VERSION]", "version_type": "VERSION_TYPE_DEPLOY", "target_name": "default", - "display_name": "dms-redeploy-after-destroy" + "display_name": "dms-redeploy-after-destroy", + "workspace_info": { + "file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-redeploy-after-destroy/default/files", + "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-redeploy-after-destroy/default" + } } } { diff --git a/acceptance/bundle/dms/version-never-created/output.txt b/acceptance/bundle/dms/version-never-created/output.txt index 23f84279395..e87ed786023 100644 --- a/acceptance/bundle/dms/version-never-created/output.txt +++ b/acceptance/bundle/dms/version-never-created/output.txt @@ -28,7 +28,7 @@ API message: Internal error >>> print_requests.py //api/2.0/bundle --get --oneline {"method": "POST", "path": "/api/2.0/bundle/deployments", "body": {"initial_parent_path": "/Workspace/Users/[USERNAME]/.bundle/dms-version-never-created/default/state", "target_name": "default"}} -{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": {"version_id": "1"}, "body": {"cli_version": "[CLI_VERSION]", "version_type": "VERSION_TYPE_DEPLOY", "target_name": "default", "display_name": "dms-version-never-created"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": {"version_id": "1"}, "body": {"cli_version": "[CLI_VERSION]", "version_type": "VERSION_TYPE_DEPLOY", "target_name": "default", "display_name": "dms-version-never-created", "workspace_info": {"file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-version-never-created/default/files", "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-version-never-created/default"}}} {"method": "GET", "path": "/api/2.0/bundle/deployments/[NUMID]/resources"} {"method": "GET", "path": "/api/2.0/bundle/deployments/[NUMID]"} -{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": {"version_id": "1"}, "body": {"cli_version": "[CLI_VERSION]", "version_type": "VERSION_TYPE_DEPLOY", "target_name": "default", "display_name": "dms-version-never-created"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": {"version_id": "1"}, "body": {"cli_version": "[CLI_VERSION]", "version_type": "VERSION_TYPE_DEPLOY", "target_name": "default", "display_name": "dms-version-never-created", "workspace_info": {"file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-version-never-created/default/files", "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-version-never-created/default"}}} diff --git a/bundle/phases/dms.go b/bundle/phases/dms.go index 346d6ca70c5..8e204208813 100644 --- a/bundle/phases/dms.go +++ b/bundle/phases/dms.go @@ -2,11 +2,14 @@ package phases import ( "context" + "strings" "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/engine" "github.com/databricks/cli/libs/dms" "github.com/databricks/databricks-sdk-go/client" + "github.com/databricks/databricks-sdk-go/service/bundledeployments" ) // newDeploymentRecorder returns a dms.Recorder for the current deployment, or @@ -46,5 +49,54 @@ func newDeploymentRecorder(ctx context.Context, b *bundle.Bundle, eng engine.Eng TargetName: b.Config.Bundle.Target, DisplayName: b.Config.Bundle.Name, VersionType: versionType, + Provenance: deploymentProvenance(b), }), nil } + +// deploymentProvenance describes the source this deploy came from and where it +// landed, mirroring what bundle/deploy/metadata computes for the metadata file. +func deploymentProvenance(b *bundle.Bundle) dms.Provenance { + p := dms.Provenance{Mode: deploymentModeToSDK(b.Config.Bundle.Mode)} + + git := b.Config.Bundle.Git + if git.Branch != "" || git.Commit != "" || git.OriginURL != "" { + p.Git = &bundledeployments.GitInfo{ + Branch: git.Branch, + Commit: git.Commit, + OriginUrl: git.OriginURL, + } + } + + ws := &bundledeployments.WorkspaceInfo{ + RootPath: b.Config.Workspace.RootPath, + FilePath: b.Config.Workspace.FilePath, + } + // In a source-linked deployment files are not copied, so resources read them + // from the sync root instead of file_path (see bundle/deploy/metadata.Compute). + if config.IsExplicitlyEnabled(b.Config.Presets.SourceLinkedDeployment) { + ws.FilePath = b.SyncRootPath + ws.SourceLinked = true + } + // Only a deploy from a Databricks Git folder has one; a local worktree does not. + // bundle_root_path is relative to it, so the service requires both or neither. + if b.WorktreeRoot != nil && strings.HasPrefix(b.WorktreeRoot.Native(), "/Workspace/") { + ws.GitFolderPath = b.WorktreeRoot.Native() + ws.BundleRootPath = git.BundleRootPath + } + p.Workspace = ws + + return p +} + +// deploymentModeToSDK maps the bundle target's mode to the DMS enum. An unset mode +// maps to empty, which the service reads as "not reported". +func deploymentModeToSDK(mode config.Mode) bundledeployments.DeploymentMode { + switch mode { + case config.Development: + return bundledeployments.DeploymentModeDeploymentModeDevelopment + case config.Production: + return bundledeployments.DeploymentModeDeploymentModeProduction + default: + return "" + } +} diff --git a/libs/dms/recorder.go b/libs/dms/recorder.go index cc9fdd0d87f..0306507913a 100644 --- a/libs/dms/recorder.go +++ b/libs/dms/recorder.go @@ -46,6 +46,12 @@ type createVersionRequest struct { // PreviousVersionId is the deployment's most recent version, unset for a // deployment's first version. PreviousVersionId string `json:"previous_version_id,omitempty"` + // DeploymentMode is the bundle target's mode, unset when the target sets none. + DeploymentMode bundledeployments.DeploymentMode `json:"deployment_mode,omitempty"` + // GitInfo and WorkspaceInfo record where the deployed source came from and + // where it landed. The service denormalizes both onto the deployment. + GitInfo *bundledeployments.GitInfo `json:"git_info,omitempty"` + WorkspaceInfo *bundledeployments.WorkspaceInfo `json:"workspace_info,omitempty"` } // versionCreator creates a version under a deployment. It exists because the @@ -91,6 +97,7 @@ type Recorder struct { targetName string displayName string versionType VersionType + provenance Provenance // populated by CreateVersion versionNum int64 @@ -117,6 +124,18 @@ type RecorderOptions struct { TargetName string DisplayName string VersionType VersionType + // Provenance records where the deployed source came from; see Provenance. + Provenance Provenance +} + +// Provenance is what a version records about the source it deployed and where it +// landed. The service denormalizes these onto the deployment, so they describe the +// deployment as of its most recent version. +type Provenance struct { + // Mode is the bundle target's mode, empty when the target sets none. + Mode bundledeployments.DeploymentMode + Git *bundledeployments.GitInfo + Workspace *bundledeployments.WorkspaceInfo } // NewRecorder returns a Recorder for the deployment described by opts. @@ -129,6 +148,7 @@ func NewRecorder(opts RecorderOptions) *Recorder { targetName: opts.TargetName, displayName: opts.DisplayName, versionType: opts.VersionType, + provenance: opts.Provenance, } } @@ -235,6 +255,10 @@ func (r *Recorder) createDeploymentVersion(ctx context.Context) (versionID strin Name: "deployments/" + r.deploymentID, }) switch { + case getErr == nil && dep.LastVersionId == "": + // The record exists but carries no version: a deploy whose first version was + // rejected still leaves the record behind. Retry at version 1. + versionID = "1" case getErr == nil: lastVersion, parseErr := strconv.ParseInt(dep.LastVersionId, 10, 64) if parseErr != nil { @@ -280,6 +304,9 @@ func (r *Recorder) createDeploymentVersion(ctx context.Context) (versionID strin TargetName: r.targetName, DisplayName: r.displayName, PreviousVersionId: previousVersionID, + DeploymentMode: r.provenance.Mode, + GitInfo: r.provenance.Git, + WorkspaceInfo: r.provenance.Workspace, }) if versionErr != nil { return "", fmt.Errorf("failed to create deployment version: %w", versionErr) diff --git a/libs/testserver/bundle.go b/libs/testserver/bundle.go index 698e0bab79f..4f2571691a2 100644 --- a/libs/testserver/bundle.go +++ b/libs/testserver/bundle.go @@ -190,11 +190,26 @@ func (s *FakeWorkspace) CreateVersion(req Request, deploymentID string) Response return dmsAborted("previous_version_id is outdated; the deployment's most recent version is " + d.deployment.LastVersionId) } + // bundle_root_path is relative to git_folder_path, so the service rejects a + // workspace_info that carries one without the other. + if ws := version.WorkspaceInfo; ws != nil && (ws.GitFolderPath == "") != (ws.BundleRootPath == "") { + return dmsInvalidArgument("workspace_info.git_folder_path and workspace_info.bundle_root_path must be set together") + } + d.deployment.LastVersionId = versionID version.Name = "deployments/" + deploymentID + "/versions/" + versionID version.VersionId = versionID version.Status = bundledeployments.VersionStatusVersionStatusInProgress d.versions[versionID] = &version + + // The service denormalizes the version's provenance onto the deployment, which + // is where the read APIs serve it from. display_name is excluded: the service + // keeps that on the deployment's workspace node instead. + d.deployment.TargetName = version.TargetName + d.deployment.DeploymentMode = version.DeploymentMode + d.deployment.GitInfo = version.GitInfo + d.deployment.WorkspaceInfo = version.WorkspaceInfo + return Response{Body: version} } From c08abbdd51784f945bb27f5936a6a91e284a02b1 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 5 Aug 2026 22:31:58 +0000 Subject: [PATCH 035/125] bundle: stamp the deployment and version onto jobs and pipelines AnnotateJobs/AnnotatePipelines set deployment.kind and metadata_file_path, but not deployment_id or version_id, so a job or pipeline in the workspace had no way back to the deployment that produced it. That link is what lineage resolves to attribute a job to its bundle. The version has to exist before the resources are planned: the plan snapshots the resource config, and apply deploys from that snapshot, so stamping after the plan never reaches the API. Verified: with the stamp applied post-plan the deployed job still had only kind and metadata_file_path. CreateVersion therefore moves ahead of planning, which means a cancelled deploy now leaves a version behind - completed as a failure by the deferred CompleteVersion, the same as any other failed deploy. This drops the incidental stale-plan guard the old ordering gave us: CreateVersion ran after approval, so a concurrent deploy that advanced the deployment made the ABORTED check reject a stale plan. previous_version_id still detects the race, just at claim time rather than apply time. Verified on dogfood via the raw API (the SDK hides these preview fields): the job reports deployment_id 3908151894982320 / version_id 2 and the pipeline the same deployment with version_id 5, and that ID resolves to the real DMS deployment. Co-authored-by: Isaac --- acceptance/bundle/dms/depends-on/output.txt | 4 +- .../bundle/dms/existing-state/output.txt | 2 +- acceptance/bundle/dms/provenance/output.txt | 2 +- acceptance/bundle/dms/record/output.txt | 2 +- .../dms/redeploy-after-destroy/output.txt | 2 +- .../metadata/annotate_deployment_version.go | 47 +++++++++++++++++ .../annotate_deployment_version_test.go | 50 +++++++++++++++++++ bundle/phases/deploy.go | 29 +++++++---- 8 files changed, 121 insertions(+), 17 deletions(-) create mode 100644 bundle/deploy/metadata/annotate_deployment_version.go create mode 100644 bundle/deploy/metadata/annotate_deployment_version_test.go diff --git a/acceptance/bundle/dms/depends-on/output.txt b/acceptance/bundle/dms/depends-on/output.txt index db3624d9fc1..952dcb00a86 100644 --- a/acceptance/bundle/dms/depends-on/output.txt +++ b/acceptance/bundle/dms/depends-on/output.txt @@ -17,7 +17,7 @@ Deployment complete! "action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_id": "[NUMID]", "resource_key": "jobs.child", - "state": "{\"state\":{\"deployment\":{\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-depends-on/default/state/metadata.json\"},\"description\":\"depends on [NUMID]\",\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"child\",\"queue\":{\"enabled\":true}},\"depends_on\":[{\"node\":\"resources.jobs.parent\",\"label\":\"${resources.jobs.parent.id}\"}]}", + "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-depends-on/default/state/metadata.json\",\"version_id\":\"1\"},\"description\":\"depends on [NUMID]\",\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"child\",\"queue\":{\"enabled\":true}},\"depends_on\":[{\"node\":\"resources.jobs.parent\",\"label\":\"${resources.jobs.parent.id}\"}]}", "status": "OPERATION_STATUS_SUCCEEDED" } } @@ -31,7 +31,7 @@ Deployment complete! "action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_id": "[NUMID]", "resource_key": "jobs.parent", - "state": "{\"state\":{\"deployment\":{\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-depends-on/default/state/metadata.json\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"parent\",\"queue\":{\"enabled\":true}}}", + "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-depends-on/default/state/metadata.json\",\"version_id\":\"1\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"parent\",\"queue\":{\"enabled\":true}}}", "status": "OPERATION_STATUS_SUCCEEDED" } } diff --git a/acceptance/bundle/dms/existing-state/output.txt b/acceptance/bundle/dms/existing-state/output.txt index a8923c7f36d..1897f6c4e86 100644 --- a/acceptance/bundle/dms/existing-state/output.txt +++ b/acceptance/bundle/dms/existing-state/output.txt @@ -48,5 +48,5 @@ Deployment complete! >>> print_requests.py //api/2.0/bundle --oneline {"method": "POST", "path": "/api/2.0/bundle/deployments", "body": {"initial_parent_path": "/Workspace/Users/[USERNAME]/.bundle/dms-existing-state/default/state", "target_name": "default"}} {"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": {"version_id": "1"}, "body": {"cli_version": "[CLI_VERSION]", "version_type": "VERSION_TYPE_DEPLOY", "target_name": "default", "display_name": "dms-existing-state", "workspace_info": {"file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-existing-state/default/files", "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-existing-state/default"}}} -{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations", "q": {"resource_key": "jobs.one"}, "body": {"action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_id": "[NUMID]", "resource_key": "jobs.one", "state": "{\"state\":{\"deployment\":{\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-existing-state/default/state/metadata.json\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"one\",\"queue\":{\"enabled\":true}}}", "status": "OPERATION_STATUS_SUCCEEDED"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations", "q": {"resource_key": "jobs.one"}, "body": {"action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_id": "[NUMID]", "resource_key": "jobs.one", "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-existing-state/default/state/metadata.json\",\"version_id\":\"1\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"one\",\"queue\":{\"enabled\":true}}}", "status": "OPERATION_STATUS_SUCCEEDED"}} {"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/complete", "body": {"completion_reason": "VERSION_COMPLETE_SUCCESS"}} diff --git a/acceptance/bundle/dms/provenance/output.txt b/acceptance/bundle/dms/provenance/output.txt index ed6035fb6eb..c567a12d66c 100644 --- a/acceptance/bundle/dms/provenance/output.txt +++ b/acceptance/bundle/dms/provenance/output.txt @@ -47,7 +47,7 @@ Deployment complete! "action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_id": "[NUMID]", "resource_key": "jobs.foo", - "state": "{\"state\":{\"deployment\":{\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-provenance/dev/state/metadata.json\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":4,\"name\":\"[dev [USERNAME]] foo\",\"queue\":{\"enabled\":true},\"tags\":{\"dev\":\"[USERNAME]\"}}}", + "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-provenance/dev/state/metadata.json\",\"version_id\":\"1\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":4,\"name\":\"[dev [USERNAME]] foo\",\"queue\":{\"enabled\":true},\"tags\":{\"dev\":\"[USERNAME]\"}}}", "status": "OPERATION_STATUS_SUCCEEDED" } } diff --git a/acceptance/bundle/dms/record/output.txt b/acceptance/bundle/dms/record/output.txt index f436470e073..a707e5e46d7 100644 --- a/acceptance/bundle/dms/record/output.txt +++ b/acceptance/bundle/dms/record/output.txt @@ -42,7 +42,7 @@ Deployment complete! "action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_id": "[NUMID]", "resource_key": "jobs.foo", - "state": "{\"state\":{\"deployment\":{\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-record/default/state/metadata.json\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"foo\",\"queue\":{\"enabled\":true}}}", + "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-record/default/state/metadata.json\",\"version_id\":\"1\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"foo\",\"queue\":{\"enabled\":true}}}", "status": "OPERATION_STATUS_SUCCEEDED" } } diff --git a/acceptance/bundle/dms/redeploy-after-destroy/output.txt b/acceptance/bundle/dms/redeploy-after-destroy/output.txt index 33aa1938db1..12ce1a298c4 100644 --- a/acceptance/bundle/dms/redeploy-after-destroy/output.txt +++ b/acceptance/bundle/dms/redeploy-after-destroy/output.txt @@ -67,7 +67,7 @@ Deployment complete! "action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_id": "[NUMID]", "resource_key": "jobs.foo", - "state": "{\"state\":{\"deployment\":{\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-redeploy-after-destroy/default/state/metadata.json\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"foo\",\"queue\":{\"enabled\":true}}}", + "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-redeploy-after-destroy/default/state/metadata.json\",\"version_id\":\"1\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"foo\",\"queue\":{\"enabled\":true}}}", "status": "OPERATION_STATUS_SUCCEEDED" } } diff --git a/bundle/deploy/metadata/annotate_deployment_version.go b/bundle/deploy/metadata/annotate_deployment_version.go new file mode 100644 index 00000000000..1b8139168ce --- /dev/null +++ b/bundle/deploy/metadata/annotate_deployment_version.go @@ -0,0 +1,47 @@ +package metadata + +import ( + "context" + "strconv" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/libs/diag" +) + +type annotateDeploymentVersion struct { + deploymentID string + version int64 +} + +// AnnotateDeploymentVersion stamps the DMS deployment and version onto every job +// and pipeline, so a resource in the workspace points back at the deployment that +// produced it (which is how lineage resolves a job to its bundle). +// +// AnnotateJobs/AnnotatePipelines set the rest of the deployment metadata during +// initialize, but the version - and, on a first deploy, the deployment ID - only +// exist once CreateVersion has run, so these two fields are stamped separately +// from the deploy phase. +func AnnotateDeploymentVersion(deploymentID string, version int64) bundle.Mutator { + return &annotateDeploymentVersion{deploymentID: deploymentID, version: version} +} + +func (m *annotateDeploymentVersion) Name() string { + return "metadata.AnnotateDeploymentVersion" +} + +func (m *annotateDeploymentVersion) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { + versionID := strconv.FormatInt(m.version, 10) + + for _, job := range b.Config.Resources.Jobs { + // Deployment is set by AnnotateJobs, which runs during initialize. + job.Deployment.DeploymentId = m.deploymentID + job.Deployment.VersionId = versionID + } + + for _, pipeline := range b.Config.Resources.Pipelines { + pipeline.Deployment.DeploymentId = m.deploymentID + pipeline.Deployment.VersionId = versionID + } + + return nil +} diff --git a/bundle/deploy/metadata/annotate_deployment_version_test.go b/bundle/deploy/metadata/annotate_deployment_version_test.go new file mode 100644 index 00000000000..aca51358ace --- /dev/null +++ b/bundle/deploy/metadata/annotate_deployment_version_test.go @@ -0,0 +1,50 @@ +package metadata + +import ( + "testing" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/databricks-sdk-go/service/jobs" + "github.com/databricks/databricks-sdk-go/service/pipelines" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestAnnotateDeploymentVersion(t *testing.T) { + b := &bundle.Bundle{ + Config: config.Root{ + Resources: config.Resources{ + Jobs: map[string]*resources.Job{ + "my-job": { + JobSettings: jobs.JobSettings{ + Deployment: &jobs.JobDeployment{Kind: jobs.JobDeploymentKindBundle}, + }, + }, + }, + Pipelines: map[string]*resources.Pipeline{ + "my-pipeline": { + CreatePipeline: pipelines.CreatePipeline{ + Deployment: &pipelines.PipelineDeployment{Kind: pipelines.DeploymentKindBundle}, + }, + }, + }, + }, + }, + } + + diags := bundle.ApplySeq(t.Context(), b, AnnotateDeploymentVersion("dep-123", 7)) + require.NoError(t, diags.Error()) + + job := b.Config.Resources.Jobs["my-job"].Deployment + assert.Equal(t, "dep-123", job.DeploymentId) + assert.Equal(t, "7", job.VersionId) + // The kind set by AnnotateJobs is preserved. + assert.Equal(t, jobs.JobDeploymentKindBundle, job.Kind) + + pipeline := b.Config.Resources.Pipelines["my-pipeline"].Deployment + assert.Equal(t, "dep-123", pipeline.DeploymentId) + assert.Equal(t, "7", pipeline.VersionId) + assert.Equal(t, pipelines.DeploymentKindBundle, pipeline.Kind) +} diff --git a/bundle/phases/deploy.go b/bundle/phases/deploy.go index 3d70a218b15..808ca6c012c 100644 --- a/bundle/phases/deploy.go +++ b/bundle/phases/deploy.go @@ -228,6 +228,22 @@ func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHand return } + // Create the version before planning: the plan snapshots the resource config, + // so the deployment and version have to be stamped onto the resources before it + // is computed or the applied resources would not carry them. A cancelled deploy + // therefore leaves a version behind, completed as a failure by the deferred + // CompleteVersion. + if err := recorder.CreateVersion(ctx); err != nil { + logdiag.LogError(ctx, err) + return + } + if recorder != nil { + bundle.ApplyContext(ctx, b, metadata.AnnotateDeploymentVersion(recorder.DeploymentID(), recorder.Version())) + if logdiag.HasError(ctx) { + return + } + } + planFromFile := plan != nil if plan == nil { // State is already open for read by process.go (for direct engine) @@ -271,18 +287,9 @@ func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHand return } if haveApproval { - // Record the DMS version now that the plan is approved and the state WAL - // has been opened. CreateVersion requests version_id == last_version_id + 1; - // the server returns ABORTED if a concurrent deploy advanced the deployment - // since the plan was computed, so a stale plan is not applied. - if err := recorder.CreateVersion(ctx); err != nil { - logdiag.LogError(ctx, err) - return - } if recorder != nil { - // Record operations under the version just created so DMS holds the - // deployed resource state. On a first deploy the deployment ID was only - // assigned by CreateVersion above, so this must come after it. + // Record operations under the version created before planning, so DMS holds + // the deployed resource state. b.DeploymentBundle.OpRec = direct.NewOperationRecorder( b.WorkspaceClient(ctx).BundleDeployments, recorder.DeploymentID(), From 41c41eee510e546dd7719ee895f949d70b34337b Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Thu, 6 Aug 2026 08:31:53 +0000 Subject: [PATCH 036/125] bundle: raise operation upload workers back to 8 The cap of 2 was a stopgap for a service-side limit: concurrent CreateOperation calls under one version contended on the version row, and the transaction conflict surfaced as a 500 that failed the deploy. That is fixed, so the cap can go. Verified against the service rather than assumed. The raw-API probe that originally pinned the threshold now reports 8/8 and 16/16 concurrent writes succeeding, where 4 used to fail and 8 gave 3/8. End to end at 8 workers: the 12-job bundle that previously failed on jobs.j04 deploys cleanly, 35 jobs deploy and destroy with zero 500s, and a plan from DMS state alone after wiping .databricks reports "0 to add, 12 to change" - every operation recorded, nothing duplicated. Left at 8 rather than higher: the probe shows headroom at 16, but 8 matches the apply-side parallelism, and a batch upload API remains the better way to cut the request count. Co-authored-by: Isaac --- bundle/direct/opqueue.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/bundle/direct/opqueue.go b/bundle/direct/opqueue.go index c3f4439ee4e..551c91b2874 100644 --- a/bundle/direct/opqueue.go +++ b/bundle/direct/opqueue.go @@ -19,12 +19,12 @@ const ( // operationQueueSize so a burst of operations is absorbed by the queue rather // than by one request per resource. // - // Capped at 2 because concurrent CreateOperation calls under the same version - // contend on shared state server-side and the transaction conflict surfaces as - // a 500, which fails the deploy. Measured against the service: 3 concurrent - // writes still succeeded, 4 did not. The real fix is a batch upload API that - // commits every operation in one transaction; until then, keep this at 2. - operationUploadWorkers = 2 + // This was temporarily capped at 2 while concurrent CreateOperation calls under + // the same version contended on shared state server-side, surfacing the + // transaction conflict as a 500 that failed the deploy. The service now handles + // them: measured against it, 8 and 16 concurrent writes both succeed where 4 + // used to fail. + operationUploadWorkers = 8 ) // operationQueue hands recorded operations to background workers, so an apply From af8c851d81410fc06f8c695850d85025a09c273f Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Thu, 6 Aug 2026 10:05:39 +0000 Subject: [PATCH 037/125] bundle: spell out how to recover from the record-deployment-history error The old remedy read "Remove experimental.record_deployment_history, or destroy the bundle and deploy it again", which is misleading in two ways. It reads as two alternatives when the second one is really three ordered steps, and following it as written does not work: this error also fires on destroy, so a user who tries to "destroy the bundle and deploy it again" hits the same error and cannot get out. The setting has to come off first, which the old wording never said. It also did not say what the first option costs. Removing the setting is not a fix for recording, it is the choice to keep the existing resources and not record them. The message now numbers the three steps in the order they have to happen, and states the keep-the-resources option separately so the two outcomes are not confused. Verified on dogfood by reproducing the report: a bundle deployed without recording, then with the setting added, fails destroy with the new message; following the three steps destroys the bundle and then deploys with recording enabled. Co-authored-by: Isaac --- .../bundle/dms/existing-state/output.txt | 18 ++++++++++++++++-- bundle/direct/dstate/state.go | 11 ++++++++++- 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/acceptance/bundle/dms/existing-state/output.txt b/acceptance/bundle/dms/existing-state/output.txt index 1897f6c4e86..6233f158974 100644 --- a/acceptance/bundle/dms/existing-state/output.txt +++ b/acceptance/bundle/dms/existing-state/output.txt @@ -12,7 +12,14 @@ Deployment complete! >>> update_file.py databricks.yml record_deployment_history: false record_deployment_history: true >>> musterr [CLI] bundle deploy -Error: cannot record deployment history for a bundle that already has deployed resources tracked in [TEST_TMP_DIR]/.databricks/bundle/default/resources.json: only new deployments can be recorded. Remove experimental.record_deployment_history, or destroy the bundle and deploy it again +Error: cannot record deployment history for a bundle that already has deployed resources tracked in [TEST_TMP_DIR]/.databricks/bundle/default/resources.json: only new deployments can be recorded + +To record this bundle's history, start it over as a new deployment: + 1. remove experimental.record_deployment_history from your bundle configuration + 2. run "databricks bundle destroy" to delete the existing resources + 3. add experimental.record_deployment_history back and deploy again + +To keep the existing resources instead, leave experimental.record_deployment_history out === No deployment was created in DMS @@ -20,7 +27,14 @@ Error: cannot record deployment history for a bundle that already has deployed r === Still an error after wiping the local cache: deploy pulls the state file back from the workspace, so the resources are still tracked >>> musterr [CLI] bundle deploy -Error: cannot record deployment history for a bundle that already has deployed resources tracked in [TEST_TMP_DIR]/.databricks/bundle/default/resources.json: only new deployments can be recorded. Remove experimental.record_deployment_history, or destroy the bundle and deploy it again +Error: cannot record deployment history for a bundle that already has deployed resources tracked in [TEST_TMP_DIR]/.databricks/bundle/default/resources.json: only new deployments can be recorded + +To record this bundle's history, start it over as a new deployment: + 1. remove experimental.record_deployment_history from your bundle configuration + 2. run "databricks bundle destroy" to delete the existing resources + 3. add experimental.record_deployment_history back and deploy again + +To keep the existing resources instead, leave experimental.record_deployment_history out >>> print_requests.py //api/2.0/bundle --oneline diff --git a/bundle/direct/dstate/state.go b/bundle/direct/dstate/state.go index bad87a6a293..c88e6e0dedb 100644 --- a/bundle/direct/dstate/state.go +++ b/bundle/direct/dstate/state.go @@ -293,7 +293,16 @@ func (db *DeploymentState) Open(ctx context.Context, path string, withRecovery W // older CLI refuses the state instead of deploying against resources it // cannot see. if dmsSource.DeploymentID == "" && len(db.Data.State) > 0 { - return fmt.Errorf("cannot record deployment history for a bundle that already has deployed resources tracked in %s: only new deployments can be recorded. Remove experimental.record_deployment_history, or destroy the bundle and deploy it again", path) + // The remedy is ordered deliberately: this error also blocks destroy, so the + // setting has to come out first or there is no way to tear the bundle down. + return fmt.Errorf(`cannot record deployment history for a bundle that already has deployed resources tracked in %s: only new deployments can be recorded + +To record this bundle's history, start it over as a new deployment: + 1. remove experimental.record_deployment_history from your bundle configuration + 2. run "databricks bundle destroy" to delete the existing resources + 3. add experimental.record_deployment_history back and deploy again + +To keep the existing resources instead, leave experimental.record_deployment_history out`, path) } if dmsSource.DeploymentID != "" { if err := db.readDMSState(ctx, dmsSource); err != nil { From 610b71f2df754a292879829d38db83ef4ee46fc3 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Thu, 6 Aug 2026 10:20:57 +0000 Subject: [PATCH 038/125] bundle: add a bugbash guide for deployment history recording The repo already has a bugbash mechanism (internal/bugbash/exec.sh drops you into a shell with a branch's CLI on PATH), but nothing explaining how to exercise this feature once you are there. Recording needs three separate things set before it does anything, and several of its behaviours look like bugs until you know they are not, so the guide covers both. Also notes in the README that the branch needs a successful release-build run for exec.sh to have something to download, and that the workflow only triggers on main, demo-* and bugbash-* - which is not obvious from the script and is the first thing to go wrong when pointing it at a feature branch. Co-authored-by: Isaac --- internal/bugbash/README.md | 8 + internal/bugbash/record-deployment-history.md | 151 ++++++++++++++++++ 2 files changed, 159 insertions(+) create mode 100644 internal/bugbash/record-deployment-history.md diff --git a/internal/bugbash/README.md b/internal/bugbash/README.md index 941ab6227cc..1c5995a8188 100644 --- a/internal/bugbash/README.md +++ b/internal/bugbash/README.md @@ -11,3 +11,11 @@ but works without command completion with earlier versions. ```shell bash <(curl -fsSL https://raw.githubusercontent.com/databricks/cli/main/internal/bugbash/exec.sh) my-branch ``` + +The branch needs a successful `release-build` run to download a snapshot from. That +workflow runs on `main` and on any branch named `demo-*` or `bugbash-*`, so push the +branch under one of those names. + +## Feature guides + +- [Deployment history recording](./record-deployment-history.md) diff --git a/internal/bugbash/record-deployment-history.md b/internal/bugbash/record-deployment-history.md new file mode 100644 index 00000000000..db0a7172653 --- /dev/null +++ b/internal/bugbash/record-deployment-history.md @@ -0,0 +1,151 @@ +# Bugbash: deployment history recording + +Records every `bundle deploy` and `bundle destroy` with the Deployment Metadata +Service (DMS), so a deployment has a server-side history and its resource state +lives in the workspace rather than only in the local cache. + +## Get a CLI with the feature + +```shell +bash <(curl -fsSL https://raw.githubusercontent.com/databricks/cli/main/internal/bugbash/exec.sh) bugbash-record-deployment-history +``` + +That drops you into a shell with `databricks` on `$PATH`. Check you have the right +build with `databricks --version`. + +## Turn the feature on + +Three things are needed. Missing any one of them means nothing is recorded. + +```shell +export DATABRICKS_BUNDLE_ENGINE=direct +export DATABRICKS_BUNDLE_FORCE_ALLOW_RECORD_DEPLOYMENT_HISTORY=true +``` + +and in `databricks.yml`: + +```yaml +experimental: + record_deployment_history: true +``` + +The env var only unlocks the gate; the YAML flag is what enables recording. Without +the env var the CLI refuses: + +``` +Error: experimental.record_deployment_history is not supported yet +``` + +Recording is direct-engine only. On terraform the flag is rejected, and no +`/api/2.0/bundle/*` calls are made. + +The feature must be enabled from the bundle's **first** deploy. Turning it on for a +bundle that already has deployed resources is refused, because DMS would then own a +resource set it never saw and the next deploy would create everything a second time. +The error spells out the three steps to start over. + +## A bundle to start from + +```yaml +bundle: + name: my-dms-test + +experimental: + record_deployment_history: true + +resources: + jobs: + hello: + name: my-dms-test-job + tasks: + - task_key: main + notebook_task: + notebook_path: ./noop.py +``` + +with `noop.py` beside it: + +``` +# Databricks notebook source +print(1) +``` + +## Find the deployment + +The CLI stores the deployment ID nowhere. DMS registers the deployment as a workspace +node, and that node's object ID *is* the deployment ID: + +```shell +databricks workspace get-status \ + "/Workspace/Users/$(databricks current-user me | jq -r .userName)/.bundle/my-dms-test/default/state/resources.deployment.json" \ + -o json | python3 -c 'import sys,json; print(json.load(sys.stdin)["object_id"])' +``` + +Use `python3`, not `jq`, for that ID. It exceeds 2^53 and jq below 1.7 silently +rounds it, which looks like "deployment does not exist". + +Or read it straight off the summary: + +```shell +databricks bundle summary -o json | jq .bundle.deployment.history +``` + +## What to look at + +```shell +databricks api get "/api/2.0/bundle/deployments/$DID" # the deployment +databricks api get "/api/2.0/bundle/deployments/$DID/versions" # one version per deploy +databricks api get "/api/2.0/bundle/deployments/$DID/versions/$V/operations" +databricks api get "/api/2.0/bundle/deployments/$DID/resources" # current resource state +databricks api get "/api/2.0/bundle/deployments" # all deployments +``` + +`resources` and `operations` paginate at 20 with a `next_page_token`. A bundle with +more than 20 resources is not truncated; page through it. + +Jobs and pipelines carry a back-reference to the deployment, but the SDK hides those +fields, so read them raw: + +```shell +databricks api get "/api/2.0/jobs/get?job_id=$JID" | jq .settings.deployment +databricks api get "/api/2.0/pipelines/$PID" | jq .spec.deployment +``` + +Both should show `deployment_id` and `version_id` next to `kind: BUNDLE`. + +## Worth exercising + +- **Iterate.** Deploy, change a field, deploy again. Each deploy claims a version; + only changed resources get an operation. +- **Wipe the local cache.** `rm -rf .databricks`, then `bundle plan`. It should report + your resources as unchanged, reconstructed from DMS. It must never plan to create + something that already exists. +- **Break a resource.** Give a job an invalid cron expression. The failed resource is + recorded with `status: OPERATION_STATUS_FAILED` and an `error_message`, the version + completes with `VERSION_COMPLETE_FAILURE`, and a later plan wants to create it. +- **Destroy.** A destroy records its own version with a DELETE per resource, then + deletes the deployment record. +- **Non-job resources.** Pipelines, schemas, volumes, experiments, registered models, + secret scopes and dashboards are all recorded. Each has a differently-shaped + resource id (numeric, UUID, `catalog.schema.name`, a scope name). +- **Targets.** Each target has its own state path, so `-t dev` and `-t prod` are + separate deployments with separate version chains. +- **Provenance.** Deploy from a git repo and check `git_info` on the version; + `deployment_mode` reflects the target's `mode`. + +## Not bugs + +- A redeploy with no changes still creates a version, with no operations under it. +- After `destroy`, `GetDeployment` still returns the record with + `status: DEPLOYMENT_STATUS_DELETED`. That is a soft delete. +- `state` on an operation or resource is a **quoted JSON string**, not an embedded + object. Parse it once to get `{"state": {...}, "depends_on": [...]}`. +- DMS resource keys have no `resources.` prefix (`jobs.foo`), unlike local state keys. +- Sub-resources get their own operation, e.g. `secret_scopes.mine.permissions`. +- Permissions are not set on the deployment node. It inherits from the state folder, + which the bundle's `permissions:` section already governs. + +## Reporting + +Include the deployment ID, the version, and the request/response for anything that +looks wrong. `databricks bundle deploy --log-level debug` logs the DMS calls. From d3bda0c4c3eaea4be48656cd7d10b6dc1c1601a1 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Thu, 6 Aug 2026 11:03:27 +0000 Subject: [PATCH 039/125] bundle: report the recorded deployment after a deploy A deploy said nothing about the deployment it had just recorded, so finding it meant knowing that the ID is the object ID of a workspace node and running get-status by hand. Deploy now ends with: Recorded deployment 996980114755597 version 2 at /Workspace/Users/.../state/resources.deployment.json The path is printed rather than a workspace URL. A deployment is a BUNDLE_DEPLOYMENT tree node with no page of its own: there is no entry for it in workspaceurls, and the file browser is fed by GraphQL tree messages that have no component handling that node type yet, so any URL would 404. The path is what exists today and is enough to look the node up. The line is skipped when recording is off, so a deploy that does not record prints nothing new - checked both ways on dogfood. Co-authored-by: Isaac --- acceptance/bundle/dms/depends-on/output.txt | 1 + .../bundle/dms/existing-state/output.txt | 1 + .../bundle/dms/multiple-resources/output.txt | 2 ++ acceptance/bundle/dms/no-resources/output.txt | 2 ++ acceptance/bundle/dms/provenance/output.txt | 1 + acceptance/bundle/dms/record/output.txt | 2 ++ .../dms/redeploy-after-destroy/output.txt | 2 ++ acceptance/bundle/dms/summary/output.txt | 2 ++ bundle/phases/deploy.go | 5 +++-- bundle/phases/dms.go | 21 +++++++++++++++++++ 10 files changed, 37 insertions(+), 2 deletions(-) diff --git a/acceptance/bundle/dms/depends-on/output.txt b/acceptance/bundle/dms/depends-on/output.txt index 952dcb00a86..695b24cd6c6 100644 --- a/acceptance/bundle/dms/depends-on/output.txt +++ b/acceptance/bundle/dms/depends-on/output.txt @@ -5,6 +5,7 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-depends-on/def Deploying resources... Updating deployment state... Deployment complete! +Recorded deployment [NUMID] version 1 at /Workspace/Users/[USERNAME]/.bundle/dms-depends-on/default/state/resources.deployment.json >>> print_requests.py //versions/1/operations --sort { diff --git a/acceptance/bundle/dms/existing-state/output.txt b/acceptance/bundle/dms/existing-state/output.txt index 6233f158974..2e34713c0c2 100644 --- a/acceptance/bundle/dms/existing-state/output.txt +++ b/acceptance/bundle/dms/existing-state/output.txt @@ -58,6 +58,7 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-existing-state Deploying resources... Updating deployment state... Deployment complete! +Recorded deployment [NUMID] version 1 at /Workspace/Users/[USERNAME]/.bundle/dms-existing-state/default/state/resources.deployment.json >>> print_requests.py //api/2.0/bundle --oneline {"method": "POST", "path": "/api/2.0/bundle/deployments", "body": {"initial_parent_path": "/Workspace/Users/[USERNAME]/.bundle/dms-existing-state/default/state", "target_name": "default"}} diff --git a/acceptance/bundle/dms/multiple-resources/output.txt b/acceptance/bundle/dms/multiple-resources/output.txt index b593998dbd1..8bc9358ad44 100644 --- a/acceptance/bundle/dms/multiple-resources/output.txt +++ b/acceptance/bundle/dms/multiple-resources/output.txt @@ -5,6 +5,7 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-multiple-resou Deploying resources... Updating deployment state... Deployment complete! +Recorded deployment [NUMID] version 1 at /Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources/default/state/resources.deployment.json >>> print_requests.py //versions/1/operations --sort --del-body state --oneline {"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations", "q": {"resource_key": "jobs.five"}, "body": {"action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_id": "[NUMID]", "resource_key": "jobs.five", "status": "OPERATION_STATUS_SUCCEEDED"}} @@ -19,6 +20,7 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-multiple-resou Deploying resources... Updating deployment state... Deployment complete! +Recorded deployment [NUMID] version 2 at /Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources/default/state/resources.deployment.json >>> print_requests.py //api/2.0/bundle --sort --oneline {"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": {"version_id": "2"}, "body": {"cli_version": "[CLI_VERSION]", "version_type": "VERSION_TYPE_DEPLOY", "target_name": "default", "display_name": "dms-multiple-resources", "previous_version_id": "1", "workspace_info": {"file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources/default/files", "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources/default"}}} diff --git a/acceptance/bundle/dms/no-resources/output.txt b/acceptance/bundle/dms/no-resources/output.txt index 8801d4179b1..0766ad59906 100644 --- a/acceptance/bundle/dms/no-resources/output.txt +++ b/acceptance/bundle/dms/no-resources/output.txt @@ -4,6 +4,7 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-no-resources/default/files... Deploying resources... Deployment complete! +Recorded deployment [NUMID] version 1 at /Workspace/Users/[USERNAME]/.bundle/dms-no-resources/default/state/resources.deployment.json >>> print_requests.py //api/2.0/bundle --get { @@ -50,6 +51,7 @@ Deployment complete! Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-no-resources/default/files... Deploying resources... Deployment complete! +Recorded deployment [NUMID] version 2 at /Workspace/Users/[USERNAME]/.bundle/dms-no-resources/default/state/resources.deployment.json >>> print_requests.py //api/2.0/bundle --get { diff --git a/acceptance/bundle/dms/provenance/output.txt b/acceptance/bundle/dms/provenance/output.txt index c567a12d66c..86f8caf3a10 100644 --- a/acceptance/bundle/dms/provenance/output.txt +++ b/acceptance/bundle/dms/provenance/output.txt @@ -5,6 +5,7 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-provenance/dev Deploying resources... Updating deployment state... Deployment complete! +Recorded deployment [NUMID] version 1 at /Workspace/Users/[USERNAME]/.bundle/dms-provenance/dev/state/resources.deployment.json >>> print_requests.py //versions --sort { diff --git a/acceptance/bundle/dms/record/output.txt b/acceptance/bundle/dms/record/output.txt index a707e5e46d7..8602c252eff 100644 --- a/acceptance/bundle/dms/record/output.txt +++ b/acceptance/bundle/dms/record/output.txt @@ -5,6 +5,7 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-record/default Deploying resources... Updating deployment state... Deployment complete! +Recorded deployment [NUMID] version 1 at /Workspace/Users/[USERNAME]/.bundle/dms-record/default/state/resources.deployment.json >>> print_requests.py //api/2.0/bundle { @@ -70,6 +71,7 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-record/default Deploying resources... Updating deployment state... Deployment complete! +Recorded deployment [NUMID] version 2 at /Workspace/Users/[USERNAME]/.bundle/dms-record/default/state/resources.deployment.json >>> print_requests.py //api/2.0/bundle { diff --git a/acceptance/bundle/dms/redeploy-after-destroy/output.txt b/acceptance/bundle/dms/redeploy-after-destroy/output.txt index 12ce1a298c4..ad83e46b575 100644 --- a/acceptance/bundle/dms/redeploy-after-destroy/output.txt +++ b/acceptance/bundle/dms/redeploy-after-destroy/output.txt @@ -5,6 +5,7 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-redeploy-after Deploying resources... Updating deployment state... Deployment complete! +Recorded deployment [NUMID] version 1 at /Workspace/Users/[USERNAME]/.bundle/dms-redeploy-after-destroy/default/state/resources.deployment.json >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: @@ -24,6 +25,7 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-redeploy-after Deploying resources... Updating deployment state... Deployment complete! +Recorded deployment [NUMID] version 1 at /Workspace/Users/[USERNAME]/.bundle/dms-redeploy-after-destroy/default/state/resources.deployment.json >>> MSYS_NO_PATHCONV=1 [CLI] workspace get-status /Workspace/Users/[USERNAME]/.bundle/dms-redeploy-after-destroy/default/state/resources.deployment.json { diff --git a/acceptance/bundle/dms/summary/output.txt b/acceptance/bundle/dms/summary/output.txt index c792a60fcfa..e378456b9c3 100644 --- a/acceptance/bundle/dms/summary/output.txt +++ b/acceptance/bundle/dms/summary/output.txt @@ -5,6 +5,7 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-summary/defaul Deploying resources... Updating deployment state... Deployment complete! +Recorded deployment [NUMID] version 1 at /Workspace/Users/[USERNAME]/.bundle/dms-summary/default/state/resources.deployment.json >>> [CLI] bundle summary -o json { @@ -18,6 +19,7 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-summary/defaul Deploying resources... Updating deployment state... Deployment complete! +Recorded deployment [NUMID] version 2 at /Workspace/Users/[USERNAME]/.bundle/dms-summary/default/state/resources.deployment.json >>> [CLI] bundle summary -o json { diff --git a/bundle/phases/deploy.go b/bundle/phases/deploy.go index 808ca6c012c..ba923638ab5 100644 --- a/bundle/phases/deploy.go +++ b/bundle/phases/deploy.go @@ -77,7 +77,7 @@ func approvalForDeploy(ctx context.Context, b *bundle.Bundle, plan *deployplan.P return cmdio.AskYesOrNo(ctx, "Would you like to proceed?") } -func deployCore(ctx context.Context, b *bundle.Bundle, plan *deployplan.Plan, stateEngine engine.EngineType, requestedEngine engine.EngineSetting) { +func deployCore(ctx context.Context, b *bundle.Bundle, plan *deployplan.Plan, stateEngine engine.EngineType, requestedEngine engine.EngineSetting, recorder *dms.Recorder) { // Core mutators that CRUD resources and modify deployment state. These // mutators need informed consent if they are potentially destructive. cmdio.LogString(ctx, "Deploying resources...") @@ -120,6 +120,7 @@ func deployCore(ctx context.Context, b *bundle.Bundle, plan *deployplan.Plan, st if !logdiag.HasError(ctx) { cmdio.LogString(ctx, "Deployment complete!") + logDeploymentHistory(ctx, b, recorder) } // Once the deploy is complete, dry-run the migration to the direct engine @@ -296,7 +297,7 @@ func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHand recorder.Version(), ) } - deployCore(ctx, b, plan, stateEngine, requestedEngine) + deployCore(ctx, b, plan, stateEngine, requestedEngine, recorder) } else { cmdio.LogString(ctx, "Deployment cancelled!") return diff --git a/bundle/phases/dms.go b/bundle/phases/dms.go index 8e204208813..5ebf652c639 100644 --- a/bundle/phases/dms.go +++ b/bundle/phases/dms.go @@ -2,11 +2,14 @@ package phases import ( "context" + "fmt" + "path" "strings" "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/engine" + "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/dms" "github.com/databricks/databricks-sdk-go/client" "github.com/databricks/databricks-sdk-go/service/bundledeployments" @@ -53,6 +56,24 @@ func newDeploymentRecorder(ctx context.Context, b *bundle.Bundle, eng engine.Eng }), nil } +// logDeploymentHistory reports the deployment this deploy was recorded under, so +// the user can look its history up without hunting for the ID. A nil recorder means +// recording is off, and a zero version means the version was never created. +// +// It prints the deployment's workspace path rather than a UI link: the deployment is +// a BUNDLE_DEPLOYMENT tree node with no page of its own yet, so a URL would 404. +func logDeploymentHistory(ctx context.Context, b *bundle.Bundle, recorder *dms.Recorder) { + if recorder == nil || recorder.Version() == 0 { + return + } + + cmdio.LogString(ctx, fmt.Sprintf("Recorded deployment %s version %d at %s", + recorder.DeploymentID(), + recorder.Version(), + path.Join(b.Config.Workspace.StatePath, dms.DeploymentNodeName), + )) +} + // deploymentProvenance describes the source this deploy came from and where it // landed, mirroring what bundle/deploy/metadata computes for the metadata file. func deploymentProvenance(b *bundle.Bundle) dms.Provenance { From 45c4ed221fcbd76f7735195a245fd63ead382754 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Thu, 6 Aug 2026 11:12:06 +0000 Subject: [PATCH 040/125] bundle: link to the recorded deployment after a deploy Deploy printed the deployment node's workspace path, which was only useful for an API lookup. There is a page for a deployment, so link to it instead: Deployment history: https:///deployments/996980114757453?version=2 The version pins the page to the deploy that just ran, and advances with each deploy. The workspace ID is omitted - the page redirects correctly without it, and leaving it out keeps the line short enough to stay clickable in a terminal. DeploymentURL lives in libs/workspaceurls next to the resource URL patterns, but is a separate function rather than another entry in resourceURLPatterns: a deployment is not a bundle resource type, and it takes a query parameter that none of those patterns do. It preserves an existing query so a base URL carrying ?w= for a vanity or legacy host still addresses the right workspace. An unparseable host degrades to reporting the id and version without a link, rather than failing a deploy that already succeeded. Co-authored-by: Isaac --- acceptance/bundle/dms/depends-on/output.txt | 2 +- .../bundle/dms/existing-state/output.txt | 2 +- .../bundle/dms/multiple-resources/output.txt | 4 +- acceptance/bundle/dms/no-resources/output.txt | 4 +- acceptance/bundle/dms/provenance/output.txt | 2 +- acceptance/bundle/dms/record/output.txt | 4 +- .../dms/redeploy-after-destroy/output.txt | 4 +- acceptance/bundle/dms/summary/output.txt | 4 +- bundle/phases/dms.go | 27 ++++++---- libs/workspaceurls/urls.go | 23 +++++++++ libs/workspaceurls/urls_test.go | 49 +++++++++++++++++++ 11 files changed, 102 insertions(+), 23 deletions(-) diff --git a/acceptance/bundle/dms/depends-on/output.txt b/acceptance/bundle/dms/depends-on/output.txt index 695b24cd6c6..5b9ca490227 100644 --- a/acceptance/bundle/dms/depends-on/output.txt +++ b/acceptance/bundle/dms/depends-on/output.txt @@ -5,7 +5,7 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-depends-on/def Deploying resources... Updating deployment state... Deployment complete! -Recorded deployment [NUMID] version 1 at /Workspace/Users/[USERNAME]/.bundle/dms-depends-on/default/state/resources.deployment.json +Deployment history: [DATABRICKS_URL]/deployments/[NUMID]?version=1 >>> print_requests.py //versions/1/operations --sort { diff --git a/acceptance/bundle/dms/existing-state/output.txt b/acceptance/bundle/dms/existing-state/output.txt index 2e34713c0c2..3b5da470cfa 100644 --- a/acceptance/bundle/dms/existing-state/output.txt +++ b/acceptance/bundle/dms/existing-state/output.txt @@ -58,7 +58,7 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-existing-state Deploying resources... Updating deployment state... Deployment complete! -Recorded deployment [NUMID] version 1 at /Workspace/Users/[USERNAME]/.bundle/dms-existing-state/default/state/resources.deployment.json +Deployment history: [DATABRICKS_URL]/deployments/[NUMID]?version=1 >>> print_requests.py //api/2.0/bundle --oneline {"method": "POST", "path": "/api/2.0/bundle/deployments", "body": {"initial_parent_path": "/Workspace/Users/[USERNAME]/.bundle/dms-existing-state/default/state", "target_name": "default"}} diff --git a/acceptance/bundle/dms/multiple-resources/output.txt b/acceptance/bundle/dms/multiple-resources/output.txt index 8bc9358ad44..7e81865e11a 100644 --- a/acceptance/bundle/dms/multiple-resources/output.txt +++ b/acceptance/bundle/dms/multiple-resources/output.txt @@ -5,7 +5,7 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-multiple-resou Deploying resources... Updating deployment state... Deployment complete! -Recorded deployment [NUMID] version 1 at /Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources/default/state/resources.deployment.json +Deployment history: [DATABRICKS_URL]/deployments/[NUMID]?version=1 >>> print_requests.py //versions/1/operations --sort --del-body state --oneline {"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations", "q": {"resource_key": "jobs.five"}, "body": {"action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_id": "[NUMID]", "resource_key": "jobs.five", "status": "OPERATION_STATUS_SUCCEEDED"}} @@ -20,7 +20,7 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-multiple-resou Deploying resources... Updating deployment state... Deployment complete! -Recorded deployment [NUMID] version 2 at /Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources/default/state/resources.deployment.json +Deployment history: [DATABRICKS_URL]/deployments/[NUMID]?version=2 >>> print_requests.py //api/2.0/bundle --sort --oneline {"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": {"version_id": "2"}, "body": {"cli_version": "[CLI_VERSION]", "version_type": "VERSION_TYPE_DEPLOY", "target_name": "default", "display_name": "dms-multiple-resources", "previous_version_id": "1", "workspace_info": {"file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources/default/files", "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources/default"}}} diff --git a/acceptance/bundle/dms/no-resources/output.txt b/acceptance/bundle/dms/no-resources/output.txt index 0766ad59906..e4c0341934b 100644 --- a/acceptance/bundle/dms/no-resources/output.txt +++ b/acceptance/bundle/dms/no-resources/output.txt @@ -4,7 +4,7 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-no-resources/default/files... Deploying resources... Deployment complete! -Recorded deployment [NUMID] version 1 at /Workspace/Users/[USERNAME]/.bundle/dms-no-resources/default/state/resources.deployment.json +Deployment history: [DATABRICKS_URL]/deployments/[NUMID]?version=1 >>> print_requests.py //api/2.0/bundle --get { @@ -51,7 +51,7 @@ Recorded deployment [NUMID] version 1 at /Workspace/Users/[USERNAME]/.bundle/dms Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-no-resources/default/files... Deploying resources... Deployment complete! -Recorded deployment [NUMID] version 2 at /Workspace/Users/[USERNAME]/.bundle/dms-no-resources/default/state/resources.deployment.json +Deployment history: [DATABRICKS_URL]/deployments/[NUMID]?version=2 >>> print_requests.py //api/2.0/bundle --get { diff --git a/acceptance/bundle/dms/provenance/output.txt b/acceptance/bundle/dms/provenance/output.txt index 86f8caf3a10..a4084f82728 100644 --- a/acceptance/bundle/dms/provenance/output.txt +++ b/acceptance/bundle/dms/provenance/output.txt @@ -5,7 +5,7 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-provenance/dev Deploying resources... Updating deployment state... Deployment complete! -Recorded deployment [NUMID] version 1 at /Workspace/Users/[USERNAME]/.bundle/dms-provenance/dev/state/resources.deployment.json +Deployment history: [DATABRICKS_URL]/deployments/[NUMID]?version=1 >>> print_requests.py //versions --sort { diff --git a/acceptance/bundle/dms/record/output.txt b/acceptance/bundle/dms/record/output.txt index 8602c252eff..9a51c82e60e 100644 --- a/acceptance/bundle/dms/record/output.txt +++ b/acceptance/bundle/dms/record/output.txt @@ -5,7 +5,7 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-record/default Deploying resources... Updating deployment state... Deployment complete! -Recorded deployment [NUMID] version 1 at /Workspace/Users/[USERNAME]/.bundle/dms-record/default/state/resources.deployment.json +Deployment history: [DATABRICKS_URL]/deployments/[NUMID]?version=1 >>> print_requests.py //api/2.0/bundle { @@ -71,7 +71,7 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-record/default Deploying resources... Updating deployment state... Deployment complete! -Recorded deployment [NUMID] version 2 at /Workspace/Users/[USERNAME]/.bundle/dms-record/default/state/resources.deployment.json +Deployment history: [DATABRICKS_URL]/deployments/[NUMID]?version=2 >>> print_requests.py //api/2.0/bundle { diff --git a/acceptance/bundle/dms/redeploy-after-destroy/output.txt b/acceptance/bundle/dms/redeploy-after-destroy/output.txt index ad83e46b575..64c4e6b8c4f 100644 --- a/acceptance/bundle/dms/redeploy-after-destroy/output.txt +++ b/acceptance/bundle/dms/redeploy-after-destroy/output.txt @@ -5,7 +5,7 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-redeploy-after Deploying resources... Updating deployment state... Deployment complete! -Recorded deployment [NUMID] version 1 at /Workspace/Users/[USERNAME]/.bundle/dms-redeploy-after-destroy/default/state/resources.deployment.json +Deployment history: [DATABRICKS_URL]/deployments/[NUMID]?version=1 >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: @@ -25,7 +25,7 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-redeploy-after Deploying resources... Updating deployment state... Deployment complete! -Recorded deployment [NUMID] version 1 at /Workspace/Users/[USERNAME]/.bundle/dms-redeploy-after-destroy/default/state/resources.deployment.json +Deployment history: [DATABRICKS_URL]/deployments/[NUMID]?version=1 >>> MSYS_NO_PATHCONV=1 [CLI] workspace get-status /Workspace/Users/[USERNAME]/.bundle/dms-redeploy-after-destroy/default/state/resources.deployment.json { diff --git a/acceptance/bundle/dms/summary/output.txt b/acceptance/bundle/dms/summary/output.txt index e378456b9c3..13a86eb3a21 100644 --- a/acceptance/bundle/dms/summary/output.txt +++ b/acceptance/bundle/dms/summary/output.txt @@ -5,7 +5,7 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-summary/defaul Deploying resources... Updating deployment state... Deployment complete! -Recorded deployment [NUMID] version 1 at /Workspace/Users/[USERNAME]/.bundle/dms-summary/default/state/resources.deployment.json +Deployment history: [DATABRICKS_URL]/deployments/[NUMID]?version=1 >>> [CLI] bundle summary -o json { @@ -19,7 +19,7 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-summary/defaul Deploying resources... Updating deployment state... Deployment complete! -Recorded deployment [NUMID] version 2 at /Workspace/Users/[USERNAME]/.bundle/dms-summary/default/state/resources.deployment.json +Deployment history: [DATABRICKS_URL]/deployments/[NUMID]?version=2 >>> [CLI] bundle summary -o json { diff --git a/bundle/phases/dms.go b/bundle/phases/dms.go index 5ebf652c639..aa6b1d8e810 100644 --- a/bundle/phases/dms.go +++ b/bundle/phases/dms.go @@ -3,7 +3,7 @@ package phases import ( "context" "fmt" - "path" + "net/url" "strings" "github.com/databricks/cli/bundle" @@ -11,6 +11,8 @@ import ( "github.com/databricks/cli/bundle/config/engine" "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/dms" + "github.com/databricks/cli/libs/log" + "github.com/databricks/cli/libs/workspaceurls" "github.com/databricks/databricks-sdk-go/client" "github.com/databricks/databricks-sdk-go/service/bundledeployments" ) @@ -56,22 +58,27 @@ func newDeploymentRecorder(ctx context.Context, b *bundle.Bundle, eng engine.Eng }), nil } -// logDeploymentHistory reports the deployment this deploy was recorded under, so -// the user can look its history up without hunting for the ID. A nil recorder means +// logDeploymentHistory links to the deployment this deploy was recorded under, so +// the user can open its history without hunting for the ID. A nil recorder means // recording is off, and a zero version means the version was never created. // -// It prints the deployment's workspace path rather than a UI link: the deployment is -// a BUNDLE_DEPLOYMENT tree node with no page of its own yet, so a URL would 404. +// The workspace ID is left out of the URL: the page redirects correctly without it, +// and omitting it keeps the line short enough to stay clickable in a terminal. func logDeploymentHistory(ctx context.Context, b *bundle.Bundle, recorder *dms.Recorder) { if recorder == nil || recorder.Version() == 0 { return } - cmdio.LogString(ctx, fmt.Sprintf("Recorded deployment %s version %d at %s", - recorder.DeploymentID(), - recorder.Version(), - path.Join(b.Config.Workspace.StatePath, dms.DeploymentNodeName), - )) + baseURL, err := url.Parse(b.WorkspaceClient(ctx).Config.CanonicalHostName()) + if err != nil { + // Only the link is lost, so report the deployment without it rather than + // failing a deploy that already succeeded. + log.Debugf(ctx, "Not linking to the recorded deployment: %s", err) + cmdio.LogString(ctx, fmt.Sprintf("Recorded deployment %s version %d", recorder.DeploymentID(), recorder.Version())) + return + } + + cmdio.LogString(ctx, "Deployment history: "+workspaceurls.DeploymentURL(*baseURL, recorder.DeploymentID(), recorder.Version())) } // deploymentProvenance describes the source this deploy came from and where it diff --git a/libs/workspaceurls/urls.go b/libs/workspaceurls/urls.go index a1bf973801f..7efc2156ef3 100644 --- a/libs/workspaceurls/urls.go +++ b/libs/workspaceurls/urls.go @@ -4,6 +4,7 @@ import ( "fmt" "net/url" "slices" + "strconv" "strings" ) @@ -68,6 +69,28 @@ func ResourceTypes() []string { return names } +// DeploymentURL returns the workspace URL for a bundle deployment recorded with +// the deployment metadata service, of the form +// +// /deployments/?version= +// +// The version pins the page to the deploy that produced it. It is separate from +// ResourceURL because a deployment is not a bundle resource type: it has no entry +// in resourceURLPatterns and takes a query parameter none of those do. +func DeploymentURL(baseURL url.URL, deploymentID string, version int64) string { + if deploymentID == "" { + return "" + } + + baseURL.Path = "deployments/" + deploymentID + if version > 0 { + values := baseURL.Query() + values.Set("version", strconv.FormatInt(version, 10)) + baseURL.RawQuery = values.Encode() + } + return baseURL.String() +} + // JobRunPath returns the modern workspace path for a job run, of the form // // jobs//runs/ diff --git a/libs/workspaceurls/urls_test.go b/libs/workspaceurls/urls_test.go index e39d28d9aaf..ba8299efb05 100644 --- a/libs/workspaceurls/urls_test.go +++ b/libs/workspaceurls/urls_test.go @@ -141,6 +141,55 @@ func TestResourceURL(t *testing.T) { } } +func TestDeploymentURL(t *testing.T) { + tests := []struct { + name string + deploymentID string + version int64 + base url.URL + expected string + }{ + { + name: "id and version", + deploymentID: "996980114684409", + version: 2, + base: url.URL{Scheme: "https", Host: "host.com"}, + expected: "https://host.com/deployments/996980114684409?version=2", + }, + { + // The version is only known once CreateVersion has run, so link to the + // deployment itself rather than emitting version=0. + name: "zero version omits the query", + deploymentID: "996980114684409", + version: 0, + base: url.URL{Scheme: "https", Host: "host.com"}, + expected: "https://host.com/deployments/996980114684409", + }, + { + // A base URL carrying ?w= keeps it, so a vanity or legacy + // host still addresses the right workspace. + name: "preserves an existing query", + deploymentID: "42", + version: 7, + base: url.URL{Scheme: "https", Host: "host.com", RawQuery: "w=123"}, + expected: "https://host.com/deployments/42?version=7&w=123", + }, + { + name: "empty id returns empty", + deploymentID: "", + version: 1, + base: url.URL{Scheme: "https", Host: "host.com"}, + expected: "", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.expected, DeploymentURL(tt.base, tt.deploymentID, tt.version)) + }) + } +} + func TestHasWorkspaceIDInHostname(t *testing.T) { tests := []struct { name string From a8a5148f9fdf28ae34ee164ef6c7117092b04b90 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Thu, 6 Aug 2026 12:55:58 +0000 Subject: [PATCH 041/125] bundle: send the prior state when recording a failed operation Recording a failed operation sent no state, which the service rejects for any action that acts on an existing resource: Error: recording operation for resources.jobs.test_job with the deployment metadata service: state is required for a OPERATION_ACTION_TYPE_UPDATE operation, because it acts on a resource that already exists and cannot destroy it, even when it fails (400 INVALID_PARAMETER_VALUE) So a failed update replaced the real error with this one, and the failure itself was never recorded. The rule makes sense: dropping the state would leave DMS unable to describe a resource it still owns. A failed operation now carries the resource's state from before the deploy, unchanged - the resource is whatever it was before the attempt. It stays nil for a create, where there is no prior state and no resource to describe, which is also why resource_id may be empty for CREATE and RECREATE. Verified on dogfood: a job deployed, then given an invalid cron so its update fails. The 400 is gone, the deploy reports only the real quartz error, and version 2 records the operation as FAILED with the error message and the pre-deploy state (no schedule). Co-authored-by: Isaac --- bundle/direct/bundle_apply.go | 4 ++-- bundle/direct/opqueue.go | 5 +++-- bundle/direct/oprecorder.go | 29 +++++++++++++++++++++++++---- bundle/direct/oprecorder_test.go | 4 ++-- 4 files changed, 32 insertions(+), 10 deletions(-) diff --git a/bundle/direct/bundle_apply.go b/bundle/direct/bundle_apply.go index bdd0c243e7f..c3f7f02cbad 100644 --- a/bundle/direct/bundle_apply.go +++ b/bundle/direct/bundle_apply.go @@ -104,7 +104,7 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa err = d.Destroy(ctx, &b.StateDB) } if err != nil { - opQueue.recordFailure(ctx, resourceKey, action, deletedID, err) + opQueue.recordFailure(ctx, resourceKey, action, deletedID, priorState(&b.StateDB, resourceKey), err) logdiag.LogError(ctx, fmt.Errorf("%s: %w", errorPrefix, err)) return false } @@ -140,7 +140,7 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa if err != nil { // GetResourceID is empty for a create that never got an ID, which is // what the service expects for a failed create. - opQueue.recordFailure(ctx, resourceKey, action, b.StateDB.GetResourceID(resourceKey), err) + opQueue.recordFailure(ctx, resourceKey, action, b.StateDB.GetResourceID(resourceKey), priorState(&b.StateDB, resourceKey), err) logdiag.LogError(ctx, fmt.Errorf("%s: %w", errorPrefix, err)) return false } diff --git a/bundle/direct/opqueue.go b/bundle/direct/opqueue.go index 551c91b2874..ba64d52ebd9 100644 --- a/bundle/direct/opqueue.go +++ b/bundle/direct/opqueue.go @@ -2,6 +2,7 @@ package direct import ( "context" + "encoding/json" "fmt" "sync" @@ -141,12 +142,12 @@ func (q *operationQueue) record(ctx context.Context, resourceKey string, action // Unlike record, this does not resurface an earlier upload error: the deploy is // already failing, and returning a different error here would replace the one the // user needs to see. A failure to upload this record is reported at close. -func (q *operationQueue) recordFailure(ctx context.Context, resourceKey string, action deployplan.ActionType, resourceID string, cause error) { +func (q *operationQueue) recordFailure(ctx context.Context, resourceKey string, action deployplan.ActionType, resourceID string, priorState json.RawMessage, cause error) { if q == nil { return } - op, err := newFailedOperation(action, resourceID, cause) + op, err := newFailedOperation(action, resourceID, priorState, cause) if err != nil { log.Warnf(ctx, "Not recording failure for %s: %s", resourceKey, err) return diff --git a/bundle/direct/oprecorder.go b/bundle/direct/oprecorder.go index a4595a89c08..2055384ffa8 100644 --- a/bundle/direct/oprecorder.go +++ b/bundle/direct/oprecorder.go @@ -81,10 +81,13 @@ func newRecordedOperation(action deployplan.ActionType, resourceID string, state // newFailedOperation records an operation that did not apply, so the deployment // history says why a resource failed rather than just omitting it. // -// No state is recorded: the resource was not written, so there is nothing to -// serve back as its state. CREATE and RECREATE may have no resourceID yet, which -// the service allows for exactly those two actions. -func newFailedOperation(action deployplan.ActionType, resourceID string, cause error) (recordedOperation, error) { +// priorState is the resource's state from before the deploy, carried through +// unchanged: an action other than a create acts on a resource that still exists, and +// the service rejects such an operation without state because dropping it would +// leave DMS unable to describe a resource it still owns. It is nil for a create, +// where there is no prior state and no resource to describe - which is also why the +// resourceID may be empty for CREATE and RECREATE. +func newFailedOperation(action deployplan.ActionType, resourceID string, priorState json.RawMessage, cause error) (recordedOperation, error) { actionType, err := deployActionToSDK(action) if err != nil { return recordedOperation{}, err @@ -100,9 +103,27 @@ func newFailedOperation(action deployplan.ActionType, resourceID string, cause e resourceID: resourceID, status: bundledeployments.OperationStatusOperationStatusFailed, errorMessage: message, + state: priorState, }, nil } +// priorState returns the resource's recorded state from before this deploy, in the +// same envelope form the success path uploads, or nil when the resource has none +// (a create). A failed operation reports this unchanged: the resource is whatever it +// was before the attempt. +func priorState(db *dstate.DeploymentState, resourceKey string) json.RawMessage { + entry, ok := db.GetResourceEntry(resourceKey) + if !ok || len(entry.State) == 0 { + return nil + } + + raw, err := json.Marshal(dstate.RecordedState{State: entry.State, DependsOn: entry.DependsOn}) + if err != nil { + return nil + } + return raw +} + // operationUploader records an applied resource operation with DMS. Uploads run // on the operationQueue workers, off the apply path. type operationUploader interface { diff --git a/bundle/direct/oprecorder_test.go b/bundle/direct/oprecorder_test.go index a76a91edcdb..ec5068563e2 100644 --- a/bundle/direct/oprecorder_test.go +++ b/bundle/direct/oprecorder_test.go @@ -88,7 +88,7 @@ func TestNewRecordedOperationRejectsUnsupportedAction(t *testing.T) { } func TestNewFailedOperationRecordsError(t *testing.T) { - op, err := newFailedOperation(deployplan.Create, "", errors.New("cluster spec is invalid")) + op, err := newFailedOperation(deployplan.Create, "", nil, errors.New("cluster spec is invalid")) require.NoError(t, err) assert.Equal(t, bundledeployments.OperationStatusOperationStatusFailed, op.status) @@ -100,7 +100,7 @@ func TestNewFailedOperationRecordsError(t *testing.T) { func TestNewFailedOperationTruncatesLongError(t *testing.T) { // Truncated rather than rejected: a message over the limit would make recording // fail and hide the error it is reporting. - op, err := newFailedOperation(deployplan.Update, "job-123", errors.New(strings.Repeat("x", maxOperationErrorMessageSize+100))) + op, err := newFailedOperation(deployplan.Update, "job-123", nil, errors.New(strings.Repeat("x", maxOperationErrorMessageSize+100))) require.NoError(t, err) assert.Len(t, op.errorMessage, maxOperationErrorMessageSize) From cf2d9cd957771fb914507214c499606d4533fb98 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Thu, 6 Aug 2026 13:18:12 +0000 Subject: [PATCH 042/125] bundle: take the id and state of a failed operation from the same record Recording a failed recreate sent state without an id, which the service rejects: Error: recording operation for resources.schemas.foo with the deployment metadata service: resource_id is required for an operation that records state, because state records a resource that exists (400 INVALID_PARAMETER_VALUE) So a failed recreate replaced the real error with this one. The id came from live state while the state came from the pre-deploy record, and a recreate deletes before it creates, so by the time the create failed the id was gone and the state was not. Both now come from the same pre-deploy entry, which is the only pairing the service accepts: state describes a resource that exists, so it needs the id to say which one. They stay empty together for a create, which never had either. Verified on dogfood: a schema deployed, then pointed at a nonexistent catalog so its recreate fails. The 400 is gone, the deploy reports only the real catalog error, and the operation records as FAILED with the prior catalog_name and the id. Co-authored-by: Isaac --- bundle/direct/bundle_apply.go | 10 ++++++---- bundle/direct/oprecorder.go | 21 +++++++++++++-------- bundle/direct/oprecorder_test.go | 13 +++++++++++++ 3 files changed, 32 insertions(+), 12 deletions(-) diff --git a/bundle/direct/bundle_apply.go b/bundle/direct/bundle_apply.go index c3f7f02cbad..2047f65a498 100644 --- a/bundle/direct/bundle_apply.go +++ b/bundle/direct/bundle_apply.go @@ -104,7 +104,8 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa err = d.Destroy(ctx, &b.StateDB) } if err != nil { - opQueue.recordFailure(ctx, resourceKey, action, deletedID, priorState(&b.StateDB, resourceKey), err) + _, priorState := priorRecord(&b.StateDB, resourceKey) + opQueue.recordFailure(ctx, resourceKey, action, deletedID, priorState, err) logdiag.LogError(ctx, fmt.Errorf("%s: %w", errorPrefix, err)) return false } @@ -138,9 +139,10 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa // TODO: redo calcDiff to downgrade planned action if possible (?) err = d.Deploy(ctx, &b.StateDB, sv.Value, action, entry) if err != nil { - // GetResourceID is empty for a create that never got an ID, which is - // what the service expects for a failed create. - opQueue.recordFailure(ctx, resourceKey, action, b.StateDB.GetResourceID(resourceKey), priorState(&b.StateDB, resourceKey), err) + // Both are empty for a create that never got an ID, which is what the + // service expects for a failed create. + priorID, priorState := priorRecord(&b.StateDB, resourceKey) + opQueue.recordFailure(ctx, resourceKey, action, priorID, priorState, err) logdiag.LogError(ctx, fmt.Errorf("%s: %w", errorPrefix, err)) return false } diff --git a/bundle/direct/oprecorder.go b/bundle/direct/oprecorder.go index 2055384ffa8..32494c07325 100644 --- a/bundle/direct/oprecorder.go +++ b/bundle/direct/oprecorder.go @@ -107,21 +107,26 @@ func newFailedOperation(action deployplan.ActionType, resourceID string, priorSt }, nil } -// priorState returns the resource's recorded state from before this deploy, in the -// same envelope form the success path uploads, or nil when the resource has none -// (a create). A failed operation reports this unchanged: the resource is whatever it -// was before the attempt. -func priorState(db *dstate.DeploymentState, resourceKey string) json.RawMessage { +// priorRecord returns the resource's id and state from before this deploy, in the +// same envelope form the success path uploads, or empty values when the resource has +// no prior record (a create). A failed operation reports these unchanged: the resource +// is whatever it was before the attempt. +// +// Both come from the same pre-deploy entry because the service requires an id +// alongside state: state describes a resource that exists, so it needs the id to say +// which one. Reading the id from live state instead would return "" for a failed +// recreate, whose delete step already dropped it, and the mismatch is rejected. +func priorRecord(db *dstate.DeploymentState, resourceKey string) (string, json.RawMessage) { entry, ok := db.GetResourceEntry(resourceKey) if !ok || len(entry.State) == 0 { - return nil + return "", nil } raw, err := json.Marshal(dstate.RecordedState{State: entry.State, DependsOn: entry.DependsOn}) if err != nil { - return nil + return "", nil } - return raw + return entry.ID, raw } // operationUploader records an applied resource operation with DMS. Uploads run diff --git a/bundle/direct/oprecorder_test.go b/bundle/direct/oprecorder_test.go index ec5068563e2..b793ab65a08 100644 --- a/bundle/direct/oprecorder_test.go +++ b/bundle/direct/oprecorder_test.go @@ -2,6 +2,7 @@ package direct import ( "context" + "encoding/json" "errors" "strings" "sync" @@ -97,6 +98,18 @@ func TestNewFailedOperationRecordsError(t *testing.T) { assert.Nil(t, op.state) } +func TestNewFailedOperationRecordsPriorStateWithID(t *testing.T) { + // A failed recreate has already deleted the resource, so the id must come from + // the pre-deploy record alongside the state: the service rejects state without + // an id, since state describes a resource that exists. + op, err := newFailedOperation(deployplan.Recreate, "main.some_schema", json.RawMessage(`{"state":{"catalog_name":"main"}}`), errors.New("Catalog 'mainx' does not exist")) + require.NoError(t, err) + + assert.Equal(t, bundledeployments.OperationStatusOperationStatusFailed, op.status) + assert.Equal(t, "main.some_schema", op.resourceID) + assert.JSONEq(t, `{"state":{"catalog_name":"main"}}`, string(op.state)) +} + func TestNewFailedOperationTruncatesLongError(t *testing.T) { // Truncated rather than rejected: a message over the limit would make recording // fail and hide the error it is reporting. From 015af745e828edae2e370fa1fb6292aef2e705da Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Thu, 6 Aug 2026 13:30:48 +0000 Subject: [PATCH 043/125] bundle: stop the deployment stamp from showing as drift After a deploy, planning an untouched bundle reported an update on every job and pipeline: "deployment.deployment_id": { "action": "update", "old": "261237257843077", "remote": "261237257843077" } old and remote agree and new is absent, which is the tell: only the deploy phase stamps deployment_id, because it is not known until the version is claimed. A plain `bundle plan` never runs that mutator, so the field is set in the state and in the workspace but empty in the local config, and the absence reads as a change. version_id was already ignored as a local change for a different reason (it changes on every deploy). deployment_id was left out deliberately, on the grounds that it is stable so a change to it is worth showing - but the value it is compared against is never populated at plan time, so the rule only ever fired on this phantom. It is now ignored as a local change too, for jobs and pipelines. The stamp itself is unaffected: verified on dogfood that the deployed job still carries deployment_id, and the drift is gone (0 to change, 1 unchanged). The new acceptance test plans an untouched bundle after deploying it, which no existing DMS test did - that gap is what let this through. Co-authored-by: Isaac --- acceptance/bundle/dms/no-drift/databricks.yml | 16 +++++ acceptance/bundle/dms/no-drift/out.test.toml | 3 + acceptance/bundle/dms/no-drift/output.txt | 58 +++++++++++++++++++ acceptance/bundle/dms/no-drift/script | 12 ++++ bundle/direct/dresources/resources.yml | 16 +++-- 5 files changed, 101 insertions(+), 4 deletions(-) create mode 100644 acceptance/bundle/dms/no-drift/databricks.yml create mode 100644 acceptance/bundle/dms/no-drift/out.test.toml create mode 100644 acceptance/bundle/dms/no-drift/output.txt create mode 100644 acceptance/bundle/dms/no-drift/script diff --git a/acceptance/bundle/dms/no-drift/databricks.yml b/acceptance/bundle/dms/no-drift/databricks.yml new file mode 100644 index 00000000000..ca2ed6bd96d --- /dev/null +++ b/acceptance/bundle/dms/no-drift/databricks.yml @@ -0,0 +1,16 @@ +bundle: + name: dms-no-drift + +experimental: + record_deployment_history: true + +resources: + jobs: + foo: + name: foo + pipelines: + bar: + name: bar + catalog: main + schema: default + serverless: true diff --git a/acceptance/bundle/dms/no-drift/out.test.toml b/acceptance/bundle/dms/no-drift/out.test.toml new file mode 100644 index 00000000000..e90b6d5d1ba --- /dev/null +++ b/acceptance/bundle/dms/no-drift/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/dms/no-drift/output.txt b/acceptance/bundle/dms/no-drift/output.txt new file mode 100644 index 00000000000..0307d57910b --- /dev/null +++ b/acceptance/bundle/dms/no-drift/output.txt @@ -0,0 +1,58 @@ + +=== Deploy, then plan without touching anything: the deployment stamp must not show as drift +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-no-drift/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! +Deployment history: [DATABRICKS_URL]/deployments/[NUMID]?version=1 + +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged + +=== A second deploy is a no-op too: no update request for either resource +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-no-drift/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! +Deployment history: [DATABRICKS_URL]/deployments/[NUMID]?version=2 + +>>> print_requests.py //api/2.2/jobs //api/2.0/pipelines --sort +{ + "method": "POST", + "path": "/api/2.0/pipelines", + "body": { + "catalog": "main", + "channel": "CURRENT", + "deployment": { + "deployment_id": "[NUMID]", + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-no-drift/default/state/metadata.json", + "version_id": "1" + }, + "edition": "ADVANCED", + "name": "bar", + "schema": "default", + "serverless": true + } +} +{ + "method": "POST", + "path": "/api/2.2/jobs/create", + "body": { + "deployment": { + "deployment_id": "[NUMID]", + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-no-drift/default/state/metadata.json", + "version_id": "1" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "foo", + "queue": { + "enabled": true + } + } +} diff --git a/acceptance/bundle/dms/no-drift/script b/acceptance/bundle/dms/no-drift/script new file mode 100644 index 00000000000..c67c2c28a74 --- /dev/null +++ b/acceptance/bundle/dms/no-drift/script @@ -0,0 +1,12 @@ +title "Deploy, then plan without touching anything: the deployment stamp must not show as drift" +trace $CLI bundle deploy + +# Only the deploy phase stamps deployment.deployment_id (it is not known until the +# version is claimed), so plan sees it in the state and in the workspace but not in the +# local config. Without an ignore_local_changes rule that absence plans an update on a +# job and a pipeline nobody edited. +trace $CLI bundle plan | contains.py "Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged" + +title "A second deploy is a no-op too: no update request for either resource" +trace $CLI bundle deploy +trace print_requests.py //api/2.2/jobs //api/2.0/pipelines --sort diff --git a/bundle/direct/dresources/resources.yml b/bundle/direct/dresources/resources.yml index c9df091551f..7345f19c506 100644 --- a/bundle/direct/dresources/resources.yml +++ b/bundle/direct/dresources/resources.yml @@ -26,11 +26,17 @@ resources: # it changes constantly. Ignoring it as a local and remote change keeps that # churn from driving an update or showing as drift on its own; when the job is # updated for any other reason, DoUpdate sends the full config via Reset, so - # the current version_id is still recorded. deployment_id is intentionally - # left out: it is stable across versions, so a change to it is worth showing. + # the current version_id is still recorded. + # + # deployment_id is ignored as a local change for a different reason: only the + # deploy phase stamps it (it is not known until the version is claimed), so + # during a plain `bundle plan` the local config has none while the state and the + # workspace both do, and the absence would show as drift on an untouched job. ignore_local_changes: - field: deployment.version_id reason: managed by the deployment metadata service + - field: deployment.deployment_id + reason: managed by the deployment metadata service ignore_remote_changes: - field: deployment.version_id @@ -168,8 +174,8 @@ resources: - field: ingestion_definition.ingest_from_uc_foreign_catalog reason: immutable - # See jobs above: version_id is set on every deploy, so it is ignored as a - # local/remote change. deployment_id is left out so a change to it still shows. + # See jobs above: version_id is set on every deploy, and deployment_id is only + # stamped by the deploy phase, so both are ignored as local changes. ignore_remote_changes: - field: deployment.version_id reason: managed by the deployment metadata service @@ -184,6 +190,8 @@ resources: ignore_local_changes: - field: deployment.version_id reason: managed by the deployment metadata service + - field: deployment.deployment_id + reason: managed by the deployment metadata service # "id" is output-only, providing it in config would be a mistake - field: id reason: "!drop" From a49f16f5227c1a622b660cfbc024f7f039f1d1df Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Fri, 7 Aug 2026 00:07:57 +0000 Subject: [PATCH 044/125] bundle: group what a version records into dms.Metadata Provenance held the deployment mode, git info and workspace info, while the display name and target name travelled as separate RecorderOptions fields even though they are the same kind of thing: what a version records about the deploy, which the service denormalizes onto the deployment. They are now one Metadata struct, renamed from Provenance since it no longer describes only the source. Two not-found branches go away with it. A deployment ID is always the object ID of a BUNDLE_DEPLOYMENT node that get-status just returned, and the service has a deployment for every such node, so GetDeployment cannot report not-found for it. The recorder now says so instead of quietly retrying at version 1: internal error: no deployment found for the file with object id 2612372578 The fake server modelled the record as created by the first version rather than by CreateDeployment, which is what made those branches look reachable. It now creates both together, with last_version_id empty until the first version - so the "registered then failed" case still reaches the retry-at-1 path, via an empty last_version_id rather than a 404. InitializeDeploymentHistory was wired to InitIDs, so bundle open and three pipelines commands paid for its API calls while only bundle summary reports the result. It has its own option now. Also drops the bugbash guide this PR had added. Co-authored-by: Isaac --- .../mutator/initialize_deployment_history.go | 23 +-- bundle/phases/dms.go | 14 +- cmd/bundle/summary.go | 9 +- cmd/bundle/utils/process.go | 18 ++- internal/bugbash/README.md | 8 - internal/bugbash/record-deployment-history.md | 151 ------------------ libs/dms/recorder.go | 58 ++++--- libs/dms/recorder_test.go | 34 ++-- libs/testserver/bundle.go | 30 ++-- 9 files changed, 90 insertions(+), 255 deletions(-) delete mode 100644 internal/bugbash/record-deployment-history.md diff --git a/bundle/config/mutator/initialize_deployment_history.go b/bundle/config/mutator/initialize_deployment_history.go index 91a3f1352c9..99aad70c7cc 100644 --- a/bundle/config/mutator/initialize_deployment_history.go +++ b/bundle/config/mutator/initialize_deployment_history.go @@ -2,14 +2,11 @@ package mutator import ( "context" - "errors" "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dms" - "github.com/databricks/cli/libs/log" - "github.com/databricks/databricks-sdk-go/apierr" "github.com/databricks/databricks-sdk-go/service/bundledeployments" ) @@ -46,23 +43,19 @@ func (m *initializeDeploymentHistory) Apply(ctx context.Context, b *bundle.Bundl return nil } - history := &config.DeploymentHistory{DeploymentID: deploymentID} - - // The deployment's record is created by its first version, so a resolved ID can - // name a deployment that has none yet (a deploy that registered the deployment - // and then failed). Report the ID without a version rather than failing summary. + // The ID came from a BUNDLE_DEPLOYMENT node that get-status returned, and by + // design the service has a deployment for every such node, so this get does not + // have a not-found case. last_version_id is empty until the first version. dep, err := w.BundleDeployments.GetDeployment(ctx, bundledeployments.GetDeploymentRequest{ Name: "deployments/" + deploymentID, }) - switch { - case err == nil: - history.LatestVersionID = dep.LastVersionId - case errors.Is(err, apierr.ErrNotFound), errors.Is(err, apierr.ErrResourceDoesNotExist): - log.Debugf(ctx, "No deployment record for %s yet; reporting the ID without a version", deploymentID) - default: + if err != nil { return diag.FromErr(err) } - b.Config.Bundle.Deployment.History = history + b.Config.Bundle.Deployment.History = &config.DeploymentHistory{ + DeploymentID: deploymentID, + LatestVersionID: dep.LastVersionId, + } return nil } diff --git a/bundle/phases/dms.go b/bundle/phases/dms.go index aa6b1d8e810..cb8f6ad2db4 100644 --- a/bundle/phases/dms.go +++ b/bundle/phases/dms.go @@ -51,10 +51,8 @@ func newDeploymentRecorder(ctx context.Context, b *bundle.Bundle, eng engine.Eng Versions: dms.NewAPIVersionCreator(apiClient), DeploymentID: deploymentID, StatePath: statePath, - TargetName: b.Config.Bundle.Target, - DisplayName: b.Config.Bundle.Name, VersionType: versionType, - Provenance: deploymentProvenance(b), + Metadata: deploymentMetadata(b), }), nil } @@ -81,10 +79,14 @@ func logDeploymentHistory(ctx context.Context, b *bundle.Bundle, recorder *dms.R cmdio.LogString(ctx, "Deployment history: "+workspaceurls.DeploymentURL(*baseURL, recorder.DeploymentID(), recorder.Version())) } -// deploymentProvenance describes the source this deploy came from and where it +// deploymentMetadata describes the bundle this deploy came from and where it // landed, mirroring what bundle/deploy/metadata computes for the metadata file. -func deploymentProvenance(b *bundle.Bundle) dms.Provenance { - p := dms.Provenance{Mode: deploymentModeToSDK(b.Config.Bundle.Mode)} +func deploymentMetadata(b *bundle.Bundle) dms.Metadata { + p := dms.Metadata{ + DisplayName: b.Config.Bundle.Name, + TargetName: b.Config.Bundle.Target, + Mode: deploymentModeToSDK(b.Config.Bundle.Mode), + } git := b.Config.Bundle.Git if git.Branch != "" || git.Commit != "" || git.OriginURL != "" { diff --git a/cmd/bundle/summary.go b/cmd/bundle/summary.go index b3a55a607cc..d533517f998 100644 --- a/cmd/bundle/summary.go +++ b/cmd/bundle/summary.go @@ -27,10 +27,11 @@ Useful after deployment to see what was created and where to find it.`, cmd.RunE = func(cmd *cobra.Command, args []string) error { b, err := utils.ProcessBundle(cmd, utils.ProcessOptions{ - ReadState: true, - AlwaysPull: forcePull, - IncludeLocations: includeLocations, - InitIDs: true, + ReadState: true, + AlwaysPull: forcePull, + IncludeLocations: includeLocations, + InitIDs: true, + InitDeploymentHistory: true, }) if err != nil { return err diff --git a/cmd/bundle/utils/process.go b/cmd/bundle/utils/process.go index 30eefcac586..1cf67b52132 100644 --- a/cmd/bundle/utils/process.go +++ b/cmd/bundle/utils/process.go @@ -56,6 +56,12 @@ type ProcessOptions struct { // Implies ReadState InitIDs bool + // If true, calls InitializeDeploymentHistory() to look up the bundle's recorded + // deployment. Separate from InitIDs because it costs its own API calls and only + // 'bundle summary' reports the result. + // Implies InitIDs + InitDeploymentHistory bool + // if true, pass ErrorOnEmptyState to statemgmt.Load // Implies ReadState ErrorOnEmptyState bool @@ -89,6 +95,11 @@ func ProcessBundle(cmd *cobra.Command, opts ProcessOptions) (*bundle.Bundle, err func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle, stateDesc *statemgmt.StateDesc, retErr error) { var err error + // The deployment history is looked up alongside the resource IDs, so asking for + // it implies them. Normalized here so the options below only test InitIDs. + if opts.InitDeploymentHistory { + opts.InitIDs = true + } ctx := cmd.Context() if opts.SkipInitContext { if !logdiag.IsSetup(ctx) { @@ -258,9 +269,12 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle statemgmt.Load(state, modes...), } // InitializeURLs makes an extra API call; only run it when URLs are needed. - // InitializeDeploymentHistory likewise, and only for bundles that record it. if opts.InitIDs { - mutators = append(mutators, mutator.InitializeURLs(), mutator.InitializeDeploymentHistory()) + mutators = append(mutators, mutator.InitializeURLs()) + } + // Same for InitializeDeploymentHistory, which only 'bundle summary' reports. + if opts.InitDeploymentHistory { + mutators = append(mutators, mutator.InitializeDeploymentHistory()) } bundle.ApplySeqContext(ctx, b, mutators...) if logdiag.HasError(ctx) { diff --git a/internal/bugbash/README.md b/internal/bugbash/README.md index 1c5995a8188..941ab6227cc 100644 --- a/internal/bugbash/README.md +++ b/internal/bugbash/README.md @@ -11,11 +11,3 @@ but works without command completion with earlier versions. ```shell bash <(curl -fsSL https://raw.githubusercontent.com/databricks/cli/main/internal/bugbash/exec.sh) my-branch ``` - -The branch needs a successful `release-build` run to download a snapshot from. That -workflow runs on `main` and on any branch named `demo-*` or `bugbash-*`, so push the -branch under one of those names. - -## Feature guides - -- [Deployment history recording](./record-deployment-history.md) diff --git a/internal/bugbash/record-deployment-history.md b/internal/bugbash/record-deployment-history.md deleted file mode 100644 index db0a7172653..00000000000 --- a/internal/bugbash/record-deployment-history.md +++ /dev/null @@ -1,151 +0,0 @@ -# Bugbash: deployment history recording - -Records every `bundle deploy` and `bundle destroy` with the Deployment Metadata -Service (DMS), so a deployment has a server-side history and its resource state -lives in the workspace rather than only in the local cache. - -## Get a CLI with the feature - -```shell -bash <(curl -fsSL https://raw.githubusercontent.com/databricks/cli/main/internal/bugbash/exec.sh) bugbash-record-deployment-history -``` - -That drops you into a shell with `databricks` on `$PATH`. Check you have the right -build with `databricks --version`. - -## Turn the feature on - -Three things are needed. Missing any one of them means nothing is recorded. - -```shell -export DATABRICKS_BUNDLE_ENGINE=direct -export DATABRICKS_BUNDLE_FORCE_ALLOW_RECORD_DEPLOYMENT_HISTORY=true -``` - -and in `databricks.yml`: - -```yaml -experimental: - record_deployment_history: true -``` - -The env var only unlocks the gate; the YAML flag is what enables recording. Without -the env var the CLI refuses: - -``` -Error: experimental.record_deployment_history is not supported yet -``` - -Recording is direct-engine only. On terraform the flag is rejected, and no -`/api/2.0/bundle/*` calls are made. - -The feature must be enabled from the bundle's **first** deploy. Turning it on for a -bundle that already has deployed resources is refused, because DMS would then own a -resource set it never saw and the next deploy would create everything a second time. -The error spells out the three steps to start over. - -## A bundle to start from - -```yaml -bundle: - name: my-dms-test - -experimental: - record_deployment_history: true - -resources: - jobs: - hello: - name: my-dms-test-job - tasks: - - task_key: main - notebook_task: - notebook_path: ./noop.py -``` - -with `noop.py` beside it: - -``` -# Databricks notebook source -print(1) -``` - -## Find the deployment - -The CLI stores the deployment ID nowhere. DMS registers the deployment as a workspace -node, and that node's object ID *is* the deployment ID: - -```shell -databricks workspace get-status \ - "/Workspace/Users/$(databricks current-user me | jq -r .userName)/.bundle/my-dms-test/default/state/resources.deployment.json" \ - -o json | python3 -c 'import sys,json; print(json.load(sys.stdin)["object_id"])' -``` - -Use `python3`, not `jq`, for that ID. It exceeds 2^53 and jq below 1.7 silently -rounds it, which looks like "deployment does not exist". - -Or read it straight off the summary: - -```shell -databricks bundle summary -o json | jq .bundle.deployment.history -``` - -## What to look at - -```shell -databricks api get "/api/2.0/bundle/deployments/$DID" # the deployment -databricks api get "/api/2.0/bundle/deployments/$DID/versions" # one version per deploy -databricks api get "/api/2.0/bundle/deployments/$DID/versions/$V/operations" -databricks api get "/api/2.0/bundle/deployments/$DID/resources" # current resource state -databricks api get "/api/2.0/bundle/deployments" # all deployments -``` - -`resources` and `operations` paginate at 20 with a `next_page_token`. A bundle with -more than 20 resources is not truncated; page through it. - -Jobs and pipelines carry a back-reference to the deployment, but the SDK hides those -fields, so read them raw: - -```shell -databricks api get "/api/2.0/jobs/get?job_id=$JID" | jq .settings.deployment -databricks api get "/api/2.0/pipelines/$PID" | jq .spec.deployment -``` - -Both should show `deployment_id` and `version_id` next to `kind: BUNDLE`. - -## Worth exercising - -- **Iterate.** Deploy, change a field, deploy again. Each deploy claims a version; - only changed resources get an operation. -- **Wipe the local cache.** `rm -rf .databricks`, then `bundle plan`. It should report - your resources as unchanged, reconstructed from DMS. It must never plan to create - something that already exists. -- **Break a resource.** Give a job an invalid cron expression. The failed resource is - recorded with `status: OPERATION_STATUS_FAILED` and an `error_message`, the version - completes with `VERSION_COMPLETE_FAILURE`, and a later plan wants to create it. -- **Destroy.** A destroy records its own version with a DELETE per resource, then - deletes the deployment record. -- **Non-job resources.** Pipelines, schemas, volumes, experiments, registered models, - secret scopes and dashboards are all recorded. Each has a differently-shaped - resource id (numeric, UUID, `catalog.schema.name`, a scope name). -- **Targets.** Each target has its own state path, so `-t dev` and `-t prod` are - separate deployments with separate version chains. -- **Provenance.** Deploy from a git repo and check `git_info` on the version; - `deployment_mode` reflects the target's `mode`. - -## Not bugs - -- A redeploy with no changes still creates a version, with no operations under it. -- After `destroy`, `GetDeployment` still returns the record with - `status: DEPLOYMENT_STATUS_DELETED`. That is a soft delete. -- `state` on an operation or resource is a **quoted JSON string**, not an embedded - object. Parse it once to get `{"state": {...}, "depends_on": [...]}`. -- DMS resource keys have no `resources.` prefix (`jobs.foo`), unlike local state keys. -- Sub-resources get their own operation, e.g. `secret_scopes.mine.permissions`. -- Permissions are not set on the deployment node. It inherits from the state folder, - which the bundle's `permissions:` section already governs. - -## Reporting - -Include the deployment ID, the version, and the request/response for anything that -looks wrong. `databricks bundle deploy --log-level debug` logs the DMS calls. diff --git a/libs/dms/recorder.go b/libs/dms/recorder.go index 0306507913a..eda0233931b 100644 --- a/libs/dms/recorder.go +++ b/libs/dms/recorder.go @@ -94,10 +94,8 @@ type Recorder struct { versions versionCreator deploymentID string statePath string - targetName string - displayName string versionType VersionType - provenance Provenance + metadata Metadata // populated by CreateVersion versionNum int64 @@ -121,17 +119,19 @@ type RecorderOptions struct { // StatePath is the bundle's remote state directory, under which DMS registers // the deployment node. StatePath string - TargetName string - DisplayName string VersionType VersionType - // Provenance records where the deployed source came from; see Provenance. - Provenance Provenance + // Metadata is what the version records about the deploy; see Metadata. + Metadata Metadata } -// Provenance is what a version records about the source it deployed and where it -// landed. The service denormalizes these onto the deployment, so they describe the -// deployment as of its most recent version. -type Provenance struct { +// Metadata is what a version records about the bundle it deployed, the source it +// came from, and where it landed. The service denormalizes these onto the +// deployment, so they describe the deployment as of its most recent version. +type Metadata struct { + // DisplayName is the bundle's name, which the deployment is listed under. + DisplayName string + // TargetName is the bundle target that was deployed. + TargetName string // Mode is the bundle target's mode, empty when the target sets none. Mode bundledeployments.DeploymentMode Git *bundledeployments.GitInfo @@ -145,10 +145,8 @@ func NewRecorder(opts RecorderOptions) *Recorder { versions: opts.Versions, deploymentID: opts.DeploymentID, statePath: opts.StatePath, - targetName: opts.TargetName, - displayName: opts.DisplayName, versionType: opts.VersionType, - provenance: opts.Provenance, + metadata: opts.Metadata, } } @@ -246,30 +244,28 @@ func (r *Recorder) createDeploymentVersion(ctx context.Context) (versionID strin // deployment's first version. var previousVersionID string if r.deploymentID != "" { - // A resolved node names the deployment, but its record is created by the - // first version, so there may be none yet: a deploy that registered the - // deployment and then failed before recording a version. Start at version 1 - // under the ID the node already names, rather than creating a second - // deployment, which would collide on the same node path. + // The ID came from a BUNDLE_DEPLOYMENT node that get-status returned, and by + // design the service has a deployment for every such node, so a not-found + // here means that invariant is broken rather than anything the user did. dep, getErr := r.svc.GetDeployment(ctx, bundledeployments.GetDeploymentRequest{ Name: "deployments/" + r.deploymentID, }) switch { - case getErr == nil && dep.LastVersionId == "": + case errors.Is(getErr, apierr.ErrNotFound), errors.Is(getErr, apierr.ErrResourceDoesNotExist): + return "", fmt.Errorf("internal error: no deployment found for the file with object id %s: %w", r.deploymentID, getErr) + case getErr != nil: + return "", fmt.Errorf("failed to get deployment: %w", getErr) + case dep.LastVersionId == "": // The record exists but carries no version: a deploy whose first version was // rejected still leaves the record behind. Retry at version 1. versionID = "1" - case getErr == nil: + default: lastVersion, parseErr := strconv.ParseInt(dep.LastVersionId, 10, 64) if parseErr != nil { return "", fmt.Errorf("failed to parse last_version_id %q: %w", dep.LastVersionId, parseErr) } versionID = strconv.FormatInt(lastVersion+1, 10) previousVersionID = dep.LastVersionId - case errors.Is(getErr, apierr.ErrNotFound), errors.Is(getErr, apierr.ErrResourceDoesNotExist): - versionID = "1" - default: - return "", fmt.Errorf("failed to get deployment: %w", getErr) } } else { // First deploy: create the deployment so the server assigns an ID. @@ -281,7 +277,7 @@ func (r *Recorder) createDeploymentVersion(ctx context.Context) (versionID strin dep, createErr := r.svc.CreateDeployment(ctx, bundledeployments.CreateDeploymentRequest{ Deployment: bundledeployments.Deployment{ InitialParentPath: r.statePath, - TargetName: r.targetName, + TargetName: r.metadata.TargetName, }, }) if createErr != nil { @@ -301,12 +297,12 @@ func (r *Recorder) createDeploymentVersion(ctx context.Context) (versionID strin version, versionErr := r.versions.CreateVersion(ctx, r.deploymentID, versionID, createVersionRequest{ CliVersion: build.GetInfo().Version, VersionType: r.versionType, - TargetName: r.targetName, - DisplayName: r.displayName, + TargetName: r.metadata.TargetName, + DisplayName: r.metadata.DisplayName, PreviousVersionId: previousVersionID, - DeploymentMode: r.provenance.Mode, - GitInfo: r.provenance.Git, - WorkspaceInfo: r.provenance.Workspace, + DeploymentMode: r.metadata.Mode, + GitInfo: r.metadata.Git, + WorkspaceInfo: r.metadata.Workspace, }) if versionErr != nil { return "", fmt.Errorf("failed to create deployment version: %w", versionErr) diff --git a/libs/dms/recorder_test.go b/libs/dms/recorder_test.go index 694fab0699a..56d5d41631d 100644 --- a/libs/dms/recorder_test.go +++ b/libs/dms/recorder_test.go @@ -87,7 +87,7 @@ func (f *fakeDMS) Heartbeat(ctx context.Context, req bundledeployments.Heartbeat func TestRecorderFirstDeployCreatesDeploymentWithServerAssignedID(t *testing.T) { f := &fakeDMS{assignedID: "server-generated-id"} // A first deploy resolves no deployment ID from the workspace. - r := NewRecorder(RecorderOptions{Service: f, Versions: fakeVersions{requests: &f.versions}, StatePath: testStatePath, TargetName: "dev", DisplayName: testDisplayName, VersionType: VersionTypeDeploy}) + r := NewRecorder(RecorderOptions{Service: f, Versions: fakeVersions{requests: &f.versions}, StatePath: testStatePath, Metadata: Metadata{TargetName: "dev", DisplayName: testDisplayName}, VersionType: VersionTypeDeploy}) require.NoError(t, r.CreateVersion(t.Context())) @@ -118,7 +118,7 @@ func TestRecorderSubsequentDeployReusesDeploymentAndIncrementsVersion(t *testing }, } // A subsequent deploy passes the stored deployment ID. - r := NewRecorder(RecorderOptions{Service: f, Versions: fakeVersions{requests: &f.versions}, DeploymentID: "stored-id", StatePath: testStatePath, TargetName: "dev", DisplayName: testDisplayName, VersionType: VersionTypeDeploy}) + r := NewRecorder(RecorderOptions{Service: f, Versions: fakeVersions{requests: &f.versions}, DeploymentID: "stored-id", StatePath: testStatePath, Metadata: Metadata{TargetName: "dev", DisplayName: testDisplayName}, VersionType: VersionTypeDeploy}) require.NoError(t, r.CreateVersion(t.Context())) @@ -134,7 +134,7 @@ func TestRecorderSubsequentDeployReusesDeploymentAndIncrementsVersion(t *testing func TestRecorderSendsDisplayNameAndNoPreviousVersionOnFirstDeploy(t *testing.T) { f := &fakeDMS{assignedID: "server-generated-id"} - r := NewRecorder(RecorderOptions{Service: f, Versions: fakeVersions{requests: &f.versions}, StatePath: testStatePath, TargetName: "dev", DisplayName: testDisplayName, VersionType: VersionTypeDeploy}) + r := NewRecorder(RecorderOptions{Service: f, Versions: fakeVersions{requests: &f.versions}, StatePath: testStatePath, Metadata: Metadata{TargetName: "dev", DisplayName: testDisplayName}, VersionType: VersionTypeDeploy}) require.NoError(t, r.CreateVersion(t.Context())) @@ -153,30 +153,28 @@ func TestRecorderGetDeploymentErrorFailsDeploy(t *testing.T) { return nil, errors.New("boom") }, } - r := NewRecorder(RecorderOptions{Service: f, Versions: fakeVersions{requests: &f.versions}, DeploymentID: "stored-id", StatePath: testStatePath, TargetName: "dev", DisplayName: testDisplayName, VersionType: VersionTypeDeploy}) + r := NewRecorder(RecorderOptions{Service: f, Versions: fakeVersions{requests: &f.versions}, DeploymentID: "stored-id", StatePath: testStatePath, Metadata: Metadata{TargetName: "dev", DisplayName: testDisplayName}, VersionType: VersionTypeDeploy}) err := r.CreateVersion(t.Context()) assert.ErrorContains(t, err, "failed to get deployment") assert.Empty(t, f.created) } -func TestRecorderMissingDeploymentRecordStartsAtVersionOne(t *testing.T) { - // The record is created by the first version, so a node can name a deployment - // that has none yet - an earlier deploy registered it and then failed. Record - // version 1 under that same ID instead of creating a second deployment, which - // would collide on the node path. +func TestRecorderMissingDeploymentIsInternalError(t *testing.T) { + // The service has a deployment for every BUNDLE_DEPLOYMENT node, so a not-found + // for a node get-status just returned is a broken invariant, not a state the + // deploy can recover from. f := &fakeDMS{ getDeployment: func(id string) (*bundledeployments.Deployment, error) { return nil, fmt.Errorf("deployment: %w", apierr.ErrNotFound) }, } - r := NewRecorder(RecorderOptions{Service: f, Versions: fakeVersions{requests: &f.versions}, DeploymentID: "stored-id", StatePath: testStatePath, TargetName: "dev", DisplayName: testDisplayName, VersionType: VersionTypeDeploy}) + r := NewRecorder(RecorderOptions{Service: f, Versions: fakeVersions{requests: &f.versions}, DeploymentID: "stored-id", StatePath: testStatePath, Metadata: Metadata{TargetName: "dev", DisplayName: testDisplayName}, VersionType: VersionTypeDeploy}) - require.NoError(t, r.CreateVersion(t.Context())) + err := r.CreateVersion(t.Context()) + assert.ErrorContains(t, err, "internal error: no deployment found for the file with object id stored-id") assert.Empty(t, f.created) - require.Len(t, f.versions, 1) - assert.Equal(t, "1", f.versions[0].versionID) - assert.Equal(t, "stored-id", f.versions[0].deploymentID) + assert.Empty(t, f.versions) } func TestRecorderDestroyDeletesDeploymentOnSuccess(t *testing.T) { @@ -185,7 +183,7 @@ func TestRecorderDestroyDeletesDeploymentOnSuccess(t *testing.T) { return &bundledeployments.Deployment{Name: "deployments/" + id, LastVersionId: "2"}, nil }, } - r := NewRecorder(RecorderOptions{Service: f, Versions: fakeVersions{requests: &f.versions}, DeploymentID: "stored-id", StatePath: testStatePath, TargetName: "dev", DisplayName: testDisplayName, VersionType: VersionTypeDestroy}) + r := NewRecorder(RecorderOptions{Service: f, Versions: fakeVersions{requests: &f.versions}, DeploymentID: "stored-id", StatePath: testStatePath, Metadata: Metadata{TargetName: "dev", DisplayName: testDisplayName}, VersionType: VersionTypeDestroy}) require.NoError(t, r.CreateVersion(t.Context())) assert.Equal(t, bundledeployments.VersionTypeVersionTypeDestroy, f.versions[0].body.VersionType) @@ -201,7 +199,7 @@ func TestRecorderFailedDestroyKeepsDeployment(t *testing.T) { return &bundledeployments.Deployment{Name: "deployments/" + id, LastVersionId: "2"}, nil }, } - r := NewRecorder(RecorderOptions{Service: f, Versions: fakeVersions{requests: &f.versions}, DeploymentID: "stored-id", StatePath: testStatePath, TargetName: "dev", DisplayName: testDisplayName, VersionType: VersionTypeDestroy}) + r := NewRecorder(RecorderOptions{Service: f, Versions: fakeVersions{requests: &f.versions}, DeploymentID: "stored-id", StatePath: testStatePath, Metadata: Metadata{TargetName: "dev", DisplayName: testDisplayName}, VersionType: VersionTypeDestroy}) require.NoError(t, r.CreateVersion(t.Context())) require.NoError(t, r.CompleteVersion(t.Context(), false)) @@ -220,7 +218,7 @@ func TestRecorderCompleteVersionIsIdempotent(t *testing.T) { return &bundledeployments.Deployment{Name: "deployments/" + id, LastVersionId: "2"}, nil }, } - r := NewRecorder(RecorderOptions{Service: f, Versions: fakeVersions{requests: &f.versions}, DeploymentID: "stored-id", StatePath: testStatePath, TargetName: "dev", DisplayName: testDisplayName, VersionType: VersionTypeDestroy}) + r := NewRecorder(RecorderOptions{Service: f, Versions: fakeVersions{requests: &f.versions}, DeploymentID: "stored-id", StatePath: testStatePath, Metadata: Metadata{TargetName: "dev", DisplayName: testDisplayName}, VersionType: VersionTypeDestroy}) require.NoError(t, r.CreateVersion(t.Context())) require.NoError(t, r.CompleteVersion(t.Context(), true)) @@ -241,7 +239,7 @@ func TestNilRecorderIsNoOp(t *testing.T) { func TestRecorderCompleteVersionNoOpWithoutCreateVersion(t *testing.T) { f := &fakeDMS{} - r := NewRecorder(RecorderOptions{Service: f, Versions: fakeVersions{requests: &f.versions}, DeploymentID: "stored-id", StatePath: testStatePath, TargetName: "dev", DisplayName: testDisplayName, VersionType: VersionTypeDeploy}) + r := NewRecorder(RecorderOptions{Service: f, Versions: fakeVersions{requests: &f.versions}, DeploymentID: "stored-id", StatePath: testStatePath, Metadata: Metadata{TargetName: "dev", DisplayName: testDisplayName}, VersionType: VersionTypeDeploy}) // CompleteVersion before CreateVersion is a no-op (nothing was claimed). require.NoError(t, r.CompleteVersion(t.Context(), true)) assert.Empty(t, f.completed) diff --git a/libs/testserver/bundle.go b/libs/testserver/bundle.go index 4f2571691a2..74a8d2156b3 100644 --- a/libs/testserver/bundle.go +++ b/libs/testserver/bundle.go @@ -67,15 +67,20 @@ func (s *FakeWorkspace) CreateDeployment(req Request) Response { }, } - // Only the node is created here. The deployment record itself is created by the - // first CreateVersion, so a client that creates a deployment and then fails - // before recording a version leaves no record behind - just the node, which - // names the ID that first version will be created under. + // The record is created together with the node, so a get on it always resolves + // for a node that exists. It carries no version yet: last_version_id stays empty + // until the first CreateVersion, which is how a client that registers a + // deployment and then fails leaves a record with no versions. deploymentID := strconv.FormatInt(objectID, 10) s.dmsDeploymentNodes[deploymentID] = nodePath dep.Name = "deployments/" + deploymentID dep.Status = bundledeployments.DeploymentStatusDeploymentStatusActive + s.dmsDeployments[deploymentID] = &dmsDeployment{ + deployment: dep, + versions: map[string]*bundledeployments.Version{}, + resources: map[string]bundledeployments.Resource{}, + } return Response{Body: dep} } @@ -154,22 +159,7 @@ func (s *FakeWorkspace) CreateVersion(req Request, deploymentID string) Response d, ok := s.dmsDeployments[deploymentID] if !ok { - // The deployment record is created by its first version, not by - // CreateDeployment. That call only registered the workspace node, so the node - // existing is what makes this ID valid. - if _, known := s.dmsDeploymentNodes[deploymentID]; !known { - return dmsNotFound("deployment " + deploymentID) - } - d = &dmsDeployment{ - deployment: bundledeployments.Deployment{ - Name: "deployments/" + deploymentID, - Status: bundledeployments.DeploymentStatusDeploymentStatusActive, - TargetName: version.TargetName, - }, - versions: map[string]*bundledeployments.Version{}, - resources: map[string]bundledeployments.Resource{}, - } - s.dmsDeployments[deploymentID] = d + return dmsNotFound("deployment " + deploymentID) } // Mirror the server-side checks: version_id must be numerically greater than From 37a8b942fed90c15d761931f734622617ae6be35 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Fri, 7 Aug 2026 00:13:09 +0000 Subject: [PATCH 045/125] bundle: shorten the comment on the DMS upload-failure check Co-authored-by: Isaac --- bundle/direct/bundle_apply.go | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/bundle/direct/bundle_apply.go b/bundle/direct/bundle_apply.go index 2047f65a498..9fa882867fc 100644 --- a/bundle/direct/bundle_apply.go +++ b/bundle/direct/bundle_apply.go @@ -69,12 +69,7 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa return false } - // Stop before touching the workspace once recording an operation has failed. - // A completed version makes DMS the source of truth for resource state (see - // dstate.readDMSState), so continuing would create resources it has no record - // of and the next deploy would create them a second time. Checked here rather - // than only where operations are recorded, which is after the resource has - // already been modified. + // Stop resource CRUD once uploading DMS state has failed. if err := opQueue.firstErr(); err != nil { logdiag.LogError(ctx, fmt.Errorf("%s: %w", errorPrefix, err)) return false From 636c9cfbae9db628e90a3a4fbcbad5beb3e8c38d Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Fri, 7 Aug 2026 00:49:33 +0000 Subject: [PATCH 046/125] bundle: stamp the deployment before planning, and keep recording after a failure Three changes. The phantom drift on deployment.deployment_id is fixed by setting the field instead of ignoring it. AnnotateDeploymentVersion stamped the id and the version together from the deploy phase, so a plain `bundle plan` left the id empty while the state and the workspace both had it, and the absence read as a change on an untouched job. It splits into AnnotateDeployment (the id, applied where the state is opened, which is before anything diffs) and AnnotateDeploymentVersion (the version, which does not exist until CreateVersion claims one). The ignore_local_changes entries added for deployment_id are dropped: resources.yml is back to what main has, and the local config now matches the workspace rather than hiding a mismatch. A failed upload no longer stops the ones behind it. record refused new work once any upload had failed, so a resource that was applied went unrecorded and DMS drifted from reality in the other direction. The workers already continued past a failure; now record does too, and close still reports the error, so the deploy fails either way. Stopping resource CRUD is unchanged - bundle_apply still checks firstErr before touching the workspace, which is the check that matters. Comments through the DMS files are cut to one or two lines each. opqueue.go keeps its structure, since the concurrency rules there are not obvious from the code. Verified on dogfood: `bundle plan` on an untouched bundle reports 0 to change with no ignore rule, and the deployed job still carries deployment_id. Co-authored-by: Isaac --- .../metadata/annotate_deployment_version.go | 50 +++++-- .../annotate_deployment_version_test.go | 2 +- bundle/direct/dresources/resources.yml | 26 ++-- bundle/direct/dstate/dms.go | 37 ++--- bundle/direct/opqueue.go | 135 ++++++------------ bundle/direct/opqueue_test.go | 25 ++-- bundle/phases/deploy.go | 16 ++- cmd/bundle/utils/process.go | 10 ++ libs/dms/recorder.go | 57 +++----- 9 files changed, 155 insertions(+), 203 deletions(-) diff --git a/bundle/deploy/metadata/annotate_deployment_version.go b/bundle/deploy/metadata/annotate_deployment_version.go index 1b8139168ce..46e0ab7ed41 100644 --- a/bundle/deploy/metadata/annotate_deployment_version.go +++ b/bundle/deploy/metadata/annotate_deployment_version.go @@ -8,21 +8,46 @@ import ( "github.com/databricks/cli/libs/diag" ) -type annotateDeploymentVersion struct { +type annotateDeployment struct { deploymentID string - version int64 } -// AnnotateDeploymentVersion stamps the DMS deployment and version onto every job -// and pipeline, so a resource in the workspace points back at the deployment that -// produced it (which is how lineage resolves a job to its bundle). +// AnnotateDeployment stamps the DMS deployment onto every job and pipeline, so a +// resource in the workspace points back at the deployment that produced it (which is +// how lineage resolves a job to its bundle). // -// AnnotateJobs/AnnotatePipelines set the rest of the deployment metadata during -// initialize, but the version - and, on a first deploy, the deployment ID - only -// exist once CreateVersion has run, so these two fields are stamped separately -// from the deploy phase. -func AnnotateDeploymentVersion(deploymentID string, version int64) bundle.Mutator { - return &annotateDeploymentVersion{deploymentID: deploymentID, version: version} +// It runs before the plan is computed, since a resource whose deployment is unset +// locally but set in the workspace would otherwise show as drift. +func AnnotateDeployment(deploymentID string) bundle.Mutator { + return &annotateDeployment{deploymentID: deploymentID} +} + +func (m *annotateDeployment) Name() string { + return "metadata.AnnotateDeployment" +} + +func (m *annotateDeployment) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { + for _, job := range b.Config.Resources.Jobs { + // Deployment is set by AnnotateJobs, which runs during initialize. + job.Deployment.DeploymentId = m.deploymentID + } + + for _, pipeline := range b.Config.Resources.Pipelines { + pipeline.Deployment.DeploymentId = m.deploymentID + } + + return nil +} + +type annotateDeploymentVersion struct { + version int64 +} + +// AnnotateDeploymentVersion stamps the DMS version onto every job and pipeline. It +// is separate from AnnotateDeployment because the version only exists once +// CreateVersion has claimed one, which happens during deploy. +func AnnotateDeploymentVersion(version int64) bundle.Mutator { + return &annotateDeploymentVersion{version: version} } func (m *annotateDeploymentVersion) Name() string { @@ -33,13 +58,10 @@ func (m *annotateDeploymentVersion) Apply(_ context.Context, b *bundle.Bundle) d versionID := strconv.FormatInt(m.version, 10) for _, job := range b.Config.Resources.Jobs { - // Deployment is set by AnnotateJobs, which runs during initialize. - job.Deployment.DeploymentId = m.deploymentID job.Deployment.VersionId = versionID } for _, pipeline := range b.Config.Resources.Pipelines { - pipeline.Deployment.DeploymentId = m.deploymentID pipeline.Deployment.VersionId = versionID } diff --git a/bundle/deploy/metadata/annotate_deployment_version_test.go b/bundle/deploy/metadata/annotate_deployment_version_test.go index aca51358ace..297ba95f8ee 100644 --- a/bundle/deploy/metadata/annotate_deployment_version_test.go +++ b/bundle/deploy/metadata/annotate_deployment_version_test.go @@ -34,7 +34,7 @@ func TestAnnotateDeploymentVersion(t *testing.T) { }, } - diags := bundle.ApplySeq(t.Context(), b, AnnotateDeploymentVersion("dep-123", 7)) + diags := bundle.ApplySeq(t.Context(), b, AnnotateDeployment("dep-123"), AnnotateDeploymentVersion(7)) require.NoError(t, diags.Error()) job := b.Config.Resources.Jobs["my-job"].Deployment diff --git a/bundle/direct/dresources/resources.yml b/bundle/direct/dresources/resources.yml index 7345f19c506..6061db0fb04 100644 --- a/bundle/direct/dresources/resources.yml +++ b/bundle/direct/dresources/resources.yml @@ -26,17 +26,11 @@ resources: # it changes constantly. Ignoring it as a local and remote change keeps that # churn from driving an update or showing as drift on its own; when the job is # updated for any other reason, DoUpdate sends the full config via Reset, so - # the current version_id is still recorded. - # - # deployment_id is ignored as a local change for a different reason: only the - # deploy phase stamps it (it is not known until the version is claimed), so - # during a plain `bundle plan` the local config has none while the state and the - # workspace both do, and the absence would show as drift on an untouched job. + # the current version_id is still recorded. deployment_id is intentionally + # left out: it is stable across versions, so a change to it is worth showing. ignore_local_changes: - field: deployment.version_id reason: managed by the deployment metadata service - - field: deployment.deployment_id - reason: managed by the deployment metadata service ignore_remote_changes: - field: deployment.version_id @@ -174,8 +168,8 @@ resources: - field: ingestion_definition.ingest_from_uc_foreign_catalog reason: immutable - # See jobs above: version_id is set on every deploy, and deployment_id is only - # stamped by the deploy phase, so both are ignored as local changes. + # See jobs above: version_id is set on every deploy, so it is ignored as a + # local/remote change. deployment_id is left out so a change to it still shows. ignore_remote_changes: - field: deployment.version_id reason: managed by the deployment metadata service @@ -184,14 +178,22 @@ resources: # Thus it shows up as a remote change since we don't set on the object. - field: id reason: "!drop" + # QQQ should this be here? When run_as is explicitly set, the GET response echoes it back + # as a structured run_as.user_name (verified on e2-dogfood with a real user), so it may not + # be truly input-only. The explicit-set case could not be confirmed on aws-cli, azure-cli, + # or gcp-cli: those envs authenticate as a service principal that lacks servicePrincipal.user + # on itself, so it can't self-bind run_as. In the default (unset) case on all three clouds, + # GET returns only the flat run_as_user_name and no structured run_as. - field: run_as reason: input_only + # Carried by CreatePipeline/EditPipeline but never returned by GET, so remote + # always reads back false and a config value of true never converges. + - field: allow_duplicate_names + reason: input_only ignore_local_changes: - field: deployment.version_id reason: managed by the deployment metadata service - - field: deployment.deployment_id - reason: managed by the deployment metadata service # "id" is output-only, providing it in config would be a mistake - field: id reason: "!drop" diff --git a/bundle/direct/dstate/dms.go b/bundle/direct/dstate/dms.go index 3d7131580e2..4ec68dea82d 100644 --- a/bundle/direct/dstate/dms.go +++ b/bundle/direct/dstate/dms.go @@ -3,48 +3,29 @@ package dstate import ( "context" "encoding/json" - "errors" "fmt" "github.com/databricks/cli/bundle/deployplan" - "github.com/databricks/cli/libs/log" - "github.com/databricks/databricks-sdk-go/apierr" "github.com/databricks/databricks-sdk-go/service/bundledeployments" ) -// RecordedState is what the CLI serializes into the DMS Operation.State field. -// -// It is an envelope rather than the bare resource config, because depends_on has -// to survive the round trip: DMS has no field for dependency edges, and they -// cannot be recomputed from the config once it is recorded (references are -// resolved to literals before serialization). Nesting depends_on inside the -// config instead would collide with resource fields of the same name, e.g. -// jobs.Task.depends_on. -// -// The shape deliberately matches the local ResourceEntry so both sides of the -// state round trip look the same. +// RecordedState is what the CLI serializes into the DMS Operation.State field. It +// wraps the config rather than being it, so depends_on survives the round trip: DMS +// has no field for dependency edges, and they cannot be recomputed once references +// are resolved to literals. Nesting them in the config would collide with resource +// fields of the same name (e.g. jobs.Task.depends_on). type RecordedState struct { State json.RawMessage `json:"state"` DependsOn []deployplan.DependsOnEntry `json:"depends_on,omitempty"` } -// readDMSState replaces the file-derived resource state with the state recorded -// in DMS. Recording is only enabled for net-new deployments, so once a -// deployment exists DMS owns its resource set outright - including when that set -// is empty, which is a successful deploy of nothing rather than missing data. -// The caller holds db.mu. +// readDMSState replaces the file-derived resource state with the state recorded in +// DMS. Recording is only enabled for net-new deployments, so once a deployment +// exists DMS owns its resource set outright - an empty set means a successful deploy +// of nothing, not missing data. The caller holds db.mu. func (db *DeploymentState) readDMSState(ctx context.Context, src *DMSSource) error { resources, err := fetchDeploymentResources(ctx, src.Client, src.DeploymentID) if err != nil { - // The deployment's record is created by its first version, so a node can - // resolve to an ID that has none yet: a deploy that registered the deployment - // and then failed before recording a version. There is nothing to read, and - // the file's resources are still empty, so carry on and let this deploy record - // the first version. - if errors.Is(err, apierr.ErrNotFound) || errors.Is(err, apierr.ErrResourceDoesNotExist) { - log.Debugf(ctx, "No deployment record for %s yet; keeping local state", src.DeploymentID) - return nil - } return err } diff --git a/bundle/direct/opqueue.go b/bundle/direct/opqueue.go index ba64d52ebd9..ec5b97ac37d 100644 --- a/bundle/direct/opqueue.go +++ b/bundle/direct/opqueue.go @@ -11,46 +11,30 @@ import ( ) const ( - // operationQueueSize bounds how many recorded operations wait for upload. - // Apply deploys at most defaultParallelism resources at a time, so a queue - // this deep means an apply worker practically never blocks on a free slot. + // operationQueueSize bounds how many recorded operations wait for upload. Deep + // enough that an apply worker practically never blocks on a free slot. operationQueueSize = 10 - // operationUploadWorkers is how many uploads run at a time. It is below - // operationQueueSize so a burst of operations is absorbed by the queue rather - // than by one request per resource. - // - // This was temporarily capped at 2 while concurrent CreateOperation calls under - // the same version contended on shared state server-side, surfacing the - // transaction conflict as a 500 that failed the deploy. The service now handles - // them: measured against it, 8 and 16 concurrent writes both succeed where 4 - // used to fail. + // operationUploadWorkers is how many uploads run at a time. operationUploadWorkers = 8 ) -// operationQueue hands recorded operations to background workers, so an apply -// worker does not wait for the CreateOperation round trip before deploying the -// next resource. +// operationQueue uploads recorded operations from background workers, so a deploy +// never waits on the CreateOperation round trip. Two rules shape it: // -// Two rules shape the design: +// - One resource, one upload at a time. DMS keeps a single state per resource, so +// overlapping uploads could land out of order and leave the older state. +// - Newest operation wins. Each carries the resource's full state, so a queued +// operation superseded by a newer one is dropped ("coalesced"). // -// - Uploads for one resource never overlap. DMS stores one state per resource -// key, so concurrent uploads could land out of order and leave stale state. -// - Only the newest operation for a resource matters. Each operation carries the -// resource's full state, not a delta, so a newer one entirely supersedes an -// older one. When both are still waiting, the older is dropped ("coalesced") -// and one upload records the result. -// -// Uploads are not fire-and-forget: close returns the first failure and fails the -// deploy. A dropped operation would leave DMS with an incomplete resource set, -// and since DMS then becomes the source of truth (see dstate.readDMSState), the -// next deploy would recreate resources that already exist. +// close reports the first upload failure, which fails the deploy: DMS becomes the +// source of truth (see dstate.readDMSState), so a missing record would make the +// next deploy create a resource that already exists. type operationQueue struct { uploader operationUploader - // queue carries resource keys, not operations. A worker looks the operation up - // when it picks the key up, so recording again before then just overwrites the - // entry in pending - that is what makes coalescing work. + // queue carries resource keys, not operations: a worker looks the operation up + // when it picks the key up, which is what makes coalescing work. queue chan string wg sync.WaitGroup @@ -61,25 +45,16 @@ type operationQueue struct { // yet. Empty for a key means everything recorded for it has been uploaded. pending map[string]recordedOperation - // queuedOrUploading marks keys that are already in the queue channel or being - // uploaded right now. Such a key must not be queued again, or two workers could - // upload the same resource at once; recording writes to pending instead, and - // the worker handling the key picks it up when its current upload finishes. - // - // No single worker "owns" a key for the whole time it is marked: a key can be - // handled by one worker, released, and later picked up by another. The mark only - // means "some worker will get to this", which is all record needs to know. + // queuedOrUploading means "some worker will get to this key". Recording such a + // key writes to pending only, so two workers never upload one resource at once. queuedOrUploading map[string]bool err error closed bool } -// newOperationQueue starts the upload workers. It returns nil when uploader is -// nil (recording disabled), and every method is a no-op on a nil queue so callers -// do not have to branch. -// -// ctx is used for the uploads, so it must stay valid until close returns. +// newOperationQueue starts the upload workers, returning nil when uploader is nil +// (recording off; every method is a no-op on a nil queue). ctx must outlive close. func newOperationQueue(ctx context.Context, uploader operationUploader) *operationQueue { if uploader == nil { return nil @@ -100,33 +75,16 @@ func newOperationQueue(ctx context.Context, uploader operationUploader) *operati return q } -// record serializes an operation and hands it to the upload workers. The upload -// itself happens on a worker, so an error returned here is either a failure to -// turn the applied resource into a payload, or an earlier upload's error -// resurfaced (see below). +// record serializes an operation and hands it to the upload workers, so an error +// here means the payload could not be built; upload errors surface at close. // -// Recording a resource that is still waiting replaces the waiting operation -// outright, since the newer one carries the resource's full state. +// An earlier upload failure does not stop this: every applied resource is still +// recorded, best effort, so DMS ends up as close to reality as it can get. func (q *operationQueue) record(ctx context.Context, resourceKey string, action deployplan.ActionType, resourceID string, state any, dependsOn []deployplan.DependsOnEntry) error { if q == nil { return nil } - // Report an earlier upload failure to the apply worker that is about to record - // the next resource, so the deploy stops instead of running to completion and - // only failing at close. That matters because a successfully completed version - // makes DMS the source of truth for resource state (see dstate.readDMSState): - // deploying everything while its records are missing leaves resources the next - // deploy would create a second time. - // - // This refuses new work only. Operations already recorded still upload - close - // drains them - so the records DMS does end up with match the resources that - // were actually applied. Resources already mid-apply also finish, so the deploy - // stops shortly after the first failure rather than exactly at it. - if err := q.firstErr(); err != nil { - return err - } - op, err := newRecordedOperation(action, resourceID, state, dependsOn) if err != nil { return err @@ -136,12 +94,9 @@ func (q *operationQueue) record(ctx context.Context, resourceKey string, action return nil } -// recordFailure records that applying a resource failed, so the deployment -// history explains the failure instead of omitting the resource. -// -// Unlike record, this does not resurface an earlier upload error: the deploy is -// already failing, and returning a different error here would replace the one the -// user needs to see. A failure to upload this record is reported at close. +// recordFailure records that applying a resource failed, so the deployment history +// explains the failure instead of omitting the resource. It returns nothing: the +// deploy is already failing, and a second error would mask the one the user needs. func (q *operationQueue) recordFailure(ctx context.Context, resourceKey string, action deployplan.ActionType, resourceID string, priorState json.RawMessage, cause error) { if q == nil { return @@ -170,9 +125,8 @@ func (q *operationQueue) enqueue(ctx context.Context, resourceKey string, op rec log.Debugf(ctx, "Coalescing queued deployment operation for %s", resourceKey) } - // A worker is already going to handle this key, and it re-reads pending before - // finishing, so it will see the operation written above. Queueing the key again - // would let a second worker upload the same resource concurrently. + // A worker will re-read pending before it finishes, so it picks up the operation + // written above. Queueing again would let a second worker upload the same key. if alreadyHandled { return } @@ -180,14 +134,12 @@ func (q *operationQueue) enqueue(ctx context.Context, resourceKey string, op rec q.queue <- resourceKey } -// close drains the queue and returns the first upload error. All callers of -// record must have returned first: record on a closed queue panics. Calling close -// more than once is safe, so callers can defer it and still check the error at a -// specific point. +// close drains the queue and returns the first upload error. Every record caller +// must have returned first (record on a closed queue panics); calling close twice +// is safe, so it can be deferred and still checked at a specific point. // -// Unlike the other methods this one takes no lock. It runs on one goroutine after -// every apply worker has returned, so nothing else touches the queue by then, and -// the wg.Wait below orders the workers' writes to err before it is read. +// It takes no lock: it runs after every apply worker returned, and wg.Wait orders +// the workers' writes to err before it is read here. func (q *operationQueue) close() error { if q == nil { return nil @@ -206,15 +158,16 @@ func (q *operationQueue) work(ctx context.Context) { defer q.wg.Done() for resourceKey := range q.queue { - // Keep uploading this key until nothing new was recorded for it, rather than - // putting it back on the queue: a worker sending to the channel it consumes - // from can deadlock once the queue is full. + // Drain this key here instead of re-queueing it: a worker sending to the + // channel it consumes from deadlocks once the queue is full. for { op, ok := q.take(resourceKey) if !ok { break } + // Keep going after a failure, so one bad upload does not drop the records + // for every resource behind it. if err := q.uploader.upload(ctx, resourceKey, op); err != nil { q.setErr(fmt.Errorf("recording operation for %s with the deployment metadata service: %w", resourceKey, err)) } @@ -222,12 +175,9 @@ func (q *operationQueue) work(ctx context.Context) { } } -// take claims the operation waiting for resourceKey. It reports false and clears -// the queuedOrUploading mark when nothing is waiting, which is what lets the next -// record queue the key again. -// -// Clearing the mark and observing pending empty happen under one lock, so record -// can never skip queueing a key that no worker is going to look at again. +// take claims the operation waiting for resourceKey, reporting false and clearing +// the queuedOrUploading mark when nothing is left, which lets record queue it again. +// Both happen under one lock, so a key can never be left for no worker to pick up. func (q *operationQueue) take(resourceKey string) (recordedOperation, bool) { q.mu.Lock() defer q.mu.Unlock() @@ -238,11 +188,8 @@ func (q *operationQueue) take(resourceKey string) (recordedOperation, bool) { return recordedOperation{}, false } - // The key stays in queuedOrUploading: the worker keeps coming back here until - // nothing is pending for it, so anything recorded while this operation uploads - // is still picked up. The mark is only cleared above, once there is nothing - // left - which is also what stops a second worker from taking the key and - // uploading the same resource concurrently. + // The mark stays until the branch above clears it, so anything recorded during + // this upload is still picked up and no second worker takes the key meanwhile. delete(q.pending, resourceKey) return op, true } diff --git a/bundle/direct/opqueue_test.go b/bundle/direct/opqueue_test.go index 910b58d470f..7a98dc43774 100644 --- a/bundle/direct/opqueue_test.go +++ b/bundle/direct/opqueue_test.go @@ -198,10 +198,9 @@ func TestOperationQueueReturnsUploadError(t *testing.T) { assert.Contains(t, err.Error(), "resources.jobs.foo") } -func TestOperationQueueRecordFailsAfterUploadError(t *testing.T) { - // An upload failure stops the deploy at the next resource instead of surfacing - // only at close, so the apply workers do not keep creating resources that DMS - // has no record of. +func TestOperationQueueKeepsRecordingAfterUploadError(t *testing.T) { + // A failed upload must not stop the ones behind it: every applied resource is + // recorded best effort, so DMS ends up as close to reality as it can get. uploadErr := errors.New("boom") f := &fakeUploader{err: uploadErr, done: make(chan string, 1)} q := newOperationQueue(t.Context(), f) @@ -211,22 +210,22 @@ func TestOperationQueueRecordFailsAfterUploadError(t *testing.T) { require.NoError(t, q.record(t.Context(), "resources.jobs.foo", deployplan.Create, "id-1", map[string]string{"name": "v1"}, nil)) assert.Equal(t, "resources.jobs.foo", <-f.done) - // The next resource an apply worker tries to record is refused, with the upload - // error that caused it. - err := q.record(t.Context(), "resources.jobs.bar", deployplan.Create, "id-2", map[string]string{"name": "v1"}, nil) - require.Error(t, err) - assert.ErrorIs(t, err, uploadErr) + // The next resource is still accepted, even though the first upload failed. + require.NoError(t, q.record(t.Context(), "resources.jobs.bar", deployplan.Create, "id-2", map[string]string{"name": "v1"}, nil)) - // The refused resource was not queued, and close still reports the failure. + // Both were attempted, and close still reports the failure so the deploy fails. require.ErrorIs(t, q.close(), uploadErr) - assert.Equal(t, []string{`resources.jobs.foo={"state":{"name":"v1"}}`}, f.recorded()) + assert.ElementsMatch(t, []string{ + `resources.jobs.foo={"state":{"name":"v1"}}`, + `resources.jobs.bar={"state":{"name":"v1"}}`, + }, f.recorded()) assert.Empty(t, q.pending) assert.Empty(t, q.queuedOrUploading) } func TestOperationQueueDrainsQueuedOperationsAfterUploadError(t *testing.T) { - // A failure refuses new work but does not discard work already recorded: the - // records DMS ends up with have to match the resources that were applied. + // A failure does not discard work already recorded: the records DMS ends up with + // have to match the resources that were applied. uploadErr := errors.New("boom") f := &fakeUploader{err: uploadErr, block: make(chan struct{}), started: make(chan string, 1)} q := newOperationQueue(t.Context(), f) diff --git a/bundle/phases/deploy.go b/bundle/phases/deploy.go index ba923638ab5..c24efc51f11 100644 --- a/bundle/phases/deploy.go +++ b/bundle/phases/deploy.go @@ -229,17 +229,21 @@ func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHand return } - // Create the version before planning: the plan snapshots the resource config, - // so the deployment and version have to be stamped onto the resources before it - // is computed or the applied resources would not carry them. A cancelled deploy - // therefore leaves a version behind, completed as a failure by the deferred - // CompleteVersion. + // Create the version before planning: the plan snapshots the resource config, so + // the version has to be stamped on before it is computed or the applied resources + // would not carry it. A cancelled deploy therefore leaves a version behind, + // completed as a failure by the deferred CompleteVersion. if err := recorder.CreateVersion(ctx); err != nil { logdiag.LogError(ctx, err) return } if recorder != nil { - bundle.ApplyContext(ctx, b, metadata.AnnotateDeploymentVersion(recorder.DeploymentID(), recorder.Version())) + // The deployment ID is stamped earlier, when the state is opened; only the + // version is new here. A first deploy has no ID until now, so stamp both. + bundle.ApplySeqContext(ctx, b, + metadata.AnnotateDeployment(recorder.DeploymentID()), + metadata.AnnotateDeploymentVersion(recorder.Version()), + ) if logdiag.HasError(ctx) { return } diff --git a/cmd/bundle/utils/process.go b/cmd/bundle/utils/process.go index 1cf67b52132..f7fdf35e255 100644 --- a/cmd/bundle/utils/process.go +++ b/cmd/bundle/utils/process.go @@ -11,6 +11,7 @@ import ( "github.com/databricks/cli/bundle/config/engine" "github.com/databricks/cli/bundle/config/mutator" "github.com/databricks/cli/bundle/config/validate" + "github.com/databricks/cli/bundle/deploy/metadata" "github.com/databricks/cli/bundle/deploy/terraform" "github.com/databricks/cli/bundle/deployplan" "github.com/databricks/cli/bundle/direct" @@ -241,6 +242,15 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle Client: w.BundleDeployments, DeploymentID: deploymentID, } + + // Stamp the deployment onto the resources before anything diffs them. + // The workspace has it, so a plan that left it unset would report drift + // on a resource nobody touched. The version is stamped by the deploy + // phase instead, once it claims one. + bundle.ApplyContext(ctx, b, metadata.AnnotateDeployment(deploymentID)) + if logdiag.HasError(ctx) { + return b, stateDesc, root.ErrAlreadyPrinted + } } if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WithRecovery(true), dstate.WithWrite(false), dmsSource); err != nil { logdiag.LogError(ctx, err) diff --git a/libs/dms/recorder.go b/libs/dms/recorder.go index eda0233931b..001126f4fb6 100644 --- a/libs/dms/recorder.go +++ b/libs/dms/recorder.go @@ -29,19 +29,15 @@ const ( VersionTypeDestroy VersionType = bundledeployments.VersionTypeVersionTypeDestroy ) -// createVersionRequest is the CreateVersion request body. -// -// The CLI builds the body itself instead of using bundledeployments.Version -// because the generated struct has no previous_version_id field, which the -// service requires as its concurrency check. Without it every deploy after the -// first is rejected. +// createVersionRequest is the CreateVersion request body. Hand-written because the +// generated struct has no previous_version_id, which the service needs as its +// concurrency check - without it every deploy after the first is rejected. type createVersionRequest struct { CliVersion string `json:"cli_version"` VersionType VersionType `json:"version_type"` TargetName string `json:"target_name,omitempty"` - // DisplayName names the deployment in the UI. The service copies it onto the - // deployment's workspace node, which is where GetDeployment reads it from, so - // a version that omits it leaves the deployment unnamed. + // DisplayName names the deployment in the UI. The service keeps it on the + // deployment's node, so a version that omits it leaves the deployment unnamed. DisplayName string `json:"display_name,omitempty"` // PreviousVersionId is the deployment's most recent version, unset for a // deployment's first version. @@ -83,12 +79,10 @@ func (a *apiVersionCreator) CreateVersion(ctx context.Context, deploymentID, ver return &version, nil } -// Recorder records a single deploy/destroy as a version with DMS. -// -// The server assigns the deployment ID on the first deploy, i.e. when the ID -// resolved from the workspace is empty (see ResolveDeploymentID). Later deploys -// resolve the same ID and reuse the record; a destroy deletes the record and its -// node, so the next deploy starts over from empty. +// Recorder records a single deploy/destroy as a version with DMS. The server +// assigns the deployment ID on the first deploy and later deploys reuse it; a +// destroy deletes the record, so the next deploy starts over (see +// ResolveDeploymentID). type Recorder struct { svc bundledeployments.BundleDeploymentsInterface versions versionCreator @@ -112,9 +106,8 @@ type RecorderOptions struct { Service bundledeployments.BundleDeploymentsInterface // Versions handles CreateVersion; see versionCreator. Versions versionCreator - // DeploymentID is the ID resolved from the deployment's workspace node, or - // empty if this bundle has not recorded a deployment yet (the server assigns - // one during CreateVersion). + // DeploymentID is resolved from the deployment's workspace node, empty until the + // first recorded deploy (CreateVersion assigns one then). DeploymentID string // StatePath is the bundle's remote state directory, under which DMS registers // the deployment node. @@ -124,9 +117,9 @@ type RecorderOptions struct { Metadata Metadata } -// Metadata is what a version records about the bundle it deployed, the source it -// came from, and where it landed. The service denormalizes these onto the -// deployment, so they describe the deployment as of its most recent version. +// Metadata is what a version records about the bundle, its source and where it +// landed. The service copies these onto the deployment, so they describe it as of +// its most recent version. type Metadata struct { // DisplayName is the bundle's name, which the deployment is listed under. DisplayName string @@ -191,11 +184,9 @@ func (r *Recorder) CreateVersion(ctx context.Context) error { return nil } -// CompleteVersion finalizes the version created by CreateVersion. A nil -// Recorder, or one whose CreateVersion never ran or failed, is a no-op: there is -// no version on the server to complete. Callers defer it unconditionally, so this -// is the check that keeps a cancelled or failed deploy from completing a version -// that was never created. +// CompleteVersion finalizes the version created by CreateVersion. It is a no-op +// when CreateVersion never ran, which is what lets callers defer it and still not +// complete a version a cancelled deploy never created. func (r *Recorder) CompleteVersion(ctx context.Context, success bool) error { if r == nil || r.versionNum == 0 || r.completed { return nil @@ -235,10 +226,9 @@ func (r *Recorder) CompleteVersion(ctx context.Context, success bool) error { return nil } -// createDeploymentVersion ensures the deployment record exists, then creates a -// new version under it. With no deployment ID it creates the deployment and lets -// the server assign the ID; otherwise it reads the existing deployment to -// compute the next version number. +// createDeploymentVersion ensures the deployment record exists, then creates a new +// version under it: with no ID it creates the deployment, otherwise it reads the +// existing one for the next version number. func (r *Recorder) createDeploymentVersion(ctx context.Context) (versionID string, err error) { // The version this one supersedes, sent as the concurrency check. Empty for a // deployment's first version. @@ -269,11 +259,8 @@ func (r *Recorder) createDeploymentVersion(ctx context.Context) (versionID strin } } else { // First deploy: create the deployment so the server assigns an ID. - // - // initial_parent_path is required. The service creates the deployment node - // under it, and that node's ID is the deployment ID ResolveDeploymentID reads - // back later. The folder already exists by now: the deployment lock lives in - // the same directory. + // initial_parent_path is required - the node the service creates under it is + // what ResolveDeploymentID reads back later. dep, createErr := r.svc.CreateDeployment(ctx, bundledeployments.CreateDeploymentRequest{ Deployment: bundledeployments.Deployment{ InitialParentPath: r.statePath, From ff96ef01de7c012c0bf177a90dc20032d49cf41e Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Fri, 7 Aug 2026 01:24:24 +0000 Subject: [PATCH 047/125] bundle: decouple InitDeploymentHistory from InitIDs The two are unrelated: InitIDs loads resource IDs out of the state, while the deployment history is read from the service. They were coupled only because the mutator sat inside the block gated on InitIDs, so reaching it meant forcing that option on - which made 'bundle summary' look like it needed resource IDs to report a deployment ID. It runs in its own block now, and implies ReadState instead. bundle summary asks for both because it happens to want both. Co-authored-by: Isaac --- cmd/bundle/utils/process.go | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/cmd/bundle/utils/process.go b/cmd/bundle/utils/process.go index f7fdf35e255..3b90aef5c6a 100644 --- a/cmd/bundle/utils/process.go +++ b/cmd/bundle/utils/process.go @@ -58,9 +58,8 @@ type ProcessOptions struct { InitIDs bool // If true, calls InitializeDeploymentHistory() to look up the bundle's recorded - // deployment. Separate from InitIDs because it costs its own API calls and only - // 'bundle summary' reports the result. - // Implies InitIDs + // deployment. Independent of InitIDs, and costs its own API calls. + // Implies ReadState InitDeploymentHistory bool // if true, pass ErrorOnEmptyState to statemgmt.Load @@ -96,11 +95,6 @@ func ProcessBundle(cmd *cobra.Command, opts ProcessOptions) (*bundle.Bundle, err func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle, stateDesc *statemgmt.StateDesc, retErr error) { var err error - // The deployment history is looked up alongside the resource IDs, so asking for - // it implies them. Normalized here so the options below only test InitIDs. - if opts.InitDeploymentHistory { - opts.InitIDs = true - } ctx := cmd.Context() if opts.SkipInitContext { if !logdiag.IsSetup(ctx) { @@ -192,7 +186,7 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle return b, nil, err } - shouldReadState := opts.ReadState || opts.AlwaysPull || opts.InitIDs || opts.ErrorOnEmptyState || opts.PreDeployChecks || opts.Deploy || opts.ReadPlanPath != "" + shouldReadState := opts.ReadState || opts.AlwaysPull || opts.InitIDs || opts.InitDeploymentHistory || opts.ErrorOnEmptyState || opts.PreDeployChecks || opts.Deploy || opts.ReadPlanPath != "" if shouldReadState { // PullResourcesState depends on stateFiler which needs b.Config.Workspace.StatePath which is set in phases.Initialize @@ -282,15 +276,20 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle if opts.InitIDs { mutators = append(mutators, mutator.InitializeURLs()) } - // Same for InitializeDeploymentHistory, which only 'bundle summary' reports. - if opts.InitDeploymentHistory { - mutators = append(mutators, mutator.InitializeDeploymentHistory()) - } bundle.ApplySeqContext(ctx, b, mutators...) if logdiag.HasError(ctx) { return b, stateDesc, root.ErrAlreadyPrinted } } + + // Independent of the resource IDs above: this reads the deployment record, not + // the state. It makes its own API calls, so only 'bundle summary' asks for it. + if opts.InitDeploymentHistory { + bundle.ApplyContext(ctx, b, mutator.InitializeDeploymentHistory()) + if logdiag.HasError(ctx) { + return b, stateDesc, root.ErrAlreadyPrinted + } + } } var plan *deployplan.Plan From 89369d5966563be4725dbc6b6cc1b6b8062ce558 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Fri, 7 Aug 2026 01:36:40 +0000 Subject: [PATCH 048/125] bundle: fold the display-name assertions into the first-deploy test Both tests set up the same recorder and called CreateVersion; the second only added two assertions about the request body, so they move into the first. Co-authored-by: Isaac --- libs/dms/recorder_test.go | 22 +++++++--------------- 1 file changed, 7 insertions(+), 15 deletions(-) diff --git a/libs/dms/recorder_test.go b/libs/dms/recorder_test.go index 56d5d41631d..51ce63e5971 100644 --- a/libs/dms/recorder_test.go +++ b/libs/dms/recorder_test.go @@ -105,6 +105,13 @@ func TestRecorderFirstDeployCreatesDeploymentWithServerAssignedID(t *testing.T) assert.Equal(t, "server-generated-id", f.versions[0].deploymentID) assert.Equal(t, int64(1), r.Version()) + // The service copies display_name onto the deployment's workspace node, which is + // where GetDeployment reads it from; a version that omits it leaves the deployment + // unnamed in the UI. + assert.Equal(t, testDisplayName, f.versions[0].body.DisplayName) + // A first version supersedes nothing, so previous_version_id is unset. + assert.Empty(t, f.versions[0].body.PreviousVersionId) + require.NoError(t, r.CompleteVersion(t.Context(), true)) require.Len(t, f.completed, 1) assert.Equal(t, bundledeployments.VersionCompleteVersionCompleteSuccess, f.completed[0].CompletionReason) @@ -132,21 +139,6 @@ func TestRecorderSubsequentDeployReusesDeploymentAndIncrementsVersion(t *testing assert.Equal(t, "4", f.versions[0].body.PreviousVersionId) } -func TestRecorderSendsDisplayNameAndNoPreviousVersionOnFirstDeploy(t *testing.T) { - f := &fakeDMS{assignedID: "server-generated-id"} - r := NewRecorder(RecorderOptions{Service: f, Versions: fakeVersions{requests: &f.versions}, StatePath: testStatePath, Metadata: Metadata{TargetName: "dev", DisplayName: testDisplayName}, VersionType: VersionTypeDeploy}) - - require.NoError(t, r.CreateVersion(t.Context())) - - require.Len(t, f.versions, 1) - // The service copies display_name onto the deployment's workspace node, which - // is where GetDeployment reads it from; a version that omits it leaves the - // deployment unnamed in the UI. - assert.Equal(t, testDisplayName, f.versions[0].body.DisplayName) - // A first version supersedes nothing, so previous_version_id is unset. - assert.Empty(t, f.versions[0].body.PreviousVersionId) -} - func TestRecorderGetDeploymentErrorFailsDeploy(t *testing.T) { f := &fakeDMS{ getDeployment: func(id string) (*bundledeployments.Deployment, error) { From 02c35fd1dc67595af1ace3237cde2493088f743c Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Fri, 7 Aug 2026 10:56:05 +0000 Subject: [PATCH 049/125] bundle: record DMS operations from the state writes Operations were recorded from bundle_apply, once per resource after Deploy returned, so a resource that writes state more than once in a deploy reported only the end result. Recreate is the case that matters: it drops the state entry, then saves the new resource, and the intermediate step was invisible. SaveState and DeleteState now record, so what DMS holds follows the WAL. Both take the action to report and a context; the sink is installed on the state DB during apply and is nil everywhere else (migration and bind/unbind write state without a DMS version, so they record nothing). Three things fell out of it: The upload queue no longer coalesces. It kept only the newest operation per resource, which is exactly the intermediate write we want to keep, so pending holds a per- resource FIFO and every write is uploaded oldest-first. The operations API is called directly instead of through the generated client. The service sends sequence_id as a JSON string (proto3 encodes 64-bit ints that way) while the SDK types it int64, so reading a CreateOperation response fails with "invalid character '1' after top-level value" - the write succeeds, only the parse does not. The testserver now emits the same string form, so tests exercise the real wire format. A separate fix for the spec/SDK is in flight. A recreate's intermediate delete is not recorded. The service keeps one operation per resource per version, with action_type fixed at creation, and it rejects a succeeded recreate that carries no state ("it leaves a resource that exists"). So the drop cannot be its own event; the save that follows reports the recreate, and the failure path reports it if that save never happens. Verified on dogfood across create, update, recreate and destroy: no recording errors, and the recreate records the new resource id and name. Co-authored-by: Isaac --- .../bundle/dms/partial-update/databricks.yml | 12 ++ .../bundle/dms/partial-update/out.test.toml | 3 + .../bundle/dms/partial-update/output.txt | 167 ++++++++++++++++++ acceptance/bundle/dms/partial-update/script | 15 ++ bundle/direct/apply.go | 18 +- bundle/direct/bind.go | 8 +- bundle/direct/bundle_apply.go | 31 ++-- bundle/direct/dstate/dms.go | 10 ++ bundle/direct/dstate/state.go | 62 ++++++- bundle/direct/dstate/state_test.go | 75 +++++++- bundle/direct/opclient.go | 80 +++++++++ bundle/direct/opqueue.go | 63 +++---- bundle/direct/opqueue_test.go | 105 ++++++----- bundle/direct/oprecorder.go | 111 ++++++++---- bundle/direct/oprecorder_test.go | 120 ++++++++----- bundle/migrate/build_state.go | 4 +- bundle/phases/deploy.go | 13 +- bundle/phases/destroy.go | 9 +- bundle/phases/dms.go | 19 ++ libs/testserver/bundle.go | 123 ++++++++++++- libs/testserver/handlers.go | 3 + 21 files changed, 834 insertions(+), 217 deletions(-) create mode 100644 acceptance/bundle/dms/partial-update/databricks.yml create mode 100644 acceptance/bundle/dms/partial-update/out.test.toml create mode 100644 acceptance/bundle/dms/partial-update/output.txt create mode 100644 acceptance/bundle/dms/partial-update/script create mode 100644 bundle/direct/opclient.go diff --git a/acceptance/bundle/dms/partial-update/databricks.yml b/acceptance/bundle/dms/partial-update/databricks.yml new file mode 100644 index 00000000000..fc61c680fb1 --- /dev/null +++ b/acceptance/bundle/dms/partial-update/databricks.yml @@ -0,0 +1,12 @@ +bundle: + name: dms-partial-update + +experimental: + record_deployment_history: true + +resources: + schemas: + foo: + name: dms_partial_update_schema + catalog_name: main + comment: v1 diff --git a/acceptance/bundle/dms/partial-update/out.test.toml b/acceptance/bundle/dms/partial-update/out.test.toml new file mode 100644 index 00000000000..e90b6d5d1ba --- /dev/null +++ b/acceptance/bundle/dms/partial-update/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/dms/partial-update/output.txt b/acceptance/bundle/dms/partial-update/output.txt new file mode 100644 index 00000000000..8c36883a9db --- /dev/null +++ b/acceptance/bundle/dms/partial-update/output.txt @@ -0,0 +1,167 @@ + +=== Deploy: the state write records the resource +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-partial-update/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! +Deployment history: [DATABRICKS_URL]/deployments/[NUMID]?version=1 + +>>> print_requests.py //api/2.0/bundle +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments", + "body": { + "initial_parent_path": "/Workspace/Users/[USERNAME]/.bundle/dms-partial-update/default/state", + "target_name": "default" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions", + "q": { + "version_id": "1" + }, + "body": { + "cli_version": "[CLI_VERSION]", + "version_type": "VERSION_TYPE_DEPLOY", + "target_name": "default", + "display_name": "dms-partial-update", + "workspace_info": { + "file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-partial-update/default/files", + "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-partial-update/default" + } + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations", + "q": { + "resource_key": "schemas.foo" + }, + "body": { + "action_type": "OPERATION_ACTION_TYPE_CREATE", + "resource_id": "main.dms_partial_update_schema", + "resource_key": "schemas.foo", + "state": "{\"state\":{\"catalog_name\":\"main\",\"comment\":\"v1\",\"name\":\"dms_partial_update_schema\"}}", + "status": "OPERATION_STATUS_SUCCEEDED" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/complete", + "body": { + "completion_reason": "VERSION_COMPLETE_SUCCESS" + } +} + +=== Recreate writes state twice - the entry is dropped, then the new resource is saved +>>> update_file.py databricks.yml catalog_name: main catalog_name: other + +>>> [CLI] bundle deploy --auto-approve +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-partial-update/default/files... + +This action will result in the deletion or recreation of the following UC schemas. Any underlying data may be lost: + recreate resources.schemas.foo +Deploying resources... +Updating deployment state... +Deployment complete! +Deployment history: [DATABRICKS_URL]/deployments/[NUMID]?version=2 + +>>> print_requests.py //api/2.0/bundle +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions", + "q": { + "version_id": "2" + }, + "body": { + "cli_version": "[CLI_VERSION]", + "version_type": "VERSION_TYPE_DEPLOY", + "target_name": "default", + "display_name": "dms-partial-update", + "previous_version_id": "1", + "workspace_info": { + "file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-partial-update/default/files", + "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-partial-update/default" + } + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations", + "q": { + "resource_key": "schemas.foo" + }, + "body": { + "action_type": "OPERATION_ACTION_TYPE_CREATE", + "resource_id": "other.dms_partial_update_schema", + "resource_key": "schemas.foo", + "state": "{\"state\":{\"catalog_name\":\"other\",\"comment\":\"v1\",\"name\":\"dms_partial_update_schema\"}}", + "status": "OPERATION_STATUS_SUCCEEDED" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/complete", + "body": { + "completion_reason": "VERSION_COMPLETE_SUCCESS" + } +} + +=== Destroy: the delete is recorded with the id and no state +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.schemas.foo + +This action will result in the deletion of the following UC schemas. Any underlying data may be lost: + delete resources.schemas.foo + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/dms-partial-update/default + +Deleting files... +Destroy complete! + +>>> print_requests.py //api/2.0/bundle +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions", + "q": { + "version_id": "3" + }, + "body": { + "cli_version": "[CLI_VERSION]", + "version_type": "VERSION_TYPE_DESTROY", + "target_name": "default", + "display_name": "dms-partial-update", + "previous_version_id": "2", + "workspace_info": { + "file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-partial-update/default/files", + "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-partial-update/default" + } + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/3/operations", + "q": { + "resource_key": "schemas.foo" + }, + "body": { + "action_type": "OPERATION_ACTION_TYPE_DELETE", + "resource_id": "other.dms_partial_update_schema", + "resource_key": "schemas.foo", + "status": "OPERATION_STATUS_SUCCEEDED" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/3/complete", + "body": { + "completion_reason": "VERSION_COMPLETE_SUCCESS" + } +} +{ + "method": "DELETE", + "path": "/api/2.0/bundle/deployments/[NUMID]" +} diff --git a/acceptance/bundle/dms/partial-update/script b/acceptance/bundle/dms/partial-update/script new file mode 100644 index 00000000000..9310ded83aa --- /dev/null +++ b/acceptance/bundle/dms/partial-update/script @@ -0,0 +1,15 @@ +title "Deploy: the state write records the resource" +trace $CLI bundle deploy +trace print_requests.py //api/2.0/bundle + +title "Recreate writes state twice - the entry is dropped, then the new resource is saved" +# Only the save is recorded. The service keeps one operation per resource per version +# and rejects a succeeded recreate that carries no state, so the intermediate drop +# cannot be its own event; the save that follows reports the recreate instead. +trace update_file.py databricks.yml "catalog_name: main" "catalog_name: other" +trace $CLI bundle deploy --auto-approve +trace print_requests.py //api/2.0/bundle + +title "Destroy: the delete is recorded with the id and no state" +trace $CLI bundle destroy --auto-approve +trace print_requests.py //api/2.0/bundle diff --git a/bundle/direct/apply.go b/bundle/direct/apply.go index cbb0a2d45ff..e8f95daa48a 100644 --- a/bundle/direct/apply.go +++ b/bundle/direct/apply.go @@ -75,7 +75,7 @@ func (d *DeploymentUnit) Create(ctx context.Context, db *dstate.DeploymentState, return err } - err = db.SaveState(d.ResourceKey, newID, newState, d.DependsOn) + err = db.SaveState(ctx, d.ResourceKey, newID, newState, d.DependsOn, deployplan.Create) if err != nil { return fmt.Errorf("saving state after creating id=%s: %w", newID, err) } @@ -116,7 +116,11 @@ func (d *DeploymentUnit) Recreate(ctx context.Context, db *dstate.DeploymentStat // Drop the state entry so a subsequent failure of Create or WaitAfterDelete // leaves no malformed (empty-ID) entry behind. The next plan will see "no // state" and retry as Create. - err = db.DeleteState(d.ResourceKey) + // + // Recorded as a recreate, not a delete: if the create below fails, this is the + // operation DMS is left with, and it says the resource is mid-recreate rather + // than deliberately removed. + err = db.DeleteState(ctx, d.ResourceKey, deployplan.Recreate) if err != nil { return fmt.Errorf("deleting state: %w", err) } @@ -158,12 +162,12 @@ func (d *DeploymentUnit) Update(ctx context.Context, db *dstate.DeploymentState, // The update emptied the resource out (e.g. all grants revoked). Keeping an entry // would report the node as tracked-and-unchanged forever, while a fresh deploy of // the same config plans no node at all; drop it so the two agree. - err = db.DeleteState(d.ResourceKey) + err = db.DeleteState(ctx, d.ResourceKey, deployplan.Update) if err != nil { return fmt.Errorf("deleting state id=%s: %w", id, err) } } else { - err = db.SaveState(d.ResourceKey, id, newState, d.DependsOn) + err = db.SaveState(ctx, d.ResourceKey, id, newState, d.DependsOn, deployplan.Update) if err != nil { return fmt.Errorf("saving state id=%s: %w", id, err) } @@ -208,7 +212,7 @@ func (d *DeploymentUnit) UpdateWithID(ctx context.Context, db *dstate.Deployment return err } - err = db.SaveState(d.ResourceKey, newID, newState, d.DependsOn) + err = db.SaveState(ctx, d.ResourceKey, newID, newState, d.DependsOn, deployplan.UpdateWithID) if err != nil { return fmt.Errorf("saving state id=%s: %w", oldID, err) } @@ -250,7 +254,7 @@ func (d *DeploymentUnit) Delete(ctx context.Context, db *dstate.DeploymentState, } } - err = db.DeleteState(d.ResourceKey) + err = db.DeleteState(ctx, d.ResourceKey, deployplan.Delete) if err != nil { return fmt.Errorf("deleting state id=%s: %w", oldID, err) } @@ -291,7 +295,7 @@ func (d *DeploymentUnit) Resize(ctx context.Context, db *dstate.DeploymentState, return fmt.Errorf("resizing id=%s: %w", id, err) } - err = db.SaveState(d.ResourceKey, id, newState, d.DependsOn) + err = db.SaveState(ctx, d.ResourceKey, id, newState, d.DependsOn, deployplan.Resize) if err != nil { return fmt.Errorf("saving state id=%s: %w", id, err) } diff --git a/bundle/direct/bind.go b/bundle/direct/bind.go index ec910b2734e..4de9c8d736a 100644 --- a/bundle/direct/bind.go +++ b/bundle/direct/bind.go @@ -93,7 +93,7 @@ func (b *DeploymentBundle) Bind(ctx context.Context, client *databricks.Workspac } // Save state with ID and empty state (like migrate does) - err = b.StateDB.SaveState(resourceKey, resourceID, struct{}{}, nil) + err = b.StateDB.SaveState(ctx, resourceKey, resourceID, struct{}{}, nil, deployplan.Create) if err != nil { os.Remove(tmpStatePath) return nil, err @@ -151,7 +151,7 @@ func (b *DeploymentBundle) Bind(ctx context.Context, client *databricks.Workspac return nil, err } - err = b.StateDB.SaveState(resourceKey, resourceID, sv.Value, dependsOn) + err = b.StateDB.SaveState(ctx, resourceKey, resourceID, sv.Value, dependsOn, deployplan.Create) if err != nil { os.Remove(tmpStatePath) return nil, err @@ -221,7 +221,7 @@ func (b *DeploymentBundle) Unbind(ctx context.Context, statePath, resourceKey st } // Delete the main resource - err = b.StateDB.DeleteState(resourceKey) + err = b.StateDB.DeleteState(ctx, resourceKey, deployplan.Delete) if err != nil { return err } @@ -235,7 +235,7 @@ func (b *DeploymentBundle) Unbind(ctx context.Context, statePath, resourceKey st for key := range b.StateDB.Data.State { if key == permissionsKey || key == grantsKey || strings.HasPrefix(key, resourceKey+".") { - err = b.StateDB.DeleteState(key) + err = b.StateDB.DeleteState(ctx, key, deployplan.Delete) if err != nil { return err } diff --git a/bundle/direct/bundle_apply.go b/bundle/direct/bundle_apply.go index 9fa882867fc..76710211c05 100644 --- a/bundle/direct/bundle_apply.go +++ b/bundle/direct/bundle_apply.go @@ -37,7 +37,15 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa // Operations are recorded with DMS from background workers so a resource's // deploy is not held up by the CreateOperation round trip. The queue is // drained below, once every apply worker has finished recording. + // + // The state DB records through it, so every state write becomes an operation and + // DMS mirrors the WAL. opQueue := newOperationQueue(ctx, b.OpRec) + if opQueue != nil { + // Assigned only when non-nil: a nil *operationQueue in an interface is not a + // nil interface, so the state DB's nil check would not see it. + b.StateDB.SetOperationSink(opQueue) + } g.Run(defaultParallelism, func(resourceKey string, failedDependency *string) bool { entry, err := plan.WriteLockEntry(resourceKey) @@ -88,13 +96,13 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa } if action == deployplan.Delete { - // Read the ID before the delete removes it from state; DMS requires it to - // identify which resource the delete operation refers to. + // Read the ID before the delete removes it from state; recording a failure + // below needs it to say which resource the operation refers to. deletedID := b.StateDB.GetResourceID(resourceKey) if entry.Gone { // Planning confirmed the resource is already deleted remotely; only // remove it from the state, without calling the delete API. - err = b.StateDB.DeleteState(resourceKey) + err = b.StateDB.DeleteState(ctx, resourceKey, action) } else { err = d.Destroy(ctx, &b.StateDB) } @@ -104,11 +112,6 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa logdiag.LogError(ctx, fmt.Errorf("%s: %w", errorPrefix, err)) return false } - // Record the delete with DMS. State is nil: the resource is gone. - if err := opQueue.record(ctx, resourceKey, action, deletedID, nil, nil); err != nil { - logdiag.LogError(ctx, fmt.Errorf("%s: %w", errorPrefix, err)) - return false - } return true } @@ -132,6 +135,9 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa } // TODO: redo calcDiff to downgrade planned action if possible (?) + // + // Success is recorded by the state writes inside Deploy, so a resource that + // writes state more than once (a recreate) reports each step. err = d.Deploy(ctx, &b.StateDB, sv.Value, action, entry) if err != nil { // Both are empty for a create that never got an ID, which is what the @@ -141,15 +147,6 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa logdiag.LogError(ctx, fmt.Errorf("%s: %w", errorPrefix, err)) return false } - - // Record the operation with DMS. The resource ID and applied config - // (sv.Value) come from the write just performed; GetResourceID reads - // the ID assigned by Deploy. depends_on is recorded alongside the config - // because it cannot be recomputed from it (see dstate.RecordedState). - if err := opQueue.record(ctx, resourceKey, action, b.StateDB.GetResourceID(resourceKey), sv.Value, d.DependsOn); err != nil { - logdiag.LogError(ctx, fmt.Errorf("%s: %w", errorPrefix, err)) - return false - } } // TODO: Note, we only really need remote state if there are remote references. diff --git a/bundle/direct/dstate/dms.go b/bundle/direct/dstate/dms.go index 4ec68dea82d..56647fe471c 100644 --- a/bundle/direct/dstate/dms.go +++ b/bundle/direct/dstate/dms.go @@ -19,6 +19,16 @@ type RecordedState struct { DependsOn []deployplan.DependsOnEntry `json:"depends_on,omitempty"` } +// OperationSink records one resource operation with the deployment metadata service. +// SaveState and DeleteState call it for every state write, so what DMS holds mirrors +// the WAL - including the intermediate writes of a recreate. +// +// It does not return an error: the upload happens on a background worker, and the +// deploy learns about a failure when the queue is drained. +type OperationSink interface { + RecordOperation(ctx context.Context, resourceKey string, action deployplan.ActionType, resourceID string, state json.RawMessage) +} + // readDMSState replaces the file-derived resource state with the state recorded in // DMS. Recording is only enabled for net-new deployments, so once a deployment // exists DMS owns its resource set outright - an empty set means a successful deploy diff --git a/bundle/direct/dstate/state.go b/bundle/direct/dstate/state.go index 06d0bb3dd25..ff1d437300b 100644 --- a/bundle/direct/dstate/state.go +++ b/bundle/direct/dstate/state.go @@ -71,6 +71,18 @@ type DeploymentState struct { // Maps resource key to ID. Unlike Data.State, this is up to date during writes (deploys). stateIDs map[string]string + + // sink records each state write with DMS. Nil unless the bundle records + // deployment history, in which case SetOperationSink installs it. + sink OperationSink +} + +// SetOperationSink makes every subsequent state write also record an operation with +// DMS. It is set after the version is created, which is why it is not an Open option. +func (db *DeploymentState) SetOperationSink(sink OperationSink) { + db.mu.Lock() + defer db.mu.Unlock() + db.sink = sink } type Header struct { @@ -119,7 +131,10 @@ func NewDatabase(lineage string, serial int) Database { } } -func (db *DeploymentState) SaveState(key, newID string, state any, dependsOn []deployplan.DependsOnEntry) error { +// SaveState records the resource's state after action was applied to it. action is +// what the deployment metadata service reports for the write; it is ignored when the +// bundle does not record deployment history. +func (db *DeploymentState) SaveState(ctx context.Context, key, newID string, state any, dependsOn []deployplan.DependsOnEntry, action deployplan.ActionType) error { db.AssertOpenedForWrite() db.mu.Lock() defer db.mu.Unlock() @@ -140,13 +155,27 @@ func (db *DeploymentState) SaveState(key, newID string, state any, dependsOn []d } err = appendJSONLine(db.walFile, WALEntry{Key: key, Value: &entry}) - if err == nil { - db.stateIDs[key] = newID + if err != nil { + return err } - return err + db.stateIDs[key] = newID + + // Recorded after the WAL write, so DMS never reports a state the deploy failed to + // persist locally. + if db.sink != nil { + recorded, err := json.Marshal(RecordedState{State: entry.State, DependsOn: dependsOn}) + if err != nil { + return err + } + db.sink.RecordOperation(ctx, key, action, newID, recorded) + } + + return nil } -func (db *DeploymentState) DeleteState(key string) error { +// DeleteState drops the resource's state entry. action distinguishes a real delete +// from the intermediate drop a recreate performs, both of which are recorded. +func (db *DeploymentState) DeleteState(ctx context.Context, key string, action deployplan.ActionType) error { db.AssertOpenedForWrite() db.mu.Lock() defer db.mu.Unlock() @@ -155,11 +184,28 @@ func (db *DeploymentState) DeleteState(key string) error { return nil } + // Read before the delete: DMS needs the id to say which resource went away. + deletedID := db.stateIDs[key] + err := appendJSONLine(db.walFile, WALEntry{Key: key}) - if err == nil { - delete(db.stateIDs, key) + if err != nil { + return err } - return err + delete(db.stateIDs, key) + + // State is nil: the resource no longer exists. + // + // A recreate is the exception. It drops the entry and then saves the new + // resource, but the service keeps one operation per resource per version whose + // action_type is fixed at creation, and it rejects a succeeded recreate that + // carries no state ("it leaves a resource that exists"). So the intermediate drop + // cannot be recorded as its own event; the save that follows reports the recreate, + // and if that save never happens the failure path reports it instead. + if db.sink != nil && action != deployplan.Recreate { + db.sink.RecordOperation(ctx, key, action, deletedID, nil) + } + + return nil } func (db *DeploymentState) GetResourceEntry(key string) (ResourceEntry, bool) { diff --git a/bundle/direct/dstate/state_test.go b/bundle/direct/dstate/state_test.go index a9c90530514..9f0374c0ec4 100644 --- a/bundle/direct/dstate/state_test.go +++ b/bundle/direct/dstate/state_test.go @@ -1,11 +1,14 @@ package dstate import ( + "context" "encoding/json" + "fmt" "os" "path/filepath" "testing" + "github.com/databricks/cli/bundle/deployplan" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -16,13 +19,75 @@ func mustFinalize(t *testing.T, db *DeploymentState) { require.NoError(t, err) } +// fakeSink captures what the state writes report to DMS. +type fakeSink struct { + ops []string +} + +func (f *fakeSink) RecordOperation(ctx context.Context, resourceKey string, action deployplan.ActionType, resourceID string, state json.RawMessage) { + f.ops = append(f.ops, fmt.Sprintf("%s %s id=%s state=%s", action, resourceKey, resourceID, string(state))) +} + +func TestStateWritesRecordOperations(t *testing.T) { + path := filepath.Join(t.TempDir(), "state.json") + sink := &fakeSink{} + + var db DeploymentState + require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil)) + db.SetOperationSink(sink) + + // A recreate: the old entry is dropped, then the new resource is saved. + require.NoError(t, db.SaveState(t.Context(), "jobs.my_job", "123", map[string]string{"key": "old"}, nil, deployplan.Create)) + require.NoError(t, db.DeleteState(t.Context(), "jobs.my_job", deployplan.Recreate)) + require.NoError(t, db.SaveState(t.Context(), "jobs.my_job", "456", map[string]string{"key": "new"}, nil, deployplan.Recreate)) + mustFinalize(t, &db) + + // The recreate's intermediate drop is not reported: the service keeps one + // operation per resource per version and rejects a succeeded recreate carrying no + // state, so the save that follows is what reports it. + assert.Equal(t, []string{ + `create jobs.my_job id=123 state={"state":{"key":"old"}}`, + `recreate jobs.my_job id=456 state={"state":{"key":"new"}}`, + }, sink.ops) +} + +func TestDeleteStateRecordsRealDelete(t *testing.T) { + path := filepath.Join(t.TempDir(), "state.json") + sink := &fakeSink{} + + var db DeploymentState + require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil)) + db.SetOperationSink(sink) + + require.NoError(t, db.SaveState(t.Context(), "jobs.my_job", "123", map[string]string{}, nil, deployplan.Create)) + require.NoError(t, db.DeleteState(t.Context(), "jobs.my_job", deployplan.Delete)) + mustFinalize(t, &db) + + // A real delete reports the id it had and no state: the resource is gone. + assert.Equal(t, []string{ + `create jobs.my_job id=123 state={"state":{}}`, + `delete jobs.my_job id=123 state=`, + }, sink.ops) +} + +func TestStateWritesRecordNothingWithoutSink(t *testing.T) { + path := filepath.Join(t.TempDir(), "state.json") + + // No sink: recording is off, and the writes still succeed. + var db DeploymentState + require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil)) + require.NoError(t, db.SaveState(t.Context(), "jobs.my_job", "123", map[string]string{}, nil, deployplan.Create)) + require.NoError(t, db.DeleteState(t.Context(), "jobs.my_job", deployplan.Delete)) + mustFinalize(t, &db) +} + func TestOpenSaveFinalizeRoundTrip(t *testing.T) { path := filepath.Join(t.TempDir(), "state.json") var db DeploymentState require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil)) - require.NoError(t, db.SaveState("jobs.my_job", "123", map[string]string{"key": "val"}, nil)) + require.NoError(t, db.SaveState(t.Context(), "jobs.my_job", "123", map[string]string{"key": "val"}, nil, deployplan.Create)) mustFinalize(t, &db) // Re-open and verify persisted data. @@ -108,7 +173,7 @@ func TestHeaderOnlyWALRecoveryDoesNotAdvanceSerial(t *testing.T) { // Commit serial 1 with one resource. var db DeploymentState require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil)) - require.NoError(t, db.SaveState("jobs.my_job", "123", map[string]string{}, nil)) + require.NoError(t, db.SaveState(t.Context(), "jobs.my_job", "123", map[string]string{}, nil, deployplan.Create)) mustFinalize(t, &db) var committed DeploymentState @@ -172,12 +237,12 @@ func TestDeleteState(t *testing.T) { var db DeploymentState require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil)) - require.NoError(t, db.SaveState("jobs.my_job", "123", map[string]string{}, nil)) + require.NoError(t, db.SaveState(t.Context(), "jobs.my_job", "123", map[string]string{}, nil, deployplan.Create)) mustFinalize(t, &db) var db2 DeploymentState require.NoError(t, db2.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil)) - require.NoError(t, db2.DeleteState("jobs.my_job")) + require.NoError(t, db2.DeleteState(t.Context(), "jobs.my_job", deployplan.Delete)) mustFinalize(t, &db2) var db3 DeploymentState @@ -205,7 +270,7 @@ func TestGetOrInitLineageReadableBeforeWriteAndPersisted(t *testing.T) { // Upgrading to write reuses the same lineage (it goes into the WAL header), // and a write makes it durable. require.NoError(t, db.UpgradeToWrite()) - require.NoError(t, db.SaveState("jobs.my_job", "123", map[string]string{}, nil)) + require.NoError(t, db.SaveState(t.Context(), "jobs.my_job", "123", map[string]string{}, nil, deployplan.Create)) mustFinalize(t, &db) // Re-open: the persisted lineage matches the one read before the write. diff --git a/bundle/direct/opclient.go b/bundle/direct/opclient.go new file mode 100644 index 00000000000..009089cdb85 --- /dev/null +++ b/bundle/direct/opclient.go @@ -0,0 +1,80 @@ +package direct + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "strings" + + "github.com/databricks/cli/libs/auth" + "github.com/databricks/databricks-sdk-go/client" + "github.com/databricks/databricks-sdk-go/service/bundledeployments" +) + +// The CLI calls the operations API directly rather than through the generated +// client because the SDK cannot read the response: it types sequence_id as an +// int64, while the service sends it as a JSON string (proto3 encodes 64-bit ints +// that way), so unmarshalling a CreateOperation response fails with +// "invalid character '1' after top-level value". The write itself succeeds - the +// status is 200 - so only the response parse is affected. + +// operationResponse is the part of an operation response the CLI reads back. +type operationResponse struct { + // SequenceId is the concurrency token for the next update of this operation. + // Typed as a string because that is what the service sends; see above. + SequenceId string `json:"sequence_id,omitempty"` +} + +// updateOperationRequest carries the fields a later write for the same resource +// changes. action_type and resource_key are omitted: the service fixes them when +// the operation is created and ignores them here. +type updateOperationRequest struct { + State *json.RawMessage `json:"state,omitempty"` + ErrorMessage string `json:"error_message,omitempty"` + ResourceId string `json:"resource_id,omitempty"` + Status bundledeployments.OperationStatus `json:"status,omitempty"` + SequenceId string `json:"sequence_id,omitempty"` +} + +// operationClient records operations under a deployment version. +type operationClient interface { + CreateOperation(ctx context.Context, parent, resourceKey string, op bundledeployments.Operation) (operationResponse, error) + UpdateOperation(ctx context.Context, parent, resourceKey string, body updateOperationRequest) (operationResponse, error) +} + +// apiOperationClient talks to the operations API through the workspace client. +type apiOperationClient struct { + client *client.DatabricksClient +} + +// newAPIOperationClient returns an operationClient that posts to the DMS API. +func newAPIOperationClient(c *client.DatabricksClient) operationClient { + return &apiOperationClient{client: c} +} + +func (a *apiOperationClient) CreateOperation(ctx context.Context, parent, resourceKey string, op bundledeployments.Operation) (operationResponse, error) { + var result operationResponse + path := fmt.Sprintf("/api/2.0/bundle/%s/operations", parent) + err := a.client.Do(ctx, http.MethodPost, path, + auth.WorkspaceIDHeaders(a.client.Config), + map[string]any{"resource_key": resourceKey}, + op, &result) + if err != nil { + return operationResponse{}, err + } + return result, nil +} + +func (a *apiOperationClient) UpdateOperation(ctx context.Context, parent, resourceKey string, body updateOperationRequest) (operationResponse, error) { + var result operationResponse + path := fmt.Sprintf("/api/2.0/bundle/%s/operations/%s", parent, resourceKey) + err := a.client.Do(ctx, http.MethodPatch, path, + auth.WorkspaceIDHeaders(a.client.Config), + map[string]any{"update_mask": strings.Join(updatableFields, ",")}, + body, &result) + if err != nil { + return operationResponse{}, err + } + return result, nil +} diff --git a/bundle/direct/opqueue.go b/bundle/direct/opqueue.go index ec5b97ac37d..ea796abf84d 100644 --- a/bundle/direct/opqueue.go +++ b/bundle/direct/opqueue.go @@ -41,9 +41,11 @@ type operationQueue struct { // mu guards the fields below. mu sync.Mutex - // pending holds the newest operation per resource key that no worker has taken - // yet. Empty for a key means everything recorded for it has been uploaded. - pending map[string]recordedOperation + // pending holds the operations waiting per resource key, oldest first. Every one + // is uploaded: a resource can write state more than once in a deploy (a recreate + // drops it, then saves the new resource), and each write is its own event, so + // dropping the older one would hide a step. No key means nothing is waiting. + pending map[string][]recordedOperation // queuedOrUploading means "some worker will get to this key". Recording such a // key writes to pending only, so two workers never upload one resource at once. @@ -63,7 +65,7 @@ func newOperationQueue(ctx context.Context, uploader operationUploader) *operati q := &operationQueue{ uploader: uploader, queue: make(chan string, operationQueueSize), - pending: make(map[string]recordedOperation), + pending: make(map[string][]recordedOperation), queuedOrUploading: make(map[string]bool), } @@ -75,23 +77,26 @@ func newOperationQueue(ctx context.Context, uploader operationUploader) *operati return q } -// record serializes an operation and hands it to the upload workers, so an error -// here means the payload could not be built; upload errors surface at close. +// RecordOperation implements dstate.OperationSink: every state write becomes an +// operation, so DMS mirrors the WAL. state is already the serialized envelope, and +// nil for a delete. // -// An earlier upload failure does not stop this: every applied resource is still -// recorded, best effort, so DMS ends up as close to reality as it can get. -func (q *operationQueue) record(ctx context.Context, resourceKey string, action deployplan.ActionType, resourceID string, state any, dependsOn []deployplan.DependsOnEntry) error { +// An earlier upload failure does not stop this: every write is still recorded, best +// effort, so DMS ends up as close to reality as it can get. +func (q *operationQueue) RecordOperation(ctx context.Context, resourceKey string, action deployplan.ActionType, resourceID string, state json.RawMessage) { if q == nil { - return nil + return } - op, err := newRecordedOperation(action, resourceID, state, dependsOn) + op, err := newStateOperation(action, resourceID, state) if err != nil { - return err + // The deploy already persisted this write locally, so failing it here would + // report an error about history for a resource that deployed fine. + log.Warnf(ctx, "Not recording operation for %s: %s", resourceKey, err) + return } q.enqueue(ctx, resourceKey, op) - return nil } // recordFailure records that applying a resource failed, so the deployment history @@ -111,22 +116,17 @@ func (q *operationQueue) recordFailure(ctx context.Context, resourceKey string, q.enqueue(ctx, resourceKey, op) } -// enqueue publishes op as the pending operation for resourceKey and makes sure a +// enqueue appends op to the operations waiting for resourceKey and makes sure a // worker will pick it up. func (q *operationQueue) enqueue(ctx context.Context, resourceKey string, op recordedOperation) { q.mu.Lock() - _, replaced := q.pending[resourceKey] - q.pending[resourceKey] = op + q.pending[resourceKey] = append(q.pending[resourceKey], op) alreadyHandled := q.queuedOrUploading[resourceKey] q.queuedOrUploading[resourceKey] = true q.mu.Unlock() - if replaced { - log.Debugf(ctx, "Coalescing queued deployment operation for %s", resourceKey) - } - // A worker will re-read pending before it finishes, so it picks up the operation - // written above. Queueing again would let a second worker upload the same key. + // appended above. Queueing again would let a second worker upload the same key. if alreadyHandled { return } @@ -175,23 +175,26 @@ func (q *operationQueue) work(ctx context.Context) { } } -// take claims the operation waiting for resourceKey, reporting false and clearing -// the queuedOrUploading mark when nothing is left, which lets record queue it again. -// Both happen under one lock, so a key can never be left for no worker to pick up. +// take claims the oldest operation waiting for resourceKey, reporting false and +// clearing the queuedOrUploading mark when nothing is left, which lets record queue +// it again. Both happen under one lock, so a key can never be left for no worker to +// pick up. func (q *operationQueue) take(resourceKey string) (recordedOperation, bool) { q.mu.Lock() defer q.mu.Unlock() - op, ok := q.pending[resourceKey] - if !ok { + ops := q.pending[resourceKey] + if len(ops) == 0 { + delete(q.pending, resourceKey) delete(q.queuedOrUploading, resourceKey) return recordedOperation{}, false } - // The mark stays until the branch above clears it, so anything recorded during - // this upload is still picked up and no second worker takes the key meanwhile. - delete(q.pending, resourceKey) - return op, true + // Oldest first, so the service sees the writes in the order they happened. The + // mark stays until the branch above clears it, so anything recorded during this + // upload is still picked up and no second worker takes the key meanwhile. + q.pending[resourceKey] = ops[1:] + return ops[0], true } // setErr keeps the first upload error; later ones are dropped because one failure diff --git a/bundle/direct/opqueue_test.go b/bundle/direct/opqueue_test.go index 7a98dc43774..1048db35d9c 100644 --- a/bundle/direct/opqueue_test.go +++ b/bundle/direct/opqueue_test.go @@ -2,6 +2,7 @@ package direct import ( "context" + "encoding/json" "errors" "strconv" "strings" @@ -9,6 +10,7 @@ import ( "testing" "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/cli/bundle/direct/dstate" "github.com/databricks/databricks-sdk-go/service/bundledeployments" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -75,9 +77,17 @@ func (f *fakeUploader) resourceIDFor(resourceKey string) string { return f.resourceIDs[resourceKey] } +// envelope builds the serialized RecordedState the state DB hands the queue. +func envelope(t *testing.T, name string) json.RawMessage { + t.Helper() + raw, err := json.Marshal(dstate.RecordedState{State: json.RawMessage(`{"name":"` + name + `"}`)}) + require.NoError(t, err) + return raw +} + func recordState(t *testing.T, q *operationQueue, resourceKey, name string) { t.Helper() - require.NoError(t, q.record(t.Context(), resourceKey, deployplan.Update, "id-1", map[string]string{"name": name}, nil)) + q.RecordOperation(t.Context(), resourceKey, deployplan.Update, "id-1", envelope(t, name)) } func TestOperationQueueUploadsEachOperation(t *testing.T) { @@ -92,10 +102,14 @@ func TestOperationQueueUploadsEachOperation(t *testing.T) { assert.Len(t, f.recorded(), 20) } -func TestOperationQueueCoalescesQueuedOperationsForSameResource(t *testing.T) { - // Hold the first upload so later operations for the same resource pile up in - // the queue and are collapsed into one. - f := &fakeUploader{block: make(chan struct{}), started: make(chan string, 1)} +func TestOperationQueueUploadsEveryWriteForSameResource(t *testing.T) { + // Hold the first upload so the writes behind it queue up. Each one is its own + // event, so all three are uploaded, oldest first - a resource can legitimately + // write state several times in one deploy (see Recreate). + // + // started is buffered for all three: every write now uploads, and a worker + // blocking on an unread send would deadlock the drain below. + f := &fakeUploader{block: make(chan struct{}), started: make(chan string, 3)} q := newOperationQueue(t.Context(), f) recordState(t, q, "resources.jobs.foo", "v1") @@ -109,17 +123,20 @@ func TestOperationQueueCoalescesQueuedOperationsForSameResource(t *testing.T) { close(f.block) require.NoError(t, q.close()) - // Two uploads, not three: v2 was superseded by v3 while both were queued, and - // the last recorded state is the one the service ends up with. assert.Equal(t, []string{ `resources.jobs.foo={"state":{"name":"v1"}}`, + `resources.jobs.foo={"state":{"name":"v2"}}`, `resources.jobs.foo={"state":{"name":"v3"}}`, }, f.recorded()) } -func TestOperationQueueCoalescingKeepsLatestOperation(t *testing.T) { - // Hold the first upload so the operations below stay queued and coalesce. - f := &fakeUploader{block: make(chan struct{}), started: make(chan string, 1)} +func TestOperationQueueUploadsQueuedWritesWhileWorkersAreBusy(t *testing.T) { + // Every worker is parked mid-upload, so the writes below sit in pending rather + // than being picked up. Both still go out, in order, once a worker frees up. + // + // started is buffered for the two foo writes as well: nothing reads it after the + // loop below, and a worker blocking on the send would deadlock the drain. + f := &fakeUploader{block: make(chan struct{}), started: make(chan string, operationUploadWorkers+2)} q := newOperationQueue(t.Context(), f) recordState(t, q, "resources.jobs.hold", "v1") @@ -131,27 +148,26 @@ func TestOperationQueueCoalescingKeepsLatestOperation(t *testing.T) { assert.Equal(t, "resources.jobs.hold"+strconv.Itoa(i), <-f.started) } - // A resource whose ID is only known after it was created: the first operation - // has no ID, the second fills it in. - require.NoError(t, q.record(t.Context(), "resources.jobs.foo", deployplan.Create, "", map[string]string{"name": "created"}, nil)) - require.NoError(t, q.record(t.Context(), "resources.jobs.foo", deployplan.Create, "id-1", map[string]string{"name": "updated"}, nil)) + // A resource whose ID is only known after it was created: the first write has no + // ID, the second fills it in. + q.RecordOperation(t.Context(), "resources.jobs.foo", deployplan.Create, "", envelope(t, "created")) + q.RecordOperation(t.Context(), "resources.jobs.foo", deployplan.Create, "id-1", envelope(t, "updated")) close(f.block) require.NoError(t, q.close()) - // One upload, not two: the second operation replaced the first while every - // worker was busy, so the extra CreateOperation round trip never happens. - var uploadsForFoo int + var uploadsForFoo []string for _, u := range f.recorded() { if strings.HasPrefix(u, "resources.jobs.foo=") { - uploadsForFoo++ + uploadsForFoo = append(uploadsForFoo, u) } } - assert.Equal(t, 1, uploadsForFoo, "the two operations should coalesce into one upload") + assert.Equal(t, []string{ + `resources.jobs.foo={"state":{"name":"created"}}`, + `resources.jobs.foo={"state":{"name":"updated"}}`, + }, uploadsForFoo) - // Everything comes from the newest operation: it carries the resource's full - // state, and the ID it learned after the create. - assert.Contains(t, f.recorded(), `resources.jobs.foo={"state":{"name":"updated"}}`) + // The ID recorded last is the one the create learned. assert.Equal(t, "id-1", f.resourceIDFor("resources.jobs.foo")) assert.Equal(t, bundledeployments.OperationActionTypeOperationActionTypeCreate, @@ -165,11 +181,11 @@ func TestOperationQueueRecordDuringUploadIsStillUploaded(t *testing.T) { f := &fakeUploader{block: make(chan struct{}), started: make(chan string, 1)} q := newOperationQueue(t.Context(), f) - require.NoError(t, q.record(t.Context(), "resources.jobs.foo", deployplan.Create, "", map[string]string{"name": "v1"}, nil)) + q.RecordOperation(t.Context(), "resources.jobs.foo", deployplan.Create, "", envelope(t, "v1")) assert.Equal(t, "resources.jobs.foo", <-f.started) // The worker has taken the key off the queue and is uploading v1 right now. - require.NoError(t, q.record(t.Context(), "resources.jobs.foo", deployplan.Create, "id-1", map[string]string{"name": "v2"}, nil)) + q.RecordOperation(t.Context(), "resources.jobs.foo", deployplan.Create, "id-1", envelope(t, "v2")) close(f.block) require.NoError(t, q.close()) @@ -207,11 +223,11 @@ func TestOperationQueueKeepsRecordingAfterUploadError(t *testing.T) { // Wait for the failing upload to finish, so the error is stored before the next // record rather than racing it. - require.NoError(t, q.record(t.Context(), "resources.jobs.foo", deployplan.Create, "id-1", map[string]string{"name": "v1"}, nil)) + q.RecordOperation(t.Context(), "resources.jobs.foo", deployplan.Create, "id-1", envelope(t, "v1")) assert.Equal(t, "resources.jobs.foo", <-f.done) // The next resource is still accepted, even though the first upload failed. - require.NoError(t, q.record(t.Context(), "resources.jobs.bar", deployplan.Create, "id-2", map[string]string{"name": "v1"}, nil)) + q.RecordOperation(t.Context(), "resources.jobs.bar", deployplan.Create, "id-2", envelope(t, "v1")) // Both were attempted, and close still reports the failure so the deploy fails. require.ErrorIs(t, q.close(), uploadErr) @@ -232,10 +248,10 @@ func TestOperationQueueDrainsQueuedOperationsAfterUploadError(t *testing.T) { // Every worker is parked mid-upload, so these stay queued. for i := range operationUploadWorkers { - require.NoError(t, q.record(t.Context(), "resources.jobs.hold"+strconv.Itoa(i), deployplan.Create, "id-1", map[string]string{"name": "v1"}, nil)) + q.RecordOperation(t.Context(), "resources.jobs.hold"+strconv.Itoa(i), deployplan.Create, "id-1", envelope(t, "v1")) assert.Equal(t, "resources.jobs.hold"+strconv.Itoa(i), <-f.started) } - require.NoError(t, q.record(t.Context(), "resources.jobs.queued", deployplan.Create, "id-2", map[string]string{"name": "v1"}, nil)) + q.RecordOperation(t.Context(), "resources.jobs.queued", deployplan.Create, "id-2", envelope(t, "v1")) close(f.block) require.ErrorIs(t, q.close(), uploadErr) @@ -245,26 +261,23 @@ func TestOperationQueueDrainsQueuedOperationsAfterUploadError(t *testing.T) { assert.Len(t, f.recorded(), operationUploadWorkers+1) } -func TestOperationQueueRecordRejectsUnsupportedAction(t *testing.T) { +func TestOperationQueueRecordDropsUnsupportedAction(t *testing.T) { f := &fakeUploader{} q := newOperationQueue(t.Context(), f) - // Serialization failures surface at record time, on the resource that caused - // them, rather than from the drain at the end of apply. - err := q.record(t.Context(), "resources.jobs.foo", deployplan.Skip, "id-1", nil, nil) - require.Error(t, err) + // The state write already succeeded, so an operation that cannot be described is + // dropped with a warning rather than failing the deploy. + q.RecordOperation(t.Context(), "resources.jobs.foo", deployplan.Skip, "id-1", nil) require.NoError(t, q.close()) assert.Empty(t, f.recorded()) } -func TestOperationQueueRecordRejectsOversizedState(t *testing.T) { +func TestOperationQueueRecordDropsOversizedState(t *testing.T) { f := &fakeUploader{} q := newOperationQueue(t.Context(), f) - big := map[string]string{"name": strings.Repeat("x", maxOperationStateSize)} - err := q.record(t.Context(), "resources.jobs.foo", deployplan.Create, "id-1", big, nil) - require.ErrorContains(t, err, "exceeds the 65536 byte limit") + q.RecordOperation(t.Context(), "resources.jobs.foo", deployplan.Create, "id-1", envelope(t, strings.Repeat("x", maxOperationStateSize))) require.NoError(t, q.close()) assert.Empty(t, f.recorded()) @@ -328,23 +341,23 @@ func TestOperationQueueUploadsOneResourceAtATime(t *testing.T) { u := &serialUploader{live: map[string]bool{}, last: map[string]string{}} q := newOperationQueue(ctx, u) - // Collect record errors instead of asserting inside the goroutines: testify - // assertions may only run on the goroutine running the test function. - errs := make(chan error, workers*perWorker) + // The envelopes are built up front: json.Marshal is fine on many goroutines, + // but the helper takes *testing.T, which is not. + states := make([]json.RawMessage, workers) + for w := range workers { + states[w] = envelope(t, strconv.Itoa(w)) + } + var wg sync.WaitGroup for w := range workers { wg.Go(func() { for i := range perWorker { key := "resources.jobs.job" + strconv.Itoa((w*perWorker+i)%distinctKeyMod) - errs <- q.record(ctx, key, deployplan.Update, "id-1", map[string]string{"name": strconv.Itoa(w)}, nil) + q.RecordOperation(ctx, key, deployplan.Update, "id-1", states[w]) } }) } wg.Wait() - close(errs) - for err := range errs { - require.NoError(t, err) - } require.NoError(t, q.close()) require.False(t, u.uneven, "two uploads overlapped for the same resource key") @@ -360,6 +373,6 @@ func TestNilOperationQueueIsNoOp(t *testing.T) { // no-op, so Apply does not have to branch. q := newOperationQueue(t.Context(), nil) require.Nil(t, q) - require.NoError(t, q.record(t.Context(), "resources.jobs.foo", deployplan.Create, "id-1", nil, nil)) + q.RecordOperation(t.Context(), "resources.jobs.foo", deployplan.Create, "id-1", nil) require.NoError(t, q.close()) } diff --git a/bundle/direct/oprecorder.go b/bundle/direct/oprecorder.go index 32494c07325..26afccd8f2f 100644 --- a/bundle/direct/oprecorder.go +++ b/bundle/direct/oprecorder.go @@ -5,9 +5,11 @@ import ( "encoding/json" "fmt" "strings" + "sync" "github.com/databricks/cli/bundle/deployplan" "github.com/databricks/cli/bundle/direct/dstate" + "github.com/databricks/databricks-sdk-go/client" "github.com/databricks/databricks-sdk-go/service/bundledeployments" ) @@ -43,39 +45,25 @@ type recordedOperation struct { state json.RawMessage } -// newRecordedOperation serializes an applied operation for upload. state is the -// local config after the operation and must be nil for delete operations. It -// errors when the serialized state exceeds maxOperationStateSize. -func newRecordedOperation(action deployplan.ActionType, resourceID string, state any, dependsOn []deployplan.DependsOnEntry) (recordedOperation, error) { +// newStateOperation describes a state write for upload. state is the serialized +// RecordedState envelope the state DB just persisted, and nil for a delete, where +// the resource is gone. It errors when the state exceeds maxOperationStateSize. +func newStateOperation(action deployplan.ActionType, resourceID string, state json.RawMessage) (recordedOperation, error) { actionType, err := deployActionToSDK(action) if err != nil { return recordedOperation{}, err } - op := recordedOperation{ + if len(state) > maxOperationStateSize { + return recordedOperation{}, fmt.Errorf("serialized state is %d bytes, which exceeds the %d byte limit for recording deployment history", len(state), maxOperationStateSize) + } + + return recordedOperation{ action: actionType, resourceID: resourceID, status: bundledeployments.OperationStatusOperationStatusSucceeded, - } - - // Operation.State carries the serialized state, which DMS serves back as - // resource state. Unset for delete: the resource is gone. - if state != nil { - config, err := json.Marshal(state) - if err != nil { - return recordedOperation{}, fmt.Errorf("serializing state: %w", err) - } - raw, err := json.Marshal(dstate.RecordedState{State: config, DependsOn: dependsOn}) - if err != nil { - return recordedOperation{}, fmt.Errorf("serializing state: %w", err) - } - if len(raw) > maxOperationStateSize { - return recordedOperation{}, fmt.Errorf("serialized state is %d bytes, which exceeds the %d byte limit for recording deployment history", len(raw), maxOperationStateSize) - } - op.state = raw - } - - return op, nil + state: state, + }, nil } // newFailedOperation records an operation that did not apply, so the deployment @@ -135,24 +123,45 @@ type operationUploader interface { upload(ctx context.Context, resourceKey string, op recordedOperation) error } -// operationRecorder uploads operations via the DMS CreateOperation API. +// operationRecorder uploads operations via the DMS operations API. type operationRecorder struct { - client bundledeployments.BundleDeploymentsInterface + ops operationClient // parent is the version the operations are recorded under, formatted as // "deployments/{deployment_id}/versions/{version_id}". parent string + + // mu guards sequenceIDs. + mu sync.Mutex + + // sequenceIDs holds the last sequence_id the service returned per resource key, + // which is how a resource already recorded in this version is recognised. The + // service names operations "operations/{resource_key}", so it keeps one per + // resource per version: the second write for a resource has to update that + // operation, and echo this value as the concurrency precondition. + sequenceIDs map[string]string } -// NewOperationRecorder returns an operationUploader backed by the DMS -// CreateOperation API. deploymentID and version identify the deployment version -// assigned by DMS that the operations are recorded under. -func NewOperationRecorder(client bundledeployments.BundleDeploymentsInterface, deploymentID string, version int64) operationUploader { +// NewOperationRecorder returns an operationUploader backed by the DMS operations +// API. deploymentID and version identify the deployment version assigned by DMS +// that the operations are recorded under. +func NewOperationRecorder(apiClient *client.DatabricksClient, deploymentID string, version int64) operationUploader { + return newOperationRecorder(newAPIOperationClient(apiClient), deploymentID, version) +} + +// newOperationRecorder is the internal constructor, so tests can supply their own +// operationClient. +func newOperationRecorder(ops operationClient, deploymentID string, version int64) operationUploader { return &operationRecorder{ - client: client, - parent: fmt.Sprintf("deployments/%s/versions/%d", deploymentID, version), + ops: ops, + parent: fmt.Sprintf("deployments/%s/versions/%d", deploymentID, version), + sequenceIDs: make(map[string]string), } } +// updatableFields are the operation fields a later write for the same resource can +// change. resource_id is included because a recreate learns a new one. +var updatableFields = []string{"state", "error_message", "resource_id", "status"} + func (r *operationRecorder) upload(ctx context.Context, resourceKey string, op recordedOperation) error { // DMS resource keys are unprefixed (e.g. "jobs.foo"), while the CLI's state // keys carry a leading "resources." (e.g. "resources.jobs.foo"). Strip it on @@ -179,12 +188,36 @@ func (r *operationRecorder) upload(ctx context.Context, resourceKey string, op r operation.State = &raw } - _, err := r.client.CreateOperation(ctx, bundledeployments.CreateOperationRequest{ - Parent: r.parent, - ResourceKey: dmsKey, - Operation: operation, - }) - return err + r.mu.Lock() + sequenceID, recorded := r.sequenceIDs[dmsKey] + r.mu.Unlock() + + var result operationResponse + var err error + if recorded { + // Only the masked fields and sequence_id are read on an update; action_type + // stays as the operation was created, so sending it would just be misleading. + result, err = r.ops.UpdateOperation(ctx, r.parent, dmsKey, updateOperationRequest{ + State: operation.State, + ErrorMessage: operation.ErrorMessage, + ResourceId: operation.ResourceId, + Status: operation.Status, + SequenceId: sequenceID, + }) + } else { + result, err = r.ops.CreateOperation(ctx, r.parent, dmsKey, operation) + } + if err != nil { + return err + } + + // Remember the sequence the service assigned, so the next write for this + // resource updates rather than re-creates. + r.mu.Lock() + r.sequenceIDs[dmsKey] = result.SequenceId + r.mu.Unlock() + + return nil } // deployActionToSDK maps a deployplan action to its DMS operation action type. diff --git a/bundle/direct/oprecorder_test.go b/bundle/direct/oprecorder_test.go index b793ab65a08..d80caa09341 100644 --- a/bundle/direct/oprecorder_test.go +++ b/bundle/direct/oprecorder_test.go @@ -14,80 +14,120 @@ import ( "github.com/stretchr/testify/require" ) +// fakeOpCall is one recorded call to the operations API. +type fakeOpCall struct { + method string + parent string + resourceKey string + op bundledeployments.Operation + update updateOperationRequest +} + type fakeOpClient struct { - bundledeployments.BundleDeploymentsInterface + mu sync.Mutex + calls []fakeOpCall + // sequence is what the service reports back; a string, as the service sends it. + sequence string +} - mu sync.Mutex - requests []bundledeployments.CreateOperationRequest +func (f *fakeOpClient) CreateOperation(ctx context.Context, parent, resourceKey string, op bundledeployments.Operation) (operationResponse, error) { + f.mu.Lock() + defer f.mu.Unlock() + f.calls = append(f.calls, fakeOpCall{method: "create", parent: parent, resourceKey: resourceKey, op: op}) + return operationResponse{SequenceId: f.sequence}, nil } -func (f *fakeOpClient) CreateOperation(ctx context.Context, req bundledeployments.CreateOperationRequest) (*bundledeployments.Operation, error) { +func (f *fakeOpClient) UpdateOperation(ctx context.Context, parent, resourceKey string, body updateOperationRequest) (operationResponse, error) { f.mu.Lock() defer f.mu.Unlock() - f.requests = append(f.requests, req) - return &bundledeployments.Operation{}, nil + f.calls = append(f.calls, fakeOpCall{method: "update", parent: parent, resourceKey: resourceKey, update: body}) + return operationResponse{SequenceId: f.sequence}, nil } // uploadOne records a single operation through the given uploader, mirroring what // an operationQueue worker does. -func uploadOne(t *testing.T, u operationUploader, resourceKey string, action deployplan.ActionType, resourceID string, state any) { +func uploadOne(t *testing.T, u operationUploader, resourceKey string, action deployplan.ActionType, resourceID string, state json.RawMessage) { t.Helper() - op, err := newRecordedOperation(action, resourceID, state, nil) + op, err := newStateOperation(action, resourceID, state) require.NoError(t, err) require.NoError(t, u.upload(t.Context(), resourceKey, op)) } func TestOperationRecorderStripsResourcePrefix(t *testing.T) { - f := &fakeOpClient{} - r := NewOperationRecorder(f, "dep-1", 2) + f := &fakeOpClient{sequence: "1"} + r := newOperationRecorder(f, "dep-1", 2) - uploadOne(t, r, "resources.jobs.foo", deployplan.Create, "job-123", map[string]string{"name": "foo"}) + uploadOne(t, r, "resources.jobs.foo", deployplan.Create, "job-123", envelope(t, "foo")) - require.Len(t, f.requests, 1) - req := f.requests[0] + require.Len(t, f.calls, 1) + c := f.calls[0] // The wire key drops the CLI-internal "resources." prefix, both in the query // param and the operation body. - assert.Equal(t, "jobs.foo", req.ResourceKey) - assert.Equal(t, "jobs.foo", req.Operation.ResourceKey) - assert.Equal(t, "deployments/dep-1/versions/2", req.Parent) - assert.Equal(t, bundledeployments.OperationActionTypeOperationActionTypeCreate, req.Operation.ActionType) - assert.Equal(t, "job-123", req.Operation.ResourceId) - require.NotNil(t, req.Operation.State) + assert.Equal(t, "create", c.method) + assert.Equal(t, "jobs.foo", c.resourceKey) + assert.Equal(t, "jobs.foo", c.op.ResourceKey) + assert.Equal(t, "deployments/dep-1/versions/2", c.parent) + assert.Equal(t, bundledeployments.OperationActionTypeOperationActionTypeCreate, c.op.ActionType) + assert.Equal(t, "job-123", c.op.ResourceId) + require.NotNil(t, c.op.State) } -func TestNewRecordedOperationRecordsStateAsIs(t *testing.T) { - state := struct { - Name string `json:"name"` - Token string `json:"token" bundle:"sensitive"` - }{Name: "foo", Token: "super-secret"} +func TestOperationRecorderUpdatesSecondWriteForSameResource(t *testing.T) { + // One operation per resource per version: the second write has to update the + // first, echoing the sequence_id the service returned as its precondition. + f := &fakeOpClient{sequence: "7"} + r := newOperationRecorder(f, "dep-1", 2) - op, err := newRecordedOperation(deployplan.Create, "job-123", state, nil) - require.NoError(t, err) + uploadOne(t, r, "resources.jobs.foo", deployplan.Recreate, "", nil) + uploadOne(t, r, "resources.jobs.foo", deployplan.Recreate, "job-456", envelope(t, "new")) + + require.Len(t, f.calls, 2) + assert.Equal(t, "create", f.calls[0].method) - // The state is serialized as-is, including fields tagged bundle:"sensitive". - assert.JSONEq(t, - `{"state":{"name":"foo","token":"super-secret"}}`, - string(op.state)) + assert.Equal(t, "update", f.calls[1].method) + assert.Equal(t, "jobs.foo", f.calls[1].resourceKey) + assert.Equal(t, "7", f.calls[1].update.SequenceId) + assert.Equal(t, "job-456", f.calls[1].update.ResourceId) + require.NotNil(t, f.calls[1].update.State) } -func TestNewRecordedOperationRecordsDependsOn(t *testing.T) { - // depends_on rides in an envelope alongside the config: it cannot be - // recomputed from the config, whose references are already resolved. - dependsOn := []deployplan.DependsOnEntry{{Node: "resources.jobs.bar", Label: "${resources.jobs.bar.id}"}} +func TestOperationRecorderTracksSequencePerResource(t *testing.T) { + // A different resource has its own operation, so its first write creates. + f := &fakeOpClient{sequence: "1"} + r := newOperationRecorder(f, "dep-1", 2) - op, err := newRecordedOperation(deployplan.Create, "job-123", map[string]string{"name": "foo"}, dependsOn) + uploadOne(t, r, "resources.jobs.foo", deployplan.Create, "id-1", envelope(t, "foo")) + uploadOne(t, r, "resources.jobs.bar", deployplan.Create, "id-2", envelope(t, "bar")) + + require.Len(t, f.calls, 2) + assert.Equal(t, "create", f.calls[0].method) + assert.Equal(t, "create", f.calls[1].method) +} + +func TestNewStateOperationRecordsEnvelopeAsIs(t *testing.T) { + // The state DB serializes the envelope (see dstate.SaveState); the operation + // carries it through untouched, sensitive fields and all. + state := json.RawMessage(`{"state":{"name":"foo","token":"super-secret"}}`) + + op, err := newStateOperation(deployplan.Create, "job-123", state) require.NoError(t, err) - assert.JSONEq(t, - `{"state":{"name":"foo"},"depends_on":[{"node":"resources.jobs.bar","label":"${resources.jobs.bar.id}"}]}`, - string(op.state)) + assert.JSONEq(t, string(state), string(op.state)) + assert.Equal(t, bundledeployments.OperationStatusOperationStatusSucceeded, op.status) } -func TestNewRecordedOperationRejectsUnsupportedAction(t *testing.T) { - _, err := newRecordedOperation(deployplan.Skip, "job-123", nil, nil) +func TestNewStateOperationRejectsUnsupportedAction(t *testing.T) { + _, err := newStateOperation(deployplan.Skip, "job-123", nil) assert.Error(t, err) } +func TestNewStateOperationRejectsOversizedState(t *testing.T) { + big := json.RawMessage(strings.Repeat("x", maxOperationStateSize+1)) + + _, err := newStateOperation(deployplan.Create, "job-123", big) + assert.ErrorContains(t, err, "exceeds the 65536 byte limit") +} + func TestNewFailedOperationRecordsError(t *testing.T) { op, err := newFailedOperation(deployplan.Create, "", nil, errors.New("cluster spec is invalid")) require.NoError(t, err) diff --git a/bundle/migrate/build_state.go b/bundle/migrate/build_state.go index c08cf01c31f..459c14a0b6d 100644 --- a/bundle/migrate/build_state.go +++ b/bundle/migrate/build_state.go @@ -233,7 +233,9 @@ func BuildStateFromTF( } } - if err := stateDB.SaveState(node, id, sv.Value, dependsOn); err != nil { + // Migration rebuilds local state from terraform's; nothing is deployed, and + // the DMS sink is never set on this state, so the action is not reported. + if err := stateDB.SaveState(ctx, node, id, sv.Value, dependsOn, deployplan.Create); err != nil { return warningsSeen, fmt.Errorf("%s: SaveState: %w", node, err) } } diff --git a/bundle/phases/deploy.go b/bundle/phases/deploy.go index 5513944cf7a..d0faca36bca 100644 --- a/bundle/phases/deploy.go +++ b/bundle/phases/deploy.go @@ -17,7 +17,6 @@ import ( "github.com/databricks/cli/bundle/deploy/snapshot" "github.com/databricks/cli/bundle/deploy/terraform" "github.com/databricks/cli/bundle/deployplan" - "github.com/databricks/cli/bundle/direct" "github.com/databricks/cli/bundle/libraries" "github.com/databricks/cli/bundle/metrics" "github.com/databricks/cli/bundle/permissions" @@ -295,15 +294,9 @@ func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHand return } if haveApproval { - if recorder != nil { - // Record operations under the version created before planning, so DMS holds - // the deployed resource state. - b.DeploymentBundle.OpRec = direct.NewOperationRecorder( - b.WorkspaceClient(ctx).BundleDeployments, - recorder.DeploymentID(), - recorder.Version(), - ) - } + // Record operations under the version created before planning, so DMS holds + // the deployed resource state. + setOperationRecorder(ctx, b, recorder) deployCore(ctx, b, plan, stateEngine, requestedEngine, recorder) } else { cmdio.LogString(ctx, "Deployment cancelled!") diff --git a/bundle/phases/destroy.go b/bundle/phases/destroy.go index d2e072d23ce..e09d5716f5f 100644 --- a/bundle/phases/destroy.go +++ b/bundle/phases/destroy.go @@ -13,7 +13,6 @@ import ( "github.com/databricks/cli/bundle/deploy/lock" "github.com/databricks/cli/bundle/deploy/terraform" "github.com/databricks/cli/bundle/deployplan" - "github.com/databricks/cli/bundle/direct" "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dms" @@ -217,13 +216,7 @@ func Destroy(ctx context.Context, b *bundle.Bundle, engine engine.EngineType) { logdiag.LogError(ctx, err) return } - if recorder != nil { - b.DeploymentBundle.OpRec = direct.NewOperationRecorder( - b.WorkspaceClient(ctx).BundleDeployments, - recorder.DeploymentID(), - recorder.Version(), - ) - } + setOperationRecorder(ctx, b, recorder) destroyCore(ctx, b, plan, engine, recorder) } else { cmdio.LogString(ctx, "Destroy cancelled!") diff --git a/bundle/phases/dms.go b/bundle/phases/dms.go index cb8f6ad2db4..23da595d4c8 100644 --- a/bundle/phases/dms.go +++ b/bundle/phases/dms.go @@ -9,9 +9,11 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/engine" + "github.com/databricks/cli/bundle/direct" "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/dms" "github.com/databricks/cli/libs/log" + "github.com/databricks/cli/libs/logdiag" "github.com/databricks/cli/libs/workspaceurls" "github.com/databricks/databricks-sdk-go/client" "github.com/databricks/databricks-sdk-go/service/bundledeployments" @@ -56,6 +58,23 @@ func newDeploymentRecorder(ctx context.Context, b *bundle.Bundle, eng engine.Eng }), nil } +// setOperationRecorder points the deployment at the version the recorder claimed, so +// the state writes during apply are recorded under it. A nil recorder means recording +// is off and leaves the deployment's uploader unset. +func setOperationRecorder(ctx context.Context, b *bundle.Bundle, recorder *dms.Recorder) { + if recorder == nil { + return + } + + apiClient, err := client.New(b.WorkspaceClient(ctx).Config) + if err != nil { + logdiag.LogError(ctx, err) + return + } + + b.DeploymentBundle.OpRec = direct.NewOperationRecorder(apiClient, recorder.DeploymentID(), recorder.Version()) +} + // logDeploymentHistory links to the deployment this deploy was recorded under, so // the user can open its history without hunting for the ID. A nil recorder means // recording is off, and a zero version means the version was never created. diff --git a/libs/testserver/bundle.go b/libs/testserver/bundle.go index 74a8d2156b3..574570ce194 100644 --- a/libs/testserver/bundle.go +++ b/libs/testserver/bundle.go @@ -29,6 +29,10 @@ type dmsDeployment struct { // resources is the latest resource state per resource key, updated as // operations are recorded. resources map[string]bundledeployments.Resource + // operations holds the recorded operations by resource name. The service keeps + // one per resource per version, so a resource written twice in a version updates + // its operation rather than adding another. + operations map[string]*bundledeployments.Operation // lastSuccessfulVersionID is the highest version that completed // successfully. The server advances last_successful_version_id only on // success (unlike last_version_id), and the read path treats a non-empty @@ -80,6 +84,7 @@ func (s *FakeWorkspace) CreateDeployment(req Request) Response { deployment: dep, versions: map[string]*bundledeployments.Version{}, resources: map[string]bundledeployments.Resource{}, + operations: map[string]*bundledeployments.Operation{}, } return Response{Body: dep} } @@ -259,8 +264,29 @@ func (s *FakeWorkspace) CreateOperation(req Request, deploymentID, versionID str return dmsInvalidArgument("error_message is only allowed when status is OPERATION_STATUS_FAILED") } - op.Name = "deployments/" + deploymentID + "/versions/" + versionID + "/operations/" + resourceKey + // The service names operations after the resource key, so it keeps one per + // resource per version: creating a second one for the same resource conflicts, + // and the caller has to use UpdateOperation instead. + opName := "deployments/" + deploymentID + "/versions/" + versionID + "/operations/" + resourceKey + if _, exists := d.operations[opName]; exists { + return Response{ + StatusCode: 409, + Body: map[string]string{"error_code": "RESOURCE_ALREADY_EXISTS", "message": "operation for " + resourceKey + " already exists in this version"}, + } + } + + op.Name = opName op.ResourceKey = resourceKey + op.SequenceId = 1 + d.operations[opName] = &op + + // The service sends sequence_id as a JSON string (proto3 encodes 64-bit ints + // that way) while the SDK struct types it as an int64, so the response is built + // by hand to match the wire format the CLI actually parses. + body, err := operationBody(&op) + if err != nil { + return Response{StatusCode: 500, Body: map[string]string{"message": err.Error()}} + } // Reflect the operation onto the deployment-level resource set the way the // backend does: a delete removes the resource, anything else upserts it. @@ -283,7 +309,100 @@ func (s *FakeWorkspace) CreateOperation(req Request, deploymentID, versionID str State: op.State, } } - return Response{Body: op} + return Response{Body: body} +} + +// operationBody renders an operation the way the service does: sequence_id as a +// JSON string, which the SDK struct cannot express (it types the field int64). +func operationBody(op *bundledeployments.Operation) (map[string]any, error) { + raw, err := json.Marshal(op) + if err != nil { + return nil, err + } + + var body map[string]any + dec := json.NewDecoder(bytes.NewReader(raw)) + dec.UseNumber() + if err := dec.Decode(&body); err != nil { + return nil, err + } + + body["sequence_id"] = strconv.FormatInt(op.SequenceId, 10) + return body, nil +} + +// UpdateOperation applies a later write for a resource already recorded in this +// version. sequence_id is the concurrency precondition and increments on success. +func (s *FakeWorkspace) UpdateOperation(req Request, deploymentID, versionID, resourceKey string) Response { + // sequence_id arrives as a string, which the SDK struct cannot hold (it types the + // field int64), so read the body twice: once for the typed fields and once for the + // precondition. + var op bundledeployments.Operation + if err := json.Unmarshal(req.Body, &op); err != nil { + return Response{StatusCode: 400, Body: map[string]string{"message": err.Error()}} + } + var precondition struct { + SequenceId string `json:"sequence_id"` + } + if err := json.Unmarshal(req.Body, &precondition); err != nil { + return Response{StatusCode: 400, Body: map[string]string{"message": err.Error()}} + } + + updateMask := req.URL.Query().Get("update_mask") + if updateMask == "" { + return dmsInvalidArgument("update_mask is required") + } + + defer s.LockUnlock()() + + d, ok := s.dmsDeployments[deploymentID] + if !ok { + return dmsNotFound("deployment " + deploymentID) + } + + opName := "deployments/" + deploymentID + "/versions/" + versionID + "/operations/" + resourceKey + existing, ok := d.operations[opName] + if !ok { + return dmsNotFound("operation " + opName) + } + if precondition.SequenceId != strconv.FormatInt(existing.SequenceId, 10) { + return dmsAborted("sequence_id is outdated; the operation is at " + strconv.FormatInt(existing.SequenceId, 10)) + } + + failed := op.Status == bundledeployments.OperationStatusOperationStatusFailed + if !failed && op.ErrorMessage != "" { + return dmsInvalidArgument("error_message is only allowed when status is OPERATION_STATUS_FAILED") + } + + // Only the mutable fields change; action_type and resource_key stay as created. + existing.State = op.State + existing.ErrorMessage = op.ErrorMessage + existing.ResourceId = op.ResourceId + existing.Status = op.Status + existing.SequenceId++ + + body, err := operationBody(existing) + if err != nil { + return Response{StatusCode: 500, Body: map[string]string{"message": err.Error()}} + } + + // Mirror onto the resource set the same way CreateOperation does, so the read + // path reflects the newest write. + if existing.ActionType == bundledeployments.OperationActionTypeOperationActionTypeDelete && !failed { + delete(d.resources, resourceKey) + } else { + d.resources[resourceKey] = bundledeployments.Resource{ + Name: "deployments/" + deploymentID + "/resources/" + resourceKey, + ResourceKey: resourceKey, + ResourceId: existing.ResourceId, + ResourceType: existing.ResourceType, + LastActionType: existing.ActionType, + LastVersionId: versionID, + State: existing.State, + } + } + + return Response{Body: body} } func (s *FakeWorkspace) ListResources(deploymentID string) Response { diff --git a/libs/testserver/handlers.go b/libs/testserver/handlers.go index a05f79986ec..1c9f7b50bb8 100644 --- a/libs/testserver/handlers.go +++ b/libs/testserver/handlers.go @@ -305,6 +305,9 @@ func AddDefaultHandlers(server *Server) { server.Handle("POST", "/api/2.0/bundle/deployments/{deployment_id}/versions/{version_id}/operations", func(req Request) any { return req.Workspace.CreateOperation(req, req.Vars["deployment_id"], req.Vars["version_id"]) }) + server.Handle("PATCH", "/api/2.0/bundle/deployments/{deployment_id}/versions/{version_id}/operations/{resource_key}", func(req Request) any { + return req.Workspace.UpdateOperation(req, req.Vars["deployment_id"], req.Vars["version_id"], req.Vars["resource_key"]) + }) server.Handle("GET", "/api/2.0/bundle/deployments/{deployment_id}/resources", func(req Request) any { return req.Workspace.ListResources(req.Vars["deployment_id"]) }) From 3f1c296be9c64b637d36cb48fba460bc820de01d Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Fri, 7 Aug 2026 11:31:48 +0000 Subject: [PATCH 050/125] bundle: add DATABRICKS_BUNDLE_DMS to enable deployment history recording Recording could only be turned on by setting experimental.record_deployment_history in the bundle, which is impractical for running the acceptance suite with DMS enabled: that would mean editing 650-odd databricks.yml files. DATABRICKS_BUNDLE_DMS turns it on for a whole run instead. The three places that branch on recording now go through env.RecordsDeploymentHistory, so the setting and the variable cannot drift apart. The validation that rejects the feature for users is deliberately left alone: it gates the yaml field, which the variable does not set. Verified on dogfood: a bundle with no experimental block records its deployment with DATABRICKS_BUNDLE_DMS=true alone. This is the groundwork for running the bundle suite with recording on. That run is not enabled yet - see the report on the state-file limitation. Co-authored-by: Isaac --- .../mutator/initialize_deployment_history.go | 4 +- bundle/env/dms.go | 25 ++++++++++++ bundle/env/dms_test.go | 38 +++++++++++++++++++ bundle/phases/dms.go | 16 ++++++-- cmd/bundle/utils/process.go | 3 +- 5 files changed, 80 insertions(+), 6 deletions(-) create mode 100644 bundle/env/dms.go create mode 100644 bundle/env/dms_test.go diff --git a/bundle/config/mutator/initialize_deployment_history.go b/bundle/config/mutator/initialize_deployment_history.go index 99aad70c7cc..b90800e4e70 100644 --- a/bundle/config/mutator/initialize_deployment_history.go +++ b/bundle/config/mutator/initialize_deployment_history.go @@ -5,6 +5,7 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/env" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dms" "github.com/databricks/databricks-sdk-go/service/bundledeployments" @@ -28,7 +29,8 @@ func (m *initializeDeploymentHistory) Name() string { } func (m *initializeDeploymentHistory) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { - if b.Config.Experimental == nil || !b.Config.Experimental.RecordDeploymentHistory { + configured := b.Config.Experimental != nil && b.Config.Experimental.RecordDeploymentHistory + if !env.RecordsDeploymentHistory(ctx, configured) { return nil } diff --git a/bundle/env/dms.go b/bundle/env/dms.go new file mode 100644 index 00000000000..53aeaccc011 --- /dev/null +++ b/bundle/env/dms.go @@ -0,0 +1,25 @@ +package env + +import "context" + +// DMSVariable names the environment variable that turns on deployment history +// recording without setting experimental.record_deployment_history in the bundle. +// It exists for the CLI's own acceptance tests, which run the whole bundle suite +// with DMS enabled: setting it here beats adding the field to every databricks.yml. +// +// Like ForceAllowRecordDeploymentHistoryVariable it is deliberately undocumented; see +// validate.ValidateRecordDeploymentHistory for why the feature is still gated off. +const DMSVariable = "DATABRICKS_BUNDLE_DMS" + +// DMS reports whether the environment turns on deployment history recording. +func DMS(ctx context.Context) bool { + value, ok := get(ctx, []string{DMSVariable}) + return ok && value != "" && value != "0" && value != "false" +} + +// RecordsDeploymentHistory reports whether this deploy records deployment history, +// from either the bundle setting or DMSVariable. It is the single predicate the +// recording code paths branch on, so the env var and the config field cannot drift. +func RecordsDeploymentHistory(ctx context.Context, configured bool) bool { + return configured || DMS(ctx) +} diff --git a/bundle/env/dms_test.go b/bundle/env/dms_test.go new file mode 100644 index 00000000000..f449d5ec6b9 --- /dev/null +++ b/bundle/env/dms_test.go @@ -0,0 +1,38 @@ +package env + +import ( + "testing" + + "github.com/databricks/cli/libs/env" + "github.com/stretchr/testify/assert" +) + +func TestDMS(t *testing.T) { + for _, tc := range []struct { + value string + want bool + }{ + {"true", true}, + {"1", true}, + {"", false}, + {"0", false}, + {"false", false}, + } { + ctx := env.Set(t.Context(), DMSVariable, tc.value) + assert.Equal(t, tc.want, DMS(ctx), "value %q", tc.value) + } +} + +func TestDMSUnset(t *testing.T) { + assert.False(t, DMS(t.Context())) +} + +func TestRecordsDeploymentHistory(t *testing.T) { + // The bundle setting alone is enough, and so is the environment; the env var + // exists so the acceptance suite can record without touching every databricks.yml. + assert.True(t, RecordsDeploymentHistory(t.Context(), true)) + assert.False(t, RecordsDeploymentHistory(t.Context(), false)) + + ctx := env.Set(t.Context(), DMSVariable, "true") + assert.True(t, RecordsDeploymentHistory(ctx, false)) +} diff --git a/bundle/phases/dms.go b/bundle/phases/dms.go index 23da595d4c8..8ca73fac94e 100644 --- a/bundle/phases/dms.go +++ b/bundle/phases/dms.go @@ -10,6 +10,7 @@ import ( "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/engine" "github.com/databricks/cli/bundle/direct" + "github.com/databricks/cli/bundle/env" "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/dms" "github.com/databricks/cli/libs/log" @@ -23,16 +24,16 @@ import ( // nil when DMS recording does not apply. A nil recorder is a no-op, so callers // do not need to branch on it. // -// Recording is enabled only when experimental.record_deployment_history is set -// AND the engine is direct: DMS resource state is tracked per direct-engine -// deployment. Returning nil for terraform leaves those deployments untouched. +// Recording is enabled only when the bundle asks for it (see +// recordsDeploymentHistory) AND the engine is direct: DMS resource state is tracked +// per direct-engine deployment. Returning nil for terraform leaves those untouched. // // The deployment ID is resolved from the workspace, not local state (see // dms.ResolveDeploymentID). The lookup happens here, after the deployment lock is // held, so it sees any deployment a concurrent deploy created. It is empty on the // first recorded deploy, where the recorder creates the deployment instead. func newDeploymentRecorder(ctx context.Context, b *bundle.Bundle, eng engine.EngineType, versionType dms.VersionType) (*dms.Recorder, error) { - if b.Config.Experimental == nil || !b.Config.Experimental.RecordDeploymentHistory { + if !recordsDeploymentHistory(ctx, b) { return nil, nil } if !eng.IsDirect() { @@ -58,6 +59,13 @@ func newDeploymentRecorder(ctx context.Context, b *bundle.Bundle, eng engine.Eng }), nil } +// recordsDeploymentHistory reports whether this bundle records deployment history, +// from experimental.record_deployment_history or DATABRICKS_BUNDLE_DMS. +func recordsDeploymentHistory(ctx context.Context, b *bundle.Bundle) bool { + configured := b.Config.Experimental != nil && b.Config.Experimental.RecordDeploymentHistory + return env.RecordsDeploymentHistory(ctx, configured) +} + // setOperationRecorder points the deployment at the version the recorder claimed, so // the state writes during apply are recorded under it. A nil recorder means recording // is off and leaves the deployment's uploader unset. diff --git a/cmd/bundle/utils/process.go b/cmd/bundle/utils/process.go index 3b90aef5c6a..36205181115 100644 --- a/cmd/bundle/utils/process.go +++ b/cmd/bundle/utils/process.go @@ -16,6 +16,7 @@ import ( "github.com/databricks/cli/bundle/deployplan" "github.com/databricks/cli/bundle/direct" "github.com/databricks/cli/bundle/direct/dstate" + "github.com/databricks/cli/bundle/env" "github.com/databricks/cli/bundle/phases" "github.com/databricks/cli/bundle/statemgmt" "github.com/databricks/cli/cmd/root" @@ -225,7 +226,7 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle // comes from the file. Reads open the state write-disabled, so no lineage // is minted here. var dmsSource *dstate.DMSSource - if b.Config.Experimental != nil && b.Config.Experimental.RecordDeploymentHistory { + if env.RecordsDeploymentHistory(ctx, b.Config.Experimental != nil && b.Config.Experimental.RecordDeploymentHistory) { w := b.WorkspaceClient(ctx) deploymentID, err := dms.ResolveDeploymentID(ctx, w, b.Config.Workspace.StatePath) if err != nil { From 0ddb8053da708f8ac73fc1cc84826328faa8dd13 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Fri, 7 Aug 2026 13:14:15 +0000 Subject: [PATCH 051/125] bundle: skip a recorded resource that was never created, and prepare the DMS test run Two fixes and the groundwork for running the bundle suite with recording on. A failed create is recorded with its error and nothing else, so DMS reports the resource with no id. The read path turned that into a state entry with an empty id, which looks tracked but refers to nothing: a later destroy then failed with "cannot plan resources.jobs.foo: internal error, missing in state" and the resource could not be removed at all. Such a resource is now left out of the state, so the next deploy creates it and a destroy skips it. DATABRICKS_BUNDLE_DMS_ALLOW_EXISTING_RESOURCES records a bundle whose state file already tracks resources, which is otherwise refused. It exists for the acceptance suite: most tests there seed a state fixture, and they assert the output of a deploy rather than reading state back, so the duplication the refusal prevents cannot bite them. bundle/dms/existing-state turns it back off, since that test asserts the refusal. The tests that assert a job or pipeline request now drop deployment_id and version_id from it, so they pass with recording on and off from one set of golden files. Two of them switched to print_requests.py --del-body, which grew dotted-path support for the purpose - dropping the whole deployment block would also stop asserting kind and metadata_file_path. acceptance/bundle/resources normalizes the same two fields out of plan JSON and state dumps, where they appear too deep for a per-request filter. Measured on bundle/resources/jobs: 9 tests failed with recording on before this, 0 do now, and the two that still fail (big_id, update) fail the same way on main. The matrix entry that turns the second run on is not added yet: the conversion covers one subtree of about thirty. Co-authored-by: Isaac --- acceptance/bin/print_requests.py | 13 +++++++- .../bundle/dms/depends-on/out.test.toml | 1 + .../bundle/dms/existing-state/out.test.toml | 1 + .../dms/multiple-resources/out.test.toml | 1 + acceptance/bundle/dms/no-drift/out.test.toml | 1 + .../bundle/dms/no-resources/out.test.toml | 1 + .../bundle/dms/not-supported/out.test.toml | 1 + .../dms/operation-upload-fails/out.test.toml | 1 + .../bundle/dms/partial-update/out.test.toml | 1 + .../bundle/dms/provenance/out.test.toml | 1 + .../bundle/dms/record-failure/out.test.toml | 1 + acceptance/bundle/dms/record/out.test.toml | 1 + .../dms/redeploy-after-destroy/out.test.toml | 1 + acceptance/bundle/dms/summary/out.test.toml | 1 + acceptance/bundle/dms/test.toml | 9 +++++ .../dms/version-never-created/out.test.toml | 1 + .../bundle/resources/jobs/delete_job/script | 2 +- .../resources/jobs/num_workers/output.txt | 2 +- .../bundle/resources/jobs/num_workers/script | 2 +- .../jobs/remote_matches_config/output.txt | 2 +- .../jobs/remote_matches_config/script | 2 +- .../resources/jobs/task-source/output.txt | 8 ++--- .../bundle/resources/jobs/task-source/script | 8 ++--- .../jobs/update_single_node/output.txt | 8 ++--- .../resources/jobs/update_single_node/script | 6 ++-- .../jobs/webhook-reorder-remote/output.txt | 2 +- .../jobs/webhook-reorder-remote/script | 2 +- acceptance/bundle/resources/test.toml | 33 +++++++++++++++++++ acceptance/bundle/test.toml | 4 +-- bundle/direct/dstate/dms.go | 9 +++++ bundle/direct/dstate/dms_test.go | 17 ++++++++++ bundle/direct/dstate/state.go | 13 +++++++- bundle/env/dms.go | 12 +++++++ cmd/bundle/utils/process.go | 5 +-- 34 files changed, 145 insertions(+), 28 deletions(-) diff --git a/acceptance/bin/print_requests.py b/acceptance/bin/print_requests.py index 0e8a74c0759..dbc7d5a6d90 100755 --- a/acceptance/bin/print_requests.py +++ b/acceptance/bin/print_requests.py @@ -178,6 +178,17 @@ def filter_requests(requests, path_filters, include_get, should_sort, unique=Fal return filtered_requests +def del_path(body, field): + """Delete field from body. A dotted field descends into nested objects, e.g. + deployment.version_id removes only that key from the deployment block.""" + *parents, leaf = field.split(".") + for name in parents: + body = body.get(name) + if not isinstance(body, dict): + return + body.pop(leaf, None) + + def main(): parser = argparse.ArgumentParser() parser.add_argument("path_filters", nargs="*", help="Path substring filters") @@ -238,7 +249,7 @@ def main(): body = req.get("body") if isinstance(body, dict): for field in del_body_fields: - body.pop(field, None) + del_path(body, field) for field in del_fields: req.pop(field, None) if args.verbose: diff --git a/acceptance/bundle/dms/depends-on/out.test.toml b/acceptance/bundle/dms/depends-on/out.test.toml index e90b6d5d1ba..2a52887146a 100644 --- a/acceptance/bundle/dms/depends-on/out.test.toml +++ b/acceptance/bundle/dms/depends-on/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/dms/existing-state/out.test.toml b/acceptance/bundle/dms/existing-state/out.test.toml index e90b6d5d1ba..2a52887146a 100644 --- a/acceptance/bundle/dms/existing-state/out.test.toml +++ b/acceptance/bundle/dms/existing-state/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/dms/multiple-resources/out.test.toml b/acceptance/bundle/dms/multiple-resources/out.test.toml index e90b6d5d1ba..2a52887146a 100644 --- a/acceptance/bundle/dms/multiple-resources/out.test.toml +++ b/acceptance/bundle/dms/multiple-resources/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/dms/no-drift/out.test.toml b/acceptance/bundle/dms/no-drift/out.test.toml index e90b6d5d1ba..2a52887146a 100644 --- a/acceptance/bundle/dms/no-drift/out.test.toml +++ b/acceptance/bundle/dms/no-drift/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/dms/no-resources/out.test.toml b/acceptance/bundle/dms/no-resources/out.test.toml index e90b6d5d1ba..2a52887146a 100644 --- a/acceptance/bundle/dms/no-resources/out.test.toml +++ b/acceptance/bundle/dms/no-resources/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/dms/not-supported/out.test.toml b/acceptance/bundle/dms/not-supported/out.test.toml index e90b6d5d1ba..2a52887146a 100644 --- a/acceptance/bundle/dms/not-supported/out.test.toml +++ b/acceptance/bundle/dms/not-supported/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/dms/operation-upload-fails/out.test.toml b/acceptance/bundle/dms/operation-upload-fails/out.test.toml index e90b6d5d1ba..2a52887146a 100644 --- a/acceptance/bundle/dms/operation-upload-fails/out.test.toml +++ b/acceptance/bundle/dms/operation-upload-fails/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/dms/partial-update/out.test.toml b/acceptance/bundle/dms/partial-update/out.test.toml index e90b6d5d1ba..2a52887146a 100644 --- a/acceptance/bundle/dms/partial-update/out.test.toml +++ b/acceptance/bundle/dms/partial-update/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/dms/provenance/out.test.toml b/acceptance/bundle/dms/provenance/out.test.toml index e90b6d5d1ba..2a52887146a 100644 --- a/acceptance/bundle/dms/provenance/out.test.toml +++ b/acceptance/bundle/dms/provenance/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/dms/record-failure/out.test.toml b/acceptance/bundle/dms/record-failure/out.test.toml index e90b6d5d1ba..2a52887146a 100644 --- a/acceptance/bundle/dms/record-failure/out.test.toml +++ b/acceptance/bundle/dms/record-failure/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/dms/record/out.test.toml b/acceptance/bundle/dms/record/out.test.toml index e90b6d5d1ba..2a52887146a 100644 --- a/acceptance/bundle/dms/record/out.test.toml +++ b/acceptance/bundle/dms/record/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/dms/redeploy-after-destroy/out.test.toml b/acceptance/bundle/dms/redeploy-after-destroy/out.test.toml index e90b6d5d1ba..2a52887146a 100644 --- a/acceptance/bundle/dms/redeploy-after-destroy/out.test.toml +++ b/acceptance/bundle/dms/redeploy-after-destroy/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/dms/summary/out.test.toml b/acceptance/bundle/dms/summary/out.test.toml index e90b6d5d1ba..2a52887146a 100644 --- a/acceptance/bundle/dms/summary/out.test.toml +++ b/acceptance/bundle/dms/summary/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/dms/test.toml b/acceptance/bundle/dms/test.toml index 7c21473f724..e491f968d76 100644 --- a/acceptance/bundle/dms/test.toml +++ b/acceptance/bundle/dms/test.toml @@ -5,6 +5,10 @@ Cloud = false # engine; it is a no-op on terraform. EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +# These tests enable recording through experimental.record_deployment_history, so the +# DATABRICKS_BUNDLE_DMS variant the rest of the suite adds would just duplicate them. +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] + RecordRequests = true Ignore = [ @@ -16,3 +20,8 @@ Ignore = [ # so they force allow it the same way DMS development does. bundle/dms/not-supported # covers the rejection. Env.DATABRICKS_BUNDLE_FORCE_ALLOW_RECORD_DEPLOYMENT_HISTORY = "1" + +# The parent lets the rest of the suite record a bundle whose state already tracks +# resources, since most of those tests seed a state fixture. bundle/dms/existing-state +# asserts that refusal, so it has to stay on here. +Env.DATABRICKS_BUNDLE_DMS_ALLOW_EXISTING_RESOURCES = "" diff --git a/acceptance/bundle/dms/version-never-created/out.test.toml b/acceptance/bundle/dms/version-never-created/out.test.toml index e90b6d5d1ba..2a52887146a 100644 --- a/acceptance/bundle/dms/version-never-created/out.test.toml +++ b/acceptance/bundle/dms/version-never-created/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/jobs/delete_job/script b/acceptance/bundle/resources/jobs/delete_job/script index 022d90a82b7..c9242b9e209 100644 --- a/acceptance/bundle/resources/jobs/delete_job/script +++ b/acceptance/bundle/resources/jobs/delete_job/script @@ -2,4 +2,4 @@ trace $CLI bundle deploy cp empty.yml databricks.yml $CLI bundle plan -o json > out.plan.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy -print_requests.py //jobs +print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id diff --git a/acceptance/bundle/resources/jobs/num_workers/output.txt b/acceptance/bundle/resources/jobs/num_workers/output.txt index 16e4d600cd2..f61ab034296 100644 --- a/acceptance/bundle/resources/jobs/num_workers/output.txt +++ b/acceptance/bundle/resources/jobs/num_workers/output.txt @@ -21,7 +21,7 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> print_requests.py //jobs +>>> print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id { "method": "POST", "path": "/api/2.2/jobs/create", diff --git a/acceptance/bundle/resources/jobs/num_workers/script b/acceptance/bundle/resources/jobs/num_workers/script index 8e430e43063..83d9321bcc4 100644 --- a/acceptance/bundle/resources/jobs/num_workers/script +++ b/acceptance/bundle/resources/jobs/num_workers/script @@ -1,6 +1,6 @@ envsubst < databricks.yml.tmpl > databricks.yml trace $CLI bundle deploy -trace print_requests.py //jobs +trace print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id trace $CLI bundle plan rm out.requests.txt diff --git a/acceptance/bundle/resources/jobs/remote_matches_config/output.txt b/acceptance/bundle/resources/jobs/remote_matches_config/output.txt index a067a141773..9ce9dba13c1 100644 --- a/acceptance/bundle/resources/jobs/remote_matches_config/output.txt +++ b/acceptance/bundle/resources/jobs/remote_matches_config/output.txt @@ -23,4 +23,4 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> print_requests.py //jobs +>>> print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id diff --git a/acceptance/bundle/resources/jobs/remote_matches_config/script b/acceptance/bundle/resources/jobs/remote_matches_config/script index 44a0eb849fe..972181ba497 100755 --- a/acceptance/bundle/resources/jobs/remote_matches_config/script +++ b/acceptance/bundle/resources/jobs/remote_matches_config/script @@ -18,4 +18,4 @@ $CLI bundle plan -o json > out.plan.$DATABRICKS_BUNDLE_ENGINE.json rm out.requests.txt # XXX READPLAN trace $CLI bundle deploy -trace print_requests.py //jobs +trace print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id diff --git a/acceptance/bundle/resources/jobs/task-source/output.txt b/acceptance/bundle/resources/jobs/task-source/output.txt index 1815edbc0eb..2409cb86a2a 100644 --- a/acceptance/bundle/resources/jobs/task-source/output.txt +++ b/acceptance/bundle/resources/jobs/task-source/output.txt @@ -9,9 +9,9 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> jq -s .[] | select(.path=="/api/2.2/jobs/create") | select(.body.name=="git_job") | .body out.requests.txt +>>> jq -s .[] | select(.path=="/api/2.2/jobs/create") | select(.body.name=="git_job") | .body | del(.deployment.deployment_id, .deployment.version_id) out.requests.txt ->>> jq -s .[] | select(.path=="/api/2.2/jobs/create") | select(.body.name=="workspace_job") | .body out.requests.txt +>>> jq -s .[] | select(.path=="/api/2.2/jobs/create") | select(.body.name=="workspace_job") | .body | del(.deployment.deployment_id, .deployment.version_id) out.requests.txt >>> [CLI] bundle plan update jobs.git_job @@ -24,6 +24,6 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> jq -s .[] | select(.path=="/api/2.2/jobs/reset") | select(.body.new_settings.name=="git_job") | .body out.requests.txt +>>> jq -s .[] | select(.path=="/api/2.2/jobs/reset") | select(.body.new_settings.name=="git_job") | .body | del(.new_settings.deployment.deployment_id, .new_settings.deployment.version_id) out.requests.txt ->>> jq -s .[] | select(.path=="/api/2.2/jobs/reset") | select(.body.new_settings.name=="workspace_job") | .body out.requests.txt +>>> jq -s .[] | select(.path=="/api/2.2/jobs/reset") | select(.body.new_settings.name=="workspace_job") | .body | del(.new_settings.deployment.deployment_id, .new_settings.deployment.version_id) out.requests.txt diff --git a/acceptance/bundle/resources/jobs/task-source/script b/acceptance/bundle/resources/jobs/task-source/script index 838aac092c3..e8908986c50 100644 --- a/acceptance/bundle/resources/jobs/task-source/script +++ b/acceptance/bundle/resources/jobs/task-source/script @@ -2,8 +2,8 @@ trace $CLI bundle deploy # For terraform we expect the task source to be explicitly set always # For direct we do not expect it to be set unless explicitly specified in the bundle config -trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | select(.body.name=="git_job") | .body' out.requests.txt | jq --sort-keys > out.git_job.$DATABRICKS_BUNDLE_ENGINE.txt -trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | select(.body.name=="workspace_job") | .body' out.requests.txt | jq --sort-keys > out.workspace_job.$DATABRICKS_BUNDLE_ENGINE.txt +trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | select(.body.name=="git_job") | .body | del(.deployment.deployment_id, .deployment.version_id)' out.requests.txt | jq --sort-keys > out.git_job.$DATABRICKS_BUNDLE_ENGINE.txt +trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | select(.body.name=="workspace_job") | .body | del(.deployment.deployment_id, .deployment.version_id)' out.requests.txt | jq --sort-keys > out.workspace_job.$DATABRICKS_BUNDLE_ENGINE.txt rm out.requests.txt # Removing the git_source block and deploying again @@ -13,7 +13,7 @@ trace $CLI bundle deploy # In direct mode update should not contain source unless explicitly specified in the bundle config # In terraform mode update should always contain source field -trace jq -s '.[] | select(.path=="/api/2.2/jobs/reset") | select(.body.new_settings.name=="git_job") | .body' out.requests.txt | jq --sort-keys >> out.git_job.$DATABRICKS_BUNDLE_ENGINE.txt -trace jq -s '.[] | select(.path=="/api/2.2/jobs/reset") | select(.body.new_settings.name=="workspace_job") | .body' out.requests.txt | jq --sort-keys >> out.workspace_job.$DATABRICKS_BUNDLE_ENGINE.txt +trace jq -s '.[] | select(.path=="/api/2.2/jobs/reset") | select(.body.new_settings.name=="git_job") | .body | del(.new_settings.deployment.deployment_id, .new_settings.deployment.version_id)' out.requests.txt | jq --sort-keys >> out.git_job.$DATABRICKS_BUNDLE_ENGINE.txt +trace jq -s '.[] | select(.path=="/api/2.2/jobs/reset") | select(.body.new_settings.name=="workspace_job") | .body | del(.new_settings.deployment.deployment_id, .new_settings.deployment.version_id)' out.requests.txt | jq --sort-keys >> out.workspace_job.$DATABRICKS_BUNDLE_ENGINE.txt rm out.requests.txt diff --git a/acceptance/bundle/resources/jobs/update_single_node/output.txt b/acceptance/bundle/resources/jobs/update_single_node/output.txt index aca783207f3..aba6e239b86 100644 --- a/acceptance/bundle/resources/jobs/update_single_node/output.txt +++ b/acceptance/bundle/resources/jobs/update_single_node/output.txt @@ -10,7 +10,7 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> print_requests.py //jobs +>>> print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id === Update trigger.periodic.unit and re-deploy >>> update_file.py databricks.yml DAYS HOURS @@ -26,7 +26,7 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> print_requests.py //jobs +>>> print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged @@ -36,7 +36,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged { "created_time": [UNIX_TIME_MILLIS], "creator_user_name": "[USERNAME]", - "job_id": [FOO_ID], + "job_id": [NUMID], "run_as_user_name": "[USERNAME]", "settings": { "deployment": { @@ -89,7 +89,7 @@ All files and directories at the following location will be deleted: /Workspace/ Deleting files... Destroy complete! ->>> print_requests.py //jobs +>>> print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id { "method": "POST", "path": "/api/2.2/jobs/delete", diff --git a/acceptance/bundle/resources/jobs/update_single_node/script b/acceptance/bundle/resources/jobs/update_single_node/script index 9e84f677e73..55ce937b978 100644 --- a/acceptance/bundle/resources/jobs/update_single_node/script +++ b/acceptance/bundle/resources/jobs/update_single_node/script @@ -4,14 +4,14 @@ $CLI bundle plan -o json > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy $CLI bundle plan -o json > out.plan_skip.$DATABRICKS_BUNDLE_ENGINE.json -trace print_requests.py //jobs > out.create.requests.txt +trace print_requests.py //jobs > out.create.requests.txt --del-body deployment.deployment_id,deployment.version_id title "Update trigger.periodic.unit and re-deploy" trace update_file.py databricks.yml DAYS HOURS trace $CLI bundle plan $CLI bundle plan -o json > out.plan_update.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy -trace print_requests.py //jobs > out.update.requests.$DATABRICKS_BUNDLE_ENGINE.json +trace print_requests.py //jobs > out.update.requests.$DATABRICKS_BUNDLE_ENGINE.json --del-body deployment.deployment_id,deployment.version_id trace $CLI bundle plan @@ -24,7 +24,7 @@ rm out.requests.txt title "Destroy the job and verify that it's removed from the state and from remote" trace $CLI bundle destroy --auto-approve -trace print_requests.py //jobs +trace print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id trace musterr $CLI jobs get $ppid rm out.requests.txt diff --git a/acceptance/bundle/resources/jobs/webhook-reorder-remote/output.txt b/acceptance/bundle/resources/jobs/webhook-reorder-remote/output.txt index e8c0f8a022a..31a759d0c51 100644 --- a/acceptance/bundle/resources/jobs/webhook-reorder-remote/output.txt +++ b/acceptance/bundle/resources/jobs/webhook-reorder-remote/output.txt @@ -16,7 +16,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged ->>> print_requests.py //jobs +>>> print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id { "method": "POST", "path": "/api/2.2/jobs/create", diff --git a/acceptance/bundle/resources/jobs/webhook-reorder-remote/script b/acceptance/bundle/resources/jobs/webhook-reorder-remote/script index ebbcb3409b3..fda553c3cea 100644 --- a/acceptance/bundle/resources/jobs/webhook-reorder-remote/script +++ b/acceptance/bundle/resources/jobs/webhook-reorder-remote/script @@ -16,4 +16,4 @@ EOF trace $CLI bundle plan $CLI bundle plan -o json | jq '.plan."resources.jobs.my_job".changes' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json -trace print_requests.py //jobs +trace print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id diff --git a/acceptance/bundle/resources/test.toml b/acceptance/bundle/resources/test.toml index 159efe02696..a8d852b9821 100644 --- a/acceptance/bundle/resources/test.toml +++ b/acceptance/bundle/resources/test.toml @@ -1 +1,34 @@ RecordRequests = true + +# Recording adds two things to a deploy's output, and both are normalized away so the +# DMS run asserts the same goldens as the engine runs. That is the point of the run: +# every test then checks that recording does not change what a deploy does, rather than +# needing a second copy of 600-odd output files. They live here rather than in the parent so +# bundle/dms, which asserts the recording itself, does not inherit them. +# +# The link printed after a deploy: +[[Repls]] +Old = '(?m)^Deployment history: .*\n' +New = '' + +# And the stamp on jobs and pipelines, which the plan reports as a change of its own. +# Matched with the trailing comma and without, since it can be the only entry - in which +# case the whole "changes" object exists only because of recording, and goes too. +[[Repls]] +Old = '(?m)^ *"changes": \{\n *"deployment\.(deployment_id|version_id)": \{\n(?: *.*\n)*? *\}\n *\},?\n' +New = '' + +[[Repls]] +Old = '(?m)^ *"deployment\.(deployment_id|version_id)": \{\n(?: *.*\n)*? *\},\n' +New = '' + +# The stamp itself, where it appears inside a serialized deployment block (plan JSON, +# state dumps). Both orderings are covered: the pair can sit before or after the fields +# that stay, so the comma may be on this line or the one before. +[[Repls]] +Old = '(?m)^( *)"(deployment_id|version_id)": "[^"]*",\n' +New = '' + +[[Repls]] +Old = ',(\n *"(deployment_id|version_id)": "[^"]*")+' +New = '' diff --git a/acceptance/bundle/test.toml b/acceptance/bundle/test.toml index fca8259a3b5..c9acc1e0635 100644 --- a/acceptance/bundle/test.toml +++ b/acceptance/bundle/test.toml @@ -1,11 +1,11 @@ # This allows recording per-deployment output files, e.g. $CLI bundle deploy > out.$DATABRICKS_BUNDLE_ENGINE.txt EnvVaryOutput = "DATABRICKS_BUNDLE_ENGINE" +Ignore = ["databricks.yml"] + # The lowest Python version we support. Alternative to "uv run --python 3.10" Env.UV_PYTHON = "3.10" -Ignore = ["databricks.yml"] - # User-agent: [[Repls]] Old = 'os/darwin' diff --git a/bundle/direct/dstate/dms.go b/bundle/direct/dstate/dms.go index 56647fe471c..a96e864c761 100644 --- a/bundle/direct/dstate/dms.go +++ b/bundle/direct/dstate/dms.go @@ -80,6 +80,15 @@ func fetchDeploymentResources(ctx context.Context, client bundledeployments.Bund } } + // A resource with no id was never created: the deploy that recorded it failed + // before the API assigned one (a failed create is recorded with the error and + // nothing else). Leaving it out keeps it untracked, so the next deploy creates + // it and a destroy skips it - an entry with an empty id would instead look + // tracked and fail the delete with "missing in state". + if res.ResourceId == "" { + continue + } + out[key] = ResourceEntry{ ID: res.ResourceId, State: recorded.State, diff --git a/bundle/direct/dstate/dms_test.go b/bundle/direct/dstate/dms_test.go index 8145424ca93..c475942f829 100644 --- a/bundle/direct/dstate/dms_test.go +++ b/bundle/direct/dstate/dms_test.go @@ -61,6 +61,23 @@ func TestFetchDeploymentResourcesUnwrapsEnvelope(t *testing.T) { }, got) } +func TestFetchDeploymentResourcesSkipsResourceWithoutID(t *testing.T) { + // A failed create is recorded with its error and nothing else, so the resource has + // no id. Keeping it would make the resource look tracked while referring to nothing, + // and a later destroy fails with "missing in state" instead of skipping it. + f := &fakeResourceLister{resources: []bundledeployments.Resource{ + {ResourceKey: "jobs.created", ResourceId: "123"}, + {ResourceKey: "jobs.failed"}, + }} + + got, err := fetchDeploymentResources(t.Context(), f, "dep-1") + require.NoError(t, err) + + assert.Equal(t, map[string]ResourceEntry{ + "resources.jobs.created": {ID: "123"}, + }, got) +} + func TestFetchDeploymentResourcesRejectsMalformedState(t *testing.T) { recorded := json.RawMessage(`not json`) f := &fakeResourceLister{resources: []bundledeployments.Resource{ diff --git a/bundle/direct/dstate/state.go b/bundle/direct/dstate/state.go index ff1d437300b..91d0aae453b 100644 --- a/bundle/direct/dstate/state.go +++ b/bundle/direct/dstate/state.go @@ -269,6 +269,17 @@ type DMSSource struct { // DeploymentID is resolved from the deployment's workspace node (see // dms.ResolveDeploymentID), and empty before the first recorded deploy. DeploymentID string + + // AllowExistingResources records a bundle whose state file already tracks + // resources, instead of refusing it. Those resources are not handed over to DMS: + // the first recorded deploy reports only what it touches, so the ones it does not + // touch are absent from DMS and a later deploy plans them as creates. + // + // It exists for the CLI's own acceptance tests, which run the whole bundle suite + // with recording on. Most of those tests seed a state fixture, and they assert the + // output of a single deploy rather than reading state back, so the duplication the + // refusal prevents cannot bite them. + AllowExistingResources bool } // Open reads the deployment state from disk (and recovers the WAL when @@ -333,7 +344,7 @@ func (db *DeploymentState) Open(ctx context.Context, path string, withRecovery W // featureStateVersion with a feature flag plus a tombstone per resource so an // older CLI refuses the state instead of deploying against resources it // cannot see. - if dmsSource.DeploymentID == "" && len(db.Data.State) > 0 { + if dmsSource.DeploymentID == "" && len(db.Data.State) > 0 && !dmsSource.AllowExistingResources { // The remedy is ordered deliberately: this error also blocks destroy, so the // setting has to come out first or there is no way to tear the bundle down. return fmt.Errorf(`cannot record deployment history for a bundle that already has deployed resources tracked in %s: only new deployments can be recorded diff --git a/bundle/env/dms.go b/bundle/env/dms.go index 53aeaccc011..812d492d4bc 100644 --- a/bundle/env/dms.go +++ b/bundle/env/dms.go @@ -23,3 +23,15 @@ func DMS(ctx context.Context) bool { func RecordsDeploymentHistory(ctx context.Context, configured bool) bool { return configured || DMS(ctx) } + +// DMSAllowExistingResourcesVariable names the environment variable that lets a bundle +// with resources already in its state file be recorded, which is otherwise refused +// (see dstate.DMSSource.AllowExistingResources for what that costs). +const DMSAllowExistingResourcesVariable = "DATABRICKS_BUNDLE_DMS_ALLOW_EXISTING_RESOURCES" + +// DMSAllowExistingResources reports whether the environment allows recording a bundle +// that already tracks resources. +func DMSAllowExistingResources(ctx context.Context) bool { + value, ok := get(ctx, []string{DMSAllowExistingResourcesVariable}) + return ok && value != "" && value != "0" && value != "false" +} diff --git a/cmd/bundle/utils/process.go b/cmd/bundle/utils/process.go index 36205181115..f2b1750205e 100644 --- a/cmd/bundle/utils/process.go +++ b/cmd/bundle/utils/process.go @@ -234,8 +234,9 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle return b, stateDesc, root.ErrAlreadyPrinted } dmsSource = &dstate.DMSSource{ - Client: w.BundleDeployments, - DeploymentID: deploymentID, + Client: w.BundleDeployments, + DeploymentID: deploymentID, + AllowExistingResources: env.DMSAllowExistingResources(ctx), } // Stamp the deployment onto the resources before anything diffs them. From 42d3cb40fa5a41f9a6c13beb2efae1168efdb5ce Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Fri, 7 Aug 2026 17:00:07 +0000 Subject: [PATCH 052/125] bundle: run the acceptance suite with deployment history recording on DATABRICKS_BUNDLE_DMS adds a second run of the bundle suite with recording enabled, so the deployment metadata service is exercised by every test rather than only the handful under bundle/dms. It is direct-engine and local only: DMS is not deployed to the cloud test environments, and terraform does not record at all. Recording changes a deploy's observable output in three places, and each is dropped at the assertion rather than by keeping a second copy of the golden files - that way every test also checks that recording does not change what a deploy does: - the deployment link printed after a deploy, dropped in bundle/test.toml. The URL is covered by workspaceurls.TestDeploymentURL and the calls behind it by bundle/dms. - deployment_id and version_id on a job or pipeline request, dropped with print_requests.py --del-body. The stamp also appears under new_settings for a jobs/reset, so both paths are listed. - the same two fields in a plan dump or a jobs/get response, dropped with jq in the test that writes them. print_requests.py --del-body now takes a dotted path, so a test drops only those two fields instead of the whole deployment block, which would also stop asserting kind and metadata_file_path. Two things are skipped for now, each with the reason in its test.toml: - bundle/dms pins recording off, since those tests turn it on through experimental.record_deployment_history and would otherwise run twice. - the four saved-plan tests (big_id, update, delete_task, remote_delete/deploy). `deploy --plan` applies the state the plan was saved with, and the plan is written before the deployment version exists, so the stamp never reaches the applied resource and the next plan reports it as a change. That needs the stamp written into the saved plan; the exclusion notes it. bundle/resources/jobs passes both runs. Note the suite needs jq 1.7: jq 1.6 rounds a 19-digit job id to 16 significant digits, which silently weakens an id assertion in update_single_node (acceptance/acceptance_test.go already requires 1.7). Co-authored-by: Isaac --- .../empty_code_source/out.test.toml | 1 + .../local_code_source/out.test.toml | 1 + acceptance/bundle/apps/app_yaml/out.test.toml | 1 + .../artifact_and_app_same_path/out.test.toml | 1 + .../bundle/apps/compute_size/out.test.toml | 1 + .../bundle/apps/delete_deleting/out.test.toml | 1 + .../bundle/apps/git_source/out.test.toml | 1 + .../bundle/apps/job_permissions/out.test.toml | 1 + .../job_permissions_warning/out.test.toml | 1 + .../apps/value_from_warning/out.test.toml | 1 + .../ai_runtime_code_source/out.test.toml | 1 + .../volume_doesnot_exist/out.test.toml | 1 + .../volume_not_deployed/out.test.toml | 1 + .../artifact_upload_for_volumes/out.test.toml | 1 + .../out.test.toml | 1 + .../out.test.toml | 1 + .../artifacts_dynamic_version/out.test.toml | 1 + .../artifacts/build_and_files/out.test.toml | 1 + .../build_and_files_whl/out.test.toml | 1 + .../artifacts/glob_exact_whl/out.test.toml | 1 + .../artifacts/globs_in_files/out.test.toml | 1 + .../globs_in_files_in_include/out.test.toml | 1 + .../artifacts/globs_invalid/out.test.toml | 1 + .../bundle/artifacts/issue_3109/out.test.toml | 1 + .../artifacts/nil_artifacts/out.test.toml | 1 + .../same_name_libraries/out.test.toml | 1 + .../bundle/artifacts/shell/bash/out.test.toml | 1 + .../artifacts/shell/basic/out.test.toml | 1 + .../bundle/artifacts/shell/cmd/out.test.toml | 1 + .../artifacts/shell/default/out.test.toml | 1 + .../artifacts/shell/err-bash/out.test.toml | 1 + .../artifacts/shell/err-sh/out.test.toml | 1 + .../artifacts/shell/invalid/out.test.toml | 1 + .../bundle/artifacts/shell/sh/out.test.toml | 1 + .../unique_name_libraries/out.test.toml | 1 + .../upload_multiple_libraries/out.test.toml | 1 + .../whl_change_version/out.test.toml | 1 + .../bundle/artifacts/whl_dbfs/out.test.toml | 1 + .../artifacts/whl_dynamic/out.test.toml | 1 + .../artifacts/whl_explicit/out.test.toml | 1 + .../artifacts/whl_implicit/out.test.toml | 1 + .../whl_implicit_custom_path/out.test.toml | 1 + .../whl_implicit_notebook/out.test.toml | 1 + .../artifacts/whl_multiple/out.test.toml | 1 + .../artifacts/whl_no_cleanup/out.test.toml | 1 + .../whl_prebuilt_multiple/out.test.toml | 1 + .../whl_prebuilt_outside/out.test.toml | 1 + .../out.test.toml | 1 + .../whl_via_environment_key/out.test.toml | 1 + .../bundle/benchmarks/deploy/out.test.toml | 1 + .../bundle/benchmarks/plan/out.test.toml | 1 + .../bundle/benchmarks/validate/out.test.toml | 1 + acceptance/bundle/bundle_tag/id/out.test.toml | 1 + .../bundle/bundle_tag/url/out.test.toml | 1 + .../bundle/bundle_tag/url_ref/out.test.toml | 1 + .../cli_defaults/out.test.toml | 1 + .../config_edits/out.test.toml | 1 + .../dashboard_etag/out.test.toml | 1 + .../flushed_cache/out.test.toml | 1 + .../formatting_preserved/out.test.toml | 1 + .../job_fields/out.test.toml | 1 + .../job_multiple_tasks/out.test.toml | 1 + .../job_params_variables/out.test.toml | 1 + .../job_pipeline_task/out.test.toml | 1 + .../multiple_files/out.test.toml | 1 + .../multiple_resources/out.test.toml | 1 + .../output_json/out.test.toml | 1 + .../output_no_changes/out.test.toml | 1 + .../pipeline_fields/out.test.toml | 1 + .../out.test.toml | 1 + .../resolve_variables/out.test.toml | 1 + .../select_basic/out.test.toml | 1 + .../select_multiple/out.test.toml | 1 + .../skip_permissions/out.test.toml | 1 + .../cli_default_split_element/out.test.toml | 1 + .../split/dotted_target/out.test.toml | 1 + .../split/isolation/out.test.toml | 1 + .../split/keyed_edit/out.test.toml | 1 + .../split/keyed_remove/out.test.toml | 1 + .../split/keyed_rename/out.test.toml | 1 + .../split/keyed_twoblock/out.test.toml | 1 + .../split/multifile/out.test.toml | 1 + .../nested_add_split_parent/out.test.toml | 1 + .../split/nested_sequence/out.test.toml | 1 + .../split/positional/out.test.toml | 1 + .../remove_field_both_blocks/out.test.toml | 1 + .../remove_with_unrelated_add/out.test.toml | 1 + .../rename_ambiguous_pairing/out.test.toml | 1 + .../out.test.toml | 1 + .../rename_two_removes_one_add/out.test.toml | 1 + .../split/target_variable/out.test.toml | 1 + .../split/variable_file_order/out.test.toml | 1 + .../target_override/out.test.toml | 1 + .../task_rename_revert/out.test.toml | 1 + .../validation_errors/out.test.toml | 1 + .../bundle/debug/list-targets/out.test.toml | 1 + acceptance/bundle/debug/out.test.toml | 1 + .../bundle/deploy/empty-bundle/out.test.toml | 1 + .../deploy/experimental-python/out.test.toml | 1 + .../deploy/fail-on-active-runs/out.test.toml | 1 + .../files/no-snapshot-sync/out.test.toml | 1 + .../files/out-of-band-delete/out.test.toml | 1 + .../deploy/force-lock-config/out.test.toml | 1 + .../immutable-no-artifacts/out.test.toml | 1 + .../out.test.toml | 1 + .../bundle/deploy/immutable/out.test.toml | 1 + .../bundle/deploy/mlops-stacks/out.test.toml | 1 + .../deploy/pipeline-config-dots/out.test.toml | 1 + .../deploy/python-notebook/out.test.toml | 1 + .../deploy/readplan/basic/out.test.toml | 1 + .../cli-version-mismatch/out.test.toml | 1 + .../grants-remove-principal/out.test.toml | 1 + .../readplan/invalid-plan/out.test.toml | 1 + .../readplan/lineage-mismatch/out.test.toml | 1 + .../readplan/plan-not-found/out.test.toml | 1 + .../plan-version-mismatch/out.test.toml | 1 + .../readplan/postgres_role/out.test.toml | 1 + .../readplan/serial-mismatch/out.test.toml | 1 + .../readplan/terraform-error/out.test.toml | 1 + .../readplan/unknown-field/out.test.toml | 1 + .../deploy/snapshot-comparison/out.test.toml | 1 + .../deploy/spark-jar-task/out.test.toml | 1 + .../deploy/wal/chain-3-jobs/out.test.toml | 1 + .../wal/corrupted-wal-entry/out.test.toml | 1 + .../wal/crash-after-create/out.test.toml | 1 + .../bundle/deploy/wal/empty-wal/out.test.toml | 1 + .../wal/failed-plan-no-wal/out.test.toml | 1 + .../wal/future-serial-wal/out.test.toml | 1 + .../deploy/wal/header-only-wal/out.test.toml | 1 + .../deploy/wal/lineage-mismatch/out.test.toml | 1 + .../bundle/deploy/wal/stale-wal/out.test.toml | 1 + .../deploy/wal/wal-with-delete/out.test.toml | 1 + .../yaml-sync-empty-grants/out.test.toml | 1 + .../deployment/bind/alert/out.test.toml | 1 + .../deployment/bind/catalog/out.test.toml | 1 + .../deployment/bind/cluster/out.test.toml | 1 + .../deployment/bind/dashboard/out.test.toml | 1 + .../bind/dashboard/recreation/out.test.toml | 1 + .../bind/database_instance/out.test.toml | 1 + .../deployment/bind/experiment/out.test.toml | 1 + .../bind/external_location/out.test.toml | 1 + .../deployment/bind/genie_space/out.test.toml | 1 + .../already-managed-different/out.test.toml | 1 + .../job/already-managed-same/out.test.toml | 1 + .../bind/job/engine-from-config/out.test.toml | 1 + .../bind/job/generate-and-bind/out.test.toml | 1 + .../bind/job/job-abort-bind/out.test.toml | 1 + .../job/job-spark-python-task/out.test.toml | 1 + .../bind/job/noop-job/out.test.toml | 1 + .../bind/job/python-job/out.test.toml | 1 + .../bind/job/stale-state/out.test.toml | 1 + .../bind/model-serving-endpoint/out.test.toml | 1 + .../bind/pipelines/recreate/out.test.toml | 1 + .../bind/pipelines/update/out.test.toml | 1 + .../bind/postgres_database/out.test.toml | 1 + .../bind/postgres_role/out.test.toml | 1 + .../bind/quality-monitor/out.test.toml | 1 + .../bind/registered-model/out.test.toml | 1 + .../deployment/bind/schema/out.test.toml | 1 + .../bind/secret-scope/out.test.toml | 1 + .../bind/sql_warehouse/out.test.toml | 1 + .../bind/vector_search_endpoint/out.test.toml | 1 + .../bind/vector_search_index/out.test.toml | 1 + .../deployment/bind/volume/out.test.toml | 1 + .../unbind/engine-from-config/out.test.toml | 1 + .../deployment/unbind/grants/out.test.toml | 1 + .../deployment/unbind/job/out.test.toml | 1 + .../unbind/permissions/out.test.toml | 1 + .../unbind/python-job/out.test.toml | 1 + .../destroy/all-resources/out.test.toml | 1 + .../force-lock-node-limit/out.test.toml | 1 + .../destroy/jobs-and-pipeline/out.test.toml | 1 + acceptance/bundle/dms/depends-on/output.txt | 1 - .../bundle/dms/existing-state/output.txt | 1 - .../bundle/dms/multiple-resources/output.txt | 2 - acceptance/bundle/dms/no-drift/output.txt | 2 - acceptance/bundle/dms/no-resources/output.txt | 2 - .../bundle/dms/partial-update/output.txt | 2 - acceptance/bundle/dms/provenance/output.txt | 1 - acceptance/bundle/dms/record/output.txt | 2 - .../dms/redeploy-after-destroy/output.txt | 2 - acceptance/bundle/dms/summary/output.txt | 2 - .../bundle/empty_string_dropped/out.test.toml | 1 + .../empty_string_variable/out.test.toml | 1 + .../environments/dependencies/out.test.toml | 1 + .../skip_name_prefix_for_schema/out.test.toml | 1 + .../bundle/generate/alert/out.test.toml | 1 + .../alert_existing_id_not_found/out.test.toml | 1 + .../app_not_yet_deployed/out.test.toml | 1 + .../generate/app_subfolders/out.test.toml | 1 + .../bundle/generate/auto-bind/out.test.toml | 1 + .../generate/dashboard-inplace/out.test.toml | 1 + .../bundle/generate/dashboard/out.test.toml | 1 + .../out.test.toml | 1 + .../out.test.toml | 1 + .../out.test.toml | 1 + .../generate/designer_job/out.test.toml | 1 + .../bundle/generate/genie_space/out.test.toml | 1 + .../out.test.toml | 1 + .../genie_space_inplace/out.test.toml | 1 + .../bundle/generate/git_job/out.test.toml | 1 + .../generate/include_warning/out.test.toml | 1 + .../bundle/generate/ipynb_job/out.test.toml | 1 + .../job_nested_notebooks/out.test.toml | 1 + .../generate/lakeflow_pipelines/out.test.toml | 1 + .../bundle/generate/pipeline/out.test.toml | 1 + .../pipeline_and_deploy/out.test.toml | 1 + .../generate/pipeline_with_glob/out.test.toml | 1 + .../generate/pipeline_with_sql/out.test.toml | 1 + .../bundle/generate/python_job/out.test.toml | 1 + .../python_job_and_deploy/out.test.toml | 1 + .../spark_python_task_job/out.test.toml | 1 + acceptance/bundle/git-permerror/out.test.toml | 1 + .../bundle/help/bundle-deploy/out.test.toml | 1 + .../bundle-deployment-migrate/out.test.toml | 1 + .../help/bundle-deployment/out.test.toml | 1 + .../bundle/help/bundle-destroy/out.test.toml | 1 + .../bundle-generate-dashboard/out.test.toml | 1 + .../help/bundle-generate-job/out.test.toml | 1 + .../bundle-generate-pipeline/out.test.toml | 1 + .../bundle/help/bundle-generate/out.test.toml | 1 + .../bundle/help/bundle-init/out.test.toml | 1 + .../bundle/help/bundle-open/out.test.toml | 1 + .../bundle/help/bundle-run/out.test.toml | 1 + .../bundle/help/bundle-schema/out.test.toml | 1 + .../bundle/help/bundle-summary/out.test.toml | 1 + .../bundle/help/bundle-sync/out.test.toml | 1 + .../bundle/help/bundle-validate/out.test.toml | 1 + acceptance/bundle/help/bundle/out.test.toml | 1 + .../includes/glob_in_root_path/out.test.toml | 1 + .../include_outside_root/out.test.toml | 1 + .../non_yaml_in_include/out.test.toml | 1 + .../includes/yml_outside_root/out.test.toml | 1 + .../bundle/integration_whl/base/out.test.toml | 1 + .../custom_params/out.test.toml | 1 + .../interactive_cluster/out.test.toml | 1 + .../out.test.toml | 1 + .../interactive_single_user/out.test.toml | 1 + .../integration_whl/serverless/out.test.toml | 1 + .../serverless_custom_params/out.test.toml | 1 + .../serverless_dynamic_version/out.test.toml | 1 + .../integration_whl/wrapper/out.test.toml | 1 + .../wrapper_custom_params/out.test.toml | 1 + .../invariant/continue_293/out.test.toml | 1 + .../invariant/delete_idempotent/out.test.toml | 1 + .../destroy_idempotent/out.test.toml | 1 + .../bundle/invariant/migrate/out.test.toml | 1 + .../bundle/invariant/no_drift/out.test.toml | 1 + .../bundle/libraries/maven/out.test.toml | 1 + .../outside_of_bundle_root/out.test.toml | 1 + .../bundle/libraries/pypi/out.test.toml | 1 + .../lifecycle/prevent-destroy/out.test.toml | 1 + .../started-validation/out.test.toml | 1 + .../bundle/lifecycle/started/out.test.toml | 1 + .../local_state_staleness/out.test.toml | 1 + acceptance/bundle/migrate/added/out.test.toml | 1 + .../migrate/auto-migrate-clean/out.test.toml | 1 + .../auto-migrate-empty-tfstate/out.test.toml | 1 + .../migrate/auto-migrate-envvar/out.test.toml | 1 + .../auto-migrate-push-failure/out.test.toml | 1 + .../out.test.toml | 1 + acceptance/bundle/migrate/basic/out.test.toml | 1 + .../bundle/migrate/dashboards/out.test.toml | 1 + .../migrate/default-python/out.test.toml | 1 + .../engine-config-direct/out.test.toml | 1 + .../engine-config-terraform/out.test.toml | 1 + .../bundle/migrate/grants/out.test.toml | 1 + .../bundle/migrate/permissions/out.test.toml | 1 + .../bundle/migrate/profile_arg/out.test.toml | 1 + .../bundle/migrate/removed/out.test.toml | 1 + acceptance/bundle/migrate/runas/out.test.toml | 1 + .../bundle/migrate/var_arg/out.test.toml | 1 + .../multi_profile/auto_select/out.test.toml | 1 + .../multi_profile/env_auth_skip/out.test.toml | 1 + .../no_workspace_profiles/out.test.toml | 1 + .../non_interactive_error/out.test.toml | 1 + acceptance/bundle/open/out.test.toml | 1 + .../bundle/override/clusters/out.test.toml | 1 + .../bundle/override/job_cluster/out.test.toml | 1 + .../override/job_cluster_var/out.test.toml | 1 + .../bundle/override/job_tasks/out.test.toml | 1 + .../override/merge-string-map/out.test.toml | 1 + .../override/pipeline_cluster/out.test.toml | 1 + .../paths/designer_notebook/out.test.toml | 1 + .../bundle/paths/fallback/out.test.toml | 1 + .../paths/git_source_jobs/out.test.toml | 1 + .../invalid_pipeline_globs/out.test.toml | 1 + acceptance/bundle/paths/nominal/out.test.toml | 1 + .../paths/outside_root_no_sync/out.test.toml | 1 + .../out.test.toml | 1 + .../bundle/paths/pipeline_globs/out.test.toml | 1 + .../out.test.toml | 1 + .../out.test.toml | 1 + .../out.test.toml | 1 + .../relative_path_outside_root/out.test.toml | 1 + .../relative_path_translation/out.test.toml | 1 + .../bundle/plan/no_upload/out.test.toml | 1 + .../presets/preset_vs_dev_mode/out.test.toml | 1 + .../out.test.toml | 1 + .../out.test.toml | 1 + .../experimental-compatibility/out.test.toml | 1 + .../python/grants-aliases/out.test.toml | 1 + .../python/mutator-ordering/out.test.toml | 1 + .../out.test.toml | 1 + .../python/pipelines-support/out.test.toml | 1 + .../python/propagates-auth-env/out.test.toml | 1 + .../python/resolve-variable/out.test.toml | 1 + .../python/resource-loading/out.test.toml | 1 + .../python/restricted-execution/out.test.toml | 1 + .../python/schemas-support/out.test.toml | 1 + .../python/unicode-support/out.test.toml | 1 + .../python/volumes-support/out.test.toml | 1 + .../bundle/quality_monitor/out.test.toml | 1 + acceptance/bundle/refschema/out.test.toml | 1 + .../bad_ref_string_to_int/out.test.toml | 1 + .../resource_deps/bad_syntax/out.test.toml | 1 + .../computed_volume_path/out.test.toml | 1 + .../resource_deps/create_error/out.test.toml | 1 + .../resource_deps/duplicate_ref/out.test.toml | 1 + .../resource_deps/grant_ref/out.test.toml | 1 + .../resource_deps/id_chain/out.test.toml | 1 + .../resource_deps/id_star/out.test.toml | 1 + .../immutable_field_ref/out.test.toml | 1 + .../out.test.toml | 1 + .../out.test.toml | 1 + .../out.test.toml | 1 + .../implicit_deps_volume/out.test.toml | 1 + .../bundle/resource_deps/job_id/out.test.toml | 1 + .../job_id_big_graph/delete_all/out.test.toml | 1 + .../job_id_big_graph/destroy/out.test.toml | 1 + .../job_id_delete_bar/out.test.toml | 1 + .../job_id_delete_foo/out.test.toml | 1 + .../resource_deps/job_tasks/out.test.toml | 1 + .../resource_deps/jobs_update/out.test.toml | 1 + .../jobs_update_remote/out.test.toml | 1 + .../resource_deps/loop_jobs/out.test.toml | 1 + .../resource_deps/loop_self/out.test.toml | 1 + .../out.test.toml | 1 + .../missing_map_key/out.test.toml | 1 + .../missing_string_field/out.test.toml | 1 + .../resource_deps/model_id_ref/out.test.toml | 1 + .../non_existent_field/out.test.toml | 1 + .../permission_ref/out.test.toml | 1 + .../pipelines_recreate/out.test.toml | 1 + .../out.test.toml | 1 + .../remote_app_url/out.test.toml | 1 + .../out.test.toml | 1 + .../remote_pipeline/out.test.toml | 1 + .../resource_deps/resources_var/out.test.toml | 1 + .../resources_var_presets/out.test.toml | 1 + .../out.test.toml | 1 + .../tf_path_only_error/out.test.toml | 1 + .../tf_path_renames/out.test.toml | 1 + .../unicode_reference/out.test.toml | 1 + .../volume_path_contains_id/out.test.toml | 1 + .../volume_path_job_ref/out.test.toml | 1 + .../resources/alerts/basic/out.test.toml | 1 + .../resources/alerts/with_file/out.test.toml | 1 + .../out.test.toml | 1 + .../with_file_run_from_subdir/out.test.toml | 1 + .../out.test.toml | 1 + .../apps/config-drift-stopped/out.test.toml | 1 + .../resources/apps/config-drift/out.test.toml | 1 + .../apps/config-no-deployment/out.test.toml | 1 + .../apps/create_already_exists/out.test.toml | 1 + .../apps/default_description/out.test.toml | 1 + .../git-source-no-deployment/out.test.toml | 1 + .../resources/apps/immutable/out.test.toml | 1 + .../apps/inline_config/out.test.toml | 1 + .../lifecycle-started-omitted/out.test.toml | 1 + .../out.test.toml | 1 + .../lifecycle-started-toggle/out.test.toml | 1 + .../apps/lifecycle-started/out.test.toml | 1 + .../apps/readplan-lifecycle/out.test.toml | 1 + .../apps/resource-refs/out.test.toml | 1 + .../resources/apps/update/out.test.toml | 1 + .../catalogs/auto-approve/out.test.toml | 1 + .../resources/catalogs/basic/out.test.toml | 1 + .../drift/managed_properties/out.test.toml | 1 + .../catalogs/empty-name/out.test.toml | 1 + .../catalogs/with-schemas/out.test.toml | 1 + .../deploy/data_security_mode/out.test.toml | 1 + .../deploy/instance_pool/out.test.toml | 1 + .../instance_pool_and_node_type/out.test.toml | 1 + .../deploy/local_ssd_count/out.test.toml | 1 + .../deploy/num_workers_absent/out.test.toml | 1 + .../clusters/deploy/simple/out.test.toml | 1 + .../deploy/update-after-create/out.test.toml | 1 + .../update-and-resize-autoscale/out.test.toml | 1 + .../deploy/update-and-resize/out.test.toml | 1 + .../deploy/workload_type/out.test.toml | 1 + .../out.test.toml | 1 + .../lifecycle-started-toggle/out.test.toml | 1 + .../clusters/lifecycle-started/out.test.toml | 1 + .../clusters/readplan-lifecycle/out.test.toml | 1 + .../resize-terminated-fallback/out.test.toml | 1 + .../run/spark_python_task/out.test.toml | 1 + .../change-embed-credentials/out.test.toml | 1 + .../dashboards/change-name/out.test.toml | 1 + .../change-parent-path/out.test.toml | 1 + .../change-serialized-dashboard/out.test.toml | 1 + .../dataset-catalog-schema/out.test.toml | 1 + .../delete-trashed-out-of-band/out.test.toml | 1 + .../dashboards/destroy/out.test.toml | 1 + .../dashboards/detect-change/out.test.toml | 1 + .../dashboards/generate_inplace/out.test.toml | 1 + .../dashboards/nested-folders/out.test.toml | 1 + .../out.test.toml | 1 + .../out.test.toml | 1 + .../out.test.toml | 1 + .../resources/dashboards/simple/out.test.toml | 1 + .../simple_outside_bundle_root/out.test.toml | 1 + .../dashboards/simple_syncroot/out.test.toml | 1 + .../unpublish-out-of-band/out.test.toml | 1 + .../database_catalogs/basic/out.test.toml | 1 + .../database_catalogs/recreate/out.test.toml | 1 + .../database_instances/recreate/out.test.toml | 1 + .../single-instance/out.test.toml | 1 + .../resources/experiments/basic/out.test.toml | 1 + .../external_locations/out.test.toml | 1 + .../genie_spaces/delete_warning/out.test.toml | 1 + .../genie_spaces/inline/out.test.toml | 1 + .../parent_path_update/out.test.toml | 1 + .../recreate_when_gone/out.test.toml | 1 + .../serialized_space/out.test.toml | 1 + .../genie_spaces/simple/out.test.toml | 1 + .../version_migration/out.test.toml | 1 + .../resources/grants/catalogs/out.test.toml | 1 + .../grants/registered_models/out.test.toml | 1 + .../schemas/all_privileges/out.test.toml | 1 + .../all_privileges_coexist/out.test.toml | 1 + .../schemas/change_privilege/out.test.toml | 1 + .../duplicate_principals/out.test.toml | 1 + .../duplicate_privileges/out.test.toml | 1 + .../grants/schemas/empty_array/out.test.toml | 1 + .../out_of_band_principal/out.test.toml | 1 + .../grants/schemas/remove_all/out.test.toml | 1 + .../schemas/remove_principal/out.test.toml | 1 + .../resources/grants/volumes/out.test.toml | 1 + .../resources/independent/out.test.toml | 1 + .../resources/instance_pools/out.test.toml | 1 + .../resources/job_runs/basic/out.test.toml | 1 + .../job_runs/job_parameters/out.test.toml | 1 + .../resources/job_runs/redeploy/out.test.toml | 1 + .../resources/jobs/alert-task/out.test.toml | 1 + .../resources/jobs/big_id/out.test.toml | 1 + .../bundle/resources/jobs/big_id/output.txt | 4 +- .../bundle/resources/jobs/big_id/script | 4 +- .../bundle/resources/jobs/big_id/test.toml | 6 +++ .../jobs/check-metadata/out.test.toml | 1 + .../resources/jobs/create-error/out.test.toml | 1 + .../resources/jobs/delete_job/out.test.toml | 1 + .../bundle/resources/jobs/delete_job/script | 4 +- .../resources/jobs/delete_task/out.test.toml | 1 + .../resources/jobs/delete_task/test.toml | 6 +++ .../jobs/double-underscore-keys/out.test.toml | 1 + .../jobs/fail-on-active-runs/out.test.toml | 1 + .../instance_pool_and_node_type/out.test.toml | 1 + .../jobs/no-git-provider/out.test.toml | 1 + .../resources/jobs/num_workers/out.test.toml | 1 + .../resources/jobs/num_workers/output.txt | 2 +- .../bundle/resources/jobs/num_workers/script | 2 +- .../jobs/on_failure_empty_slice/out.test.toml | 1 + .../jobs/remote_add_tag/out.test.toml | 1 + .../resources/jobs/remote_add_tag/script | 2 +- .../jobs/remote_delete/deploy/out.test.toml | 1 + .../jobs/remote_delete/deploy/test.toml | 6 +++ .../jobs/remote_delete/destroy/out.test.toml | 1 + .../removed_from_config/out.test.toml | 1 + .../removed_from_config/output.txt | 2 +- .../remote_delete/removed_from_config/script | 2 +- .../jobs/remote_matches_config/out.test.toml | 1 + .../jobs/remote_matches_config/output.txt | 2 +- .../jobs/remote_matches_config/script | 4 +- .../jobs/shared-root-path/out.test.toml | 1 + .../jobs/tags_empty_map/out.test.toml | 1 + .../resources/jobs/task-source/out.test.toml | 1 + .../jobs/tasks-reorder-locally/out.test.toml | 1 + .../unknown-terraform-field/out.test.toml | 1 + .../resources/jobs/update/out.test.toml | 1 + .../bundle/resources/jobs/update/output.txt | 8 ++-- .../bundle/resources/jobs/update/script | 8 ++-- .../bundle/resources/jobs/update/test.toml | 6 +++ .../jobs/update_single_node/out.test.toml | 1 + .../jobs/update_single_node/output.txt | 8 ++-- .../resources/jobs/update_single_node/script | 14 +++---- .../jobs/webhook-reorder-remote/out.test.toml | 1 + .../jobs/webhook-reorder-remote/output.txt | 2 +- .../jobs/webhook-reorder-remote/script | 4 +- .../basic/out.test.toml | 1 + .../drift/write_only/out.test.toml | 1 + .../recreate/catalog-name/out.test.toml | 1 + .../recreate/name-change/out.test.toml | 1 + .../recreate/route-optimized/out.test.toml | 1 + .../recreate/schema-name/out.test.toml | 1 + .../recreate/table-prefix/out.test.toml | 1 + .../running-endpoint/out.test.toml | 1 + .../update/ai-gateway/out.test.toml | 1 + .../both_gateway_and_tags/out.test.toml | 1 + .../update/config/out.test.toml | 1 + .../update/email-notifications/out.test.toml | 1 + .../update/tags/out.test.toml | 1 + .../resources/models/basic/out.test.toml | 1 + .../resources/models/empty-name/out.test.toml | 1 + .../models/readplan-permissions/out.test.toml | 1 + .../apps/current_can_manage/out.test.toml | 1 + .../apps/other_can_manage/out.test.toml | 1 + .../clusters/current_can_manage/out.test.toml | 1 + .../permissions/clusters/target/out.test.toml | 1 + .../dashboards/create/out.test.toml | 1 + .../current_can_manage/out.test.toml | 1 + .../current_can_manage/out.test.toml | 1 + .../permissions/factcheck/out.test.toml | 1 + .../current_can_manage/out.test.toml | 1 + .../out_of_band_deletion/out.test.toml | 1 + .../jobs/added_remotely/out.test.toml | 1 + .../jobs/current_can_manage/out.test.toml | 1 + .../jobs/current_can_manage_run/out.test.toml | 1 + .../jobs/current_is_owner/out.test.toml | 1 + .../permissions/jobs/delete_one/out.test.toml | 1 + .../jobs/deleted_remotely/out.test.toml | 1 + .../with_permissions/out.test.toml | 1 + .../without_permissions/out.test.toml | 1 + .../permissions/jobs/empty_list/out.test.toml | 1 + .../jobs/other_can_manage/out.test.toml | 1 + .../jobs/other_can_manage_run/out.test.toml | 1 + .../jobs/other_is_owner/out.test.toml | 1 + .../jobs/reorder_locally/out.test.toml | 1 + .../jobs/reorder_remotely/out.test.toml | 1 + .../permissions/jobs/update/out.test.toml | 1 + .../permissions/jobs/viewers/out.test.toml | 1 + .../models/current_can_manage/out.test.toml | 1 + .../resources/permissions/out.test.toml | 1 + .../pipelines/504/create/out.test.toml | 1 + .../pipelines/504/plan/out.test.toml | 1 + .../pipelines/504/update/out.test.toml | 1 + .../current_can_manage/out.test.toml | 1 + .../pipelines/current_is_owner/out.test.toml | 1 + .../pipelines/empty_list/out.test.toml | 1 + .../pipelines/other_can_manage/out.test.toml | 1 + .../pipelines/other_is_owner/out.test.toml | 1 + .../pipelines/update/out.test.toml | 1 + .../current_can_manage/out.test.toml | 1 + .../current_can_manage/out.test.toml | 1 + .../target_permissions/out.test.toml | 1 + .../current_can_manage/out.test.toml | 1 + .../allow-duplicate-names/out.test.toml | 1 + .../pipelines/auto-approve/out.test.toml | 1 + .../pipelines/drift/parameters/out.test.toml | 1 + .../pipelines/lakeflow-pipeline/out.test.toml | 1 + .../pipelines/num-workers-zero/out.test.toml | 1 + .../pipelines/photon-true/out.test.toml | 1 + .../change-ingestion-definition/out.test.toml | 1 + .../change-storage/out.test.toml | 1 + .../pipelines/recreate/out.test.toml | 1 + .../remote_matches_config/out.test.toml | 1 + .../resources/pipelines/update/out.test.toml | 1 + .../pipelines/zero-value-fields/out.test.toml | 1 + .../postgres_branches/basic/out.test.toml | 1 + .../purge_on_delete/out.test.toml | 1 + .../purge_on_delete_transitions/out.test.toml | 1 + .../postgres_branches/recreate/out.test.toml | 1 + .../replace_existing/out.test.toml | 1 + .../update_protected/out.test.toml | 1 + .../without_branch_id/out.test.toml | 1 + .../postgres_catalogs/basic/out.test.toml | 1 + .../postgres_catalogs/recreate/out.test.toml | 1 + .../postgres_databases/basic/out.test.toml | 1 + .../live_errors/bad_database_id/out.test.toml | 1 + .../live_errors/bad_role_ref/out.test.toml | 1 + .../postgres_databases/recreate/out.test.toml | 1 + .../replace_existing/out.test.toml | 1 + .../postgres_databases/update/out.test.toml | 1 + .../postgres_endpoints/basic/out.test.toml | 1 + .../postgres_endpoints/recreate/out.test.toml | 1 + .../replace_existing/out.test.toml | 1 + .../update_autoscaling/out.test.toml | 1 + .../without_endpoint_id/out.test.toml | 1 + .../postgres_projects/basic/out.test.toml | 1 + .../purge_on_delete/out.test.toml | 1 + .../purge_on_delete_transitions/out.test.toml | 1 + .../postgres_projects/recreate/out.test.toml | 1 + .../update_display_name/out.test.toml | 1 + .../without_project_id/out.test.toml | 1 + .../postgres_roles/basic/out.test.toml | 1 + .../inherited-role-bind/out.test.toml | 1 + .../inherited-role-conflict/out.test.toml | 1 + .../recreate-postgres-role/out.test.toml | 1 + .../postgres_roles/recreate/out.test.toml | 1 + .../replace_existing/out.test.toml | 1 + .../postgres_roles/update/out.test.toml | 1 + .../basic/out.test.toml | 1 + .../recreate/out.test.toml | 1 + .../change_assets_dir/out.test.toml | 1 + .../change_output_schema_name/out.test.toml | 1 + .../change_table_name/out.test.toml | 1 + .../quality_monitors/create/out.test.toml | 1 + .../aliases_converge/out.test.toml | 1 + .../registered_models/basic/out.test.toml | 1 + .../drift/browse_only/out.test.toml | 1 + .../schemas/auto-approve/out.test.toml | 1 + .../drift/managed_properties/out.test.toml | 1 + .../resources/schemas/recreate/out.test.toml | 1 + .../resources/schemas/update/out.test.toml | 1 + .../secret_scopes/backend-type/out.test.toml | 1 + .../secret_scopes/basic/out.test.toml | 1 + .../secret_scopes/delete_scope/out.test.toml | 1 + .../permissions-collapse/out.test.toml | 1 + .../secret_scopes/permissions/out.test.toml | 1 + .../resources/secrets/basic/out.test.toml | 1 + .../secrets/direct-only/out.test.toml | 1 + .../secrets/update-value/out.test.toml | 1 + .../out.test.toml | 1 + .../validate-no-plain-text/out.test.toml | 1 + .../lifecycle-started-edit/out.test.toml | 1 + .../out.test.toml | 1 + .../lifecycle-started-toggle/out.test.toml | 1 + .../lifecycle-started/out.test.toml | 1 + .../resources/sql_warehouses/out.test.toml | 1 + .../basic/out.test.toml | 1 + .../recreate/out.test.toml | 1 + acceptance/bundle/resources/test.toml | 41 ++++++++----------- .../basic/out.test.toml | 1 + .../drift/budget_policy/out.test.toml | 1 + .../drift/recreated_same_name/out.test.toml | 1 + .../drift/target_qps/out.test.toml | 1 + .../recreate/create-fails/out.test.toml | 1 + .../recreate/endpoint_type/out.test.toml | 1 + .../update/budget_policy/out.test.toml | 1 + .../update/target_qps/out.test.toml | 1 + .../vector_search_indexes/basic/out.test.toml | 1 + .../drift/deleted_remotely/out.test.toml | 1 + .../drift/orphaned_endpoint/out.test.toml | 1 + .../grants/select/out.test.toml | 1 + .../embedding_dimension/out.test.toml | 1 + .../recreate/pending_deletion/out.test.toml | 1 + .../recreate/with_endpoint/out.test.toml | 1 + .../schema_normalization/out.test.toml | 1 + .../volumes/catalog-var-ref/out.test.toml | 1 + .../volumes/change-comment/out.test.toml | 1 + .../volumes/change-name/out.test.toml | 1 + .../volumes/change-schema-name/out.test.toml | 1 + .../resources/volumes/recreate/out.test.toml | 1 + .../volumes/remote-change-name/out.test.toml | 1 + .../volumes/remote-delete/out.test.toml | 1 + .../set-storage-location/out.test.toml | 1 + .../volumes/set-volume-path/out.test.toml | 1 + .../volumes/uppercase-name/out.test.toml | 1 + .../root/env-not-a-directory/out.test.toml | 1 + .../bundle/root/env-not-found/out.test.toml | 1 + .../bundle/root/not-found/out.test.toml | 1 + .../bundle/root/real-empty-dir/out.test.toml | 1 + .../bundle/run/app-with-job/out.test.toml | 1 + acceptance/bundle/run/basic/out.test.toml | 1 + .../bundle/run/diagnostics/out.test.toml | 1 + .../run/inline-script/basic/out.test.toml | 1 + .../run/inline-script/cwd/out.test.toml | 1 + .../profile-is-passed/from_flag/out.test.toml | 1 + .../target-is-passed/default/out.test.toml | 1 + .../target-is-passed/from_flag/out.test.toml | 1 + .../run/inline-script/no-auth/out.test.toml | 1 + .../run/inline-script/no-bundle/out.test.toml | 1 + .../inline-script/no-separator/out.test.toml | 1 + .../bundle/run/jobs/partial_run/out.test.toml | 1 + acceptance/bundle/run/no-state/out.test.toml | 1 + .../bundle/run/refresh-flags/out.test.toml | 1 + .../bundle/run/scripts/basic/out.test.toml | 1 + .../bundle/run/scripts/cwd/out.test.toml | 1 + .../profile-is-passed/from_flag/out.test.toml | 1 + .../target-is-passed/default/out.test.toml | 1 + .../target-is-passed/from_flag/out.test.toml | 1 + .../run/scripts/env-bad-prefix/out.test.toml | 1 + .../run/scripts/env-precedence/out.test.toml | 1 + .../run/scripts/env-section/out.test.toml | 1 + .../run/scripts/exit_code/out.test.toml | 1 + .../bundle/run/scripts/io/out.test.toml | 1 + .../bundle/run/scripts/no-auth/out.test.toml | 1 + .../scripts/no-interpolation/out.test.toml | 1 + .../run/scripts/no_content/out.test.toml | 1 + .../run/scripts/shell/envvar/out.test.toml | 1 + .../run/scripts/shell/math/out.test.toml | 1 + .../out.test.toml | 1 + .../out.test.toml | 1 + .../out.test.toml | 1 + .../bundle/run/state-wiped/out.test.toml | 1 + .../run_as/allowed/regular_user/out.test.toml | 1 + .../allowed/service_principal/out.test.toml | 1 + .../run_as/dashboard_embed/out.test.toml | 1 + .../run_as/empty_override/out.test.toml | 1 + .../bundle/run_as/empty_run_as/out.test.toml | 1 + .../run_as/empty_run_as_dict/out.test.toml | 1 + .../bundle/run_as/empty_sp/out.test.toml | 1 + .../bundle/run_as/empty_user/out.test.toml | 1 + .../run_as/empty_user_and_sp/out.test.toml | 1 + .../invalid_both_sp_and_user/out.test.toml | 1 + .../bundle/run_as/job_default/out.test.toml | 1 + .../model_serving_different/out.test.toml | 1 + .../model_serving_matching/out.test.toml | 1 + acceptance/bundle/run_as/out.test.toml | 1 + .../pipelines/regular_user/out.test.toml | 1 + .../pipelines/service_principal/out.test.toml | 1 + .../run_as/pipelines_legacy/out.test.toml | 1 + .../scripts/no-trailing-newline/out.test.toml | 1 + acceptance/bundle/scripts/out.test.toml | 1 + .../restricted-execution/out.test.toml | 1 + .../bundle/select/ambiguous/out.test.toml | 1 + acceptance/bundle/select/basic/out.test.toml | 1 + .../select/grants_permissions/out.test.toml | 1 + .../bundle/select/missing/out.test.toml | 1 + .../bundle/select/rejected/out.test.toml | 1 + acceptance/bundle/state/bad_env/out.test.toml | 1 + .../bundle/state/bad_json_local/out.test.toml | 1 + acceptance/bundle/state/basic/out.test.toml | 1 + .../bundle/state/engine_default/out.test.toml | 1 + .../state/engine_mismatch/out.test.toml | 1 + .../bundle/state/feature_flags/out.test.toml | 1 + .../state/force_pull_commands/out.test.toml | 1 + .../bundle/state/future_version/out.test.toml | 1 + .../state/lineage_different/out.test.toml | 1 + .../permission_level_migration/out.test.toml | 1 + .../bundle/state/same_serial/out.test.toml | 1 + .../bundle/state/state_present/out.test.toml | 1 + .../missing-libraries-file-path/out.test.toml | 1 + .../summary/modified_status/out.test.toml | 1 + acceptance/bundle/sync/dryrun/out.test.toml | 1 + acceptance/bundle/sync/out.test.toml | 1 + .../bundle/syncroot/dotdot-git/out.test.toml | 1 + .../syncroot/dotdot-nogit/out.test.toml | 1 + .../config-remote-sync-error/out.test.toml | 1 + .../config-remote-sync-recreate/out.test.toml | 1 + .../config-remote-sync-save/out.test.toml | 1 + .../config-remote-sync/out.test.toml | 1 + .../out.test.toml | 1 + .../deploy-artifact-path-type/out.test.toml | 1 + .../deploy-artifacts-variables/out.test.toml | 1 + .../deploy-compute-type/out.test.toml | 1 + .../deploy-config-file-count/out.test.toml | 1 + .../deploy-error-message/out.test.toml | 1 + .../telemetry/deploy-error/out.test.toml | 1 + .../deploy-experimental/out.test.toml | 1 + .../telemetry/deploy-mode/out.test.toml | 1 + .../deploy-name-prefix/custom/out.test.toml | 1 + .../mode-development/out.test.toml | 1 + .../telemetry/deploy-no-uuid/out.test.toml | 1 + .../telemetry/deploy-run-as/out.test.toml | 1 + .../deploy-target-count/out.test.toml | 1 + .../deploy-variable-count/out.test.toml | 1 + .../deploy-whl-artifacts/out.test.toml | 1 + .../out.test.toml | 1 + .../bundle/telemetry/deploy/out.test.toml | 1 + .../helper_upper_lower/out.test.toml | 1 + .../helper_username/out.test.toml | 1 + .../helpers-error/out.test.toml | 1 + .../number-precision/out.test.toml | 1 + .../supported-url/out.test.toml | 1 + .../unsupported-url/out.test.toml | 1 + .../wrong-path/out.test.toml | 1 + .../wrong-url/out.test.toml | 1 + .../bundle/templates/dbt-sql/out.test.toml | 1 + .../default-minimal/python/out.test.toml | 1 + .../default-minimal/skip/out.test.toml | 1 + .../default-minimal/sql/out.test.toml | 1 + .../azure-government/out.test.toml | 1 + .../default-python/classic/out.test.toml | 1 + .../combinations/classic/out.test.toml | 1 + .../combinations/serverless/out.test.toml | 1 + .../fail-missing-uv/out.test.toml | 1 + .../integration_classic/out.test.toml | 1 + .../default-python/no-uc/out.test.toml | 1 + .../serverless-customcatalog/out.test.toml | 1 + .../default-python/serverless/out.test.toml | 1 + .../templates/default-scala/out.test.toml | 1 + .../templates/default-sql/out.test.toml | 1 + .../lakeflow-integrations/out.test.toml | 1 + .../lakeflow-pipelines/python/out.test.toml | 1 + .../lakeflow-pipelines/sql/out.test.toml | 1 + .../templates/nested-output/out.test.toml | 1 + .../pydabs/check-consistency/out.test.toml | 1 + .../pydabs/check-formatting/out.test.toml | 1 + .../pydabs/deploy-classic/out.test.toml | 1 + .../pydabs/init-classic/out.test.toml | 1 + .../telemetry/custom-template/out.test.toml | 1 + .../templates/telemetry/dbt-sql/out.test.toml | 1 + .../telemetry/default-python/out.test.toml | 1 + .../telemetry/default-sql/out.test.toml | 1 + acceptance/bundle/test.toml | 27 ++++++++++++ .../trampoline/warning_message/out.test.toml | 1 + .../out.test.toml | 1 + .../out.test.toml | 1 + .../bundle/undefined_resources/out.test.toml | 1 + .../internal_server_error/out.test.toml | 1 + .../bundle/upload/timeout/out.test.toml | 1 + acceptance/bundle/user_agent/out.test.toml | 1 + .../bundle/user_agent/simple/out.test.toml | 1 + .../validate/anchor_containers/out.test.toml | 1 + .../out.test.toml | 1 + .../validate/dashboard_defaults/out.test.toml | 1 + .../dashboard_required_name/out.test.toml | 1 + .../out.test.toml | 1 + .../definitions_yaml_anchors/out.test.toml | 1 + .../duplicate_yaml_merge_key/out.test.toml | 1 + .../empty_resources/empty_def/out.test.toml | 1 + .../empty_resources/empty_dict/out.test.toml | 1 + .../empty_resources/null/out.test.toml | 1 + .../empty_resources/with_grants/out.test.toml | 1 + .../with_permissions/out.test.toml | 1 + .../bundle/validate/empty_tasks/out.test.toml | 1 + .../engine-config-valid/out.test.toml | 1 + acceptance/bundle/validate/enum/out.test.toml | 1 + .../validate/enum_resource_refs/out.test.toml | 1 + .../genie_space_complex/out.test.toml | 1 + .../genie_space_defaults/out.test.toml | 1 + .../out.test.toml | 1 + .../grants_required_principal/out.test.toml | 1 + .../immutable_workspace_paths/out.test.toml | 1 + .../validate/include_locations/out.test.toml | 1 + .../invalid-engine-bundle/out.test.toml | 1 + .../invalid-engine-target/out.test.toml | 1 + .../validate/job-references/out.test.toml | 1 + .../out.test.toml | 1 + .../model_serving_conversion/out.test.toml | 1 + .../models/missing_name/out.test.toml | 1 + .../validate/models/user_id/out.test.toml | 1 + .../validate/no_dashboard_etag/out.test.toml | 1 + .../no_genie_space_etag/out.test.toml | 1 + .../bundle/validate/permissions/out.test.toml | 1 + .../permissions_overlap/out.test.toml | 1 + .../presets_max_concurrent_runs/out.test.toml | 1 + .../presets_name_prefix/out.test.toml | 1 + .../presets_name_prefix_dev/out.test.toml | 1 + .../validate/presets_tags/out.test.toml | 1 + .../bundle/validate/required/out.test.toml | 1 + .../reserved_deployment_fields/out.test.toml | 1 + .../sql_warehouse_required_name/out.test.toml | 1 + .../bundle/validate/strict/out.test.toml | 1 + .../validate/sync_patterns/out.test.toml | 1 + .../validate/var_in_bundle_name/out.test.toml | 1 + .../validate/volume_defaults/out.test.toml | 1 + .../bundle/variables/arg-repeat/out.test.toml | 1 + .../variables/complex-cross-ref/out.test.toml | 1 + .../complex-cycle-self/out.test.toml | 1 + .../variables/complex-cycle/out.test.toml | 1 + .../variables/complex-simple/out.test.toml | 1 + .../complex-transitive-deep/out.test.toml | 1 + .../complex-transitive-deeper/out.test.toml | 1 + .../complex-transitive/out.test.toml | 1 + .../complex-with-var-reference/out.test.toml | 1 + .../complex-within-complex/out.test.toml | 1 + .../bundle/variables/complex/out.test.toml | 1 + .../complex_multiple_files/out.test.toml | 1 + .../bundle/variables/cycle/out.test.toml | 1 + .../variables/double_underscore/out.test.toml | 1 + .../bundle/variables/empty/out.test.toml | 1 + .../variables/env_overrides/out.test.toml | 1 + .../variables/file-defaults/out.test.toml | 1 + .../bundle/variables/git-branch/out.test.toml | 1 + .../bundle/variables/host/out.test.toml | 1 + acceptance/bundle/variables/int/out.test.toml | 1 + .../bundle/variables/issue_2436/out.test.toml | 1 + .../issue_3039_lookup_with_ref/out.test.toml | 1 + .../bundle/variables/lookup/out.test.toml | 1 + .../prepend-workspace-var/out.test.toml | 1 + .../variables/resolve-builtin/out.test.toml | 1 + .../variables/resolve-empty/out.test.toml | 1 + .../out.test.toml | 1 + .../resolve-nonstrings/out.test.toml | 1 + .../resolve-resources-fields/out.test.toml | 1 + .../resolve-vars-in-root-path/out.test.toml | 1 + .../variables/unicode_reference/out.test.toml | 1 + .../bundle/variables/vanilla/out.test.toml | 1 + .../bundle/variables/var_in_var/out.test.toml | 1 + .../variable_in_resource_key/out.test.toml | 1 + .../out.test.toml | 1 + .../without_definition/out.test.toml | 1 + .../volume_path/invalid_file/out.test.toml | 1 + .../invalid_resource/out.test.toml | 1 + .../volume_path/invalid_root/out.test.toml | 1 + .../volume_path/invalid_state/out.test.toml | 1 + .../bundle/volume_path/valid/out.test.toml | 1 + 879 files changed, 951 insertions(+), 77 deletions(-) diff --git a/acceptance/bundle/ai_runtime_task/empty_code_source/out.test.toml b/acceptance/bundle/ai_runtime_task/empty_code_source/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/ai_runtime_task/empty_code_source/out.test.toml +++ b/acceptance/bundle/ai_runtime_task/empty_code_source/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/ai_runtime_task/local_code_source/out.test.toml b/acceptance/bundle/ai_runtime_task/local_code_source/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/ai_runtime_task/local_code_source/out.test.toml +++ b/acceptance/bundle/ai_runtime_task/local_code_source/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/apps/app_yaml/out.test.toml b/acceptance/bundle/apps/app_yaml/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/apps/app_yaml/out.test.toml +++ b/acceptance/bundle/apps/app_yaml/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/apps/artifact_and_app_same_path/out.test.toml b/acceptance/bundle/apps/artifact_and_app_same_path/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/apps/artifact_and_app_same_path/out.test.toml +++ b/acceptance/bundle/apps/artifact_and_app_same_path/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/apps/compute_size/out.test.toml b/acceptance/bundle/apps/compute_size/out.test.toml index bbc7fcfd1bd..78c615e06e5 100644 --- a/acceptance/bundle/apps/compute_size/out.test.toml +++ b/acceptance/bundle/apps/compute_size/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/apps/delete_deleting/out.test.toml b/acceptance/bundle/apps/delete_deleting/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/apps/delete_deleting/out.test.toml +++ b/acceptance/bundle/apps/delete_deleting/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/apps/git_source/out.test.toml b/acceptance/bundle/apps/git_source/out.test.toml index 8f6c4a03c57..dfb2766ed88 100644 --- a/acceptance/bundle/apps/git_source/out.test.toml +++ b/acceptance/bundle/apps/git_source/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false RequiresWarehouse = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/apps/job_permissions/out.test.toml b/acceptance/bundle/apps/job_permissions/out.test.toml index e849ec85ace..fdd9f954e6c 100644 --- a/acceptance/bundle/apps/job_permissions/out.test.toml +++ b/acceptance/bundle/apps/job_permissions/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/apps/job_permissions_warning/out.test.toml b/acceptance/bundle/apps/job_permissions_warning/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/apps/job_permissions_warning/out.test.toml +++ b/acceptance/bundle/apps/job_permissions_warning/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/apps/value_from_warning/out.test.toml b/acceptance/bundle/apps/value_from_warning/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/apps/value_from_warning/out.test.toml +++ b/acceptance/bundle/apps/value_from_warning/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/artifacts/ai_runtime_code_source/out.test.toml b/acceptance/bundle/artifacts/ai_runtime_code_source/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/artifacts/ai_runtime_code_source/out.test.toml +++ b/acceptance/bundle/artifacts/ai_runtime_code_source/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/artifacts/artifact_path_with_volume/volume_doesnot_exist/out.test.toml b/acceptance/bundle/artifacts/artifact_path_with_volume/volume_doesnot_exist/out.test.toml index e849ec85ace..fdd9f954e6c 100644 --- a/acceptance/bundle/artifacts/artifact_path_with_volume/volume_doesnot_exist/out.test.toml +++ b/acceptance/bundle/artifacts/artifact_path_with_volume/volume_doesnot_exist/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/artifacts/artifact_path_with_volume/volume_not_deployed/out.test.toml b/acceptance/bundle/artifacts/artifact_path_with_volume/volume_not_deployed/out.test.toml index e849ec85ace..fdd9f954e6c 100644 --- a/acceptance/bundle/artifacts/artifact_path_with_volume/volume_not_deployed/out.test.toml +++ b/acceptance/bundle/artifacts/artifact_path_with_volume/volume_not_deployed/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/artifacts/artifact_upload_for_volumes/out.test.toml b/acceptance/bundle/artifacts/artifact_upload_for_volumes/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/artifacts/artifact_upload_for_volumes/out.test.toml +++ b/acceptance/bundle/artifacts/artifact_upload_for_volumes/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/artifacts/artifact_upload_for_workspace/out.test.toml b/acceptance/bundle/artifacts/artifact_upload_for_workspace/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/artifacts/artifact_upload_for_workspace/out.test.toml +++ b/acceptance/bundle/artifacts/artifact_upload_for_workspace/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/artifacts/artifact_upload_with_no_library_reference/out.test.toml b/acceptance/bundle/artifacts/artifact_upload_with_no_library_reference/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/artifacts/artifact_upload_with_no_library_reference/out.test.toml +++ b/acceptance/bundle/artifacts/artifact_upload_with_no_library_reference/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/artifacts/artifacts_dynamic_version/out.test.toml b/acceptance/bundle/artifacts/artifacts_dynamic_version/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/artifacts/artifacts_dynamic_version/out.test.toml +++ b/acceptance/bundle/artifacts/artifacts_dynamic_version/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/artifacts/build_and_files/out.test.toml b/acceptance/bundle/artifacts/build_and_files/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/artifacts/build_and_files/out.test.toml +++ b/acceptance/bundle/artifacts/build_and_files/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/artifacts/build_and_files_whl/out.test.toml b/acceptance/bundle/artifacts/build_and_files_whl/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/artifacts/build_and_files_whl/out.test.toml +++ b/acceptance/bundle/artifacts/build_and_files_whl/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/artifacts/glob_exact_whl/out.test.toml b/acceptance/bundle/artifacts/glob_exact_whl/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/artifacts/glob_exact_whl/out.test.toml +++ b/acceptance/bundle/artifacts/glob_exact_whl/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/artifacts/globs_in_files/out.test.toml b/acceptance/bundle/artifacts/globs_in_files/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/artifacts/globs_in_files/out.test.toml +++ b/acceptance/bundle/artifacts/globs_in_files/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/artifacts/globs_in_files_in_include/out.test.toml b/acceptance/bundle/artifacts/globs_in_files_in_include/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/artifacts/globs_in_files_in_include/out.test.toml +++ b/acceptance/bundle/artifacts/globs_in_files_in_include/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/artifacts/globs_invalid/out.test.toml b/acceptance/bundle/artifacts/globs_invalid/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/artifacts/globs_invalid/out.test.toml +++ b/acceptance/bundle/artifacts/globs_invalid/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/artifacts/issue_3109/out.test.toml b/acceptance/bundle/artifacts/issue_3109/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/artifacts/issue_3109/out.test.toml +++ b/acceptance/bundle/artifacts/issue_3109/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/artifacts/nil_artifacts/out.test.toml b/acceptance/bundle/artifacts/nil_artifacts/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/artifacts/nil_artifacts/out.test.toml +++ b/acceptance/bundle/artifacts/nil_artifacts/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/artifacts/same_name_libraries/out.test.toml b/acceptance/bundle/artifacts/same_name_libraries/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/artifacts/same_name_libraries/out.test.toml +++ b/acceptance/bundle/artifacts/same_name_libraries/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/artifacts/shell/bash/out.test.toml b/acceptance/bundle/artifacts/shell/bash/out.test.toml index 1baaa898c5b..53d56511b36 100644 --- a/acceptance/bundle/artifacts/shell/bash/out.test.toml +++ b/acceptance/bundle/artifacts/shell/bash/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/artifacts/shell/basic/out.test.toml b/acceptance/bundle/artifacts/shell/basic/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/artifacts/shell/basic/out.test.toml +++ b/acceptance/bundle/artifacts/shell/basic/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/artifacts/shell/cmd/out.test.toml b/acceptance/bundle/artifacts/shell/cmd/out.test.toml index 8471d88c7f3..af56bea47fb 100644 --- a/acceptance/bundle/artifacts/shell/cmd/out.test.toml +++ b/acceptance/bundle/artifacts/shell/cmd/out.test.toml @@ -2,4 +2,5 @@ Local = true Cloud = false GOOS.darwin = false GOOS.linux = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/artifacts/shell/default/out.test.toml b/acceptance/bundle/artifacts/shell/default/out.test.toml index 1baaa898c5b..53d56511b36 100644 --- a/acceptance/bundle/artifacts/shell/default/out.test.toml +++ b/acceptance/bundle/artifacts/shell/default/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/artifacts/shell/err-bash/out.test.toml b/acceptance/bundle/artifacts/shell/err-bash/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/artifacts/shell/err-bash/out.test.toml +++ b/acceptance/bundle/artifacts/shell/err-bash/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/artifacts/shell/err-sh/out.test.toml b/acceptance/bundle/artifacts/shell/err-sh/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/artifacts/shell/err-sh/out.test.toml +++ b/acceptance/bundle/artifacts/shell/err-sh/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/artifacts/shell/invalid/out.test.toml b/acceptance/bundle/artifacts/shell/invalid/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/artifacts/shell/invalid/out.test.toml +++ b/acceptance/bundle/artifacts/shell/invalid/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/artifacts/shell/sh/out.test.toml b/acceptance/bundle/artifacts/shell/sh/out.test.toml index 1baaa898c5b..53d56511b36 100644 --- a/acceptance/bundle/artifacts/shell/sh/out.test.toml +++ b/acceptance/bundle/artifacts/shell/sh/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/artifacts/unique_name_libraries/out.test.toml b/acceptance/bundle/artifacts/unique_name_libraries/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/artifacts/unique_name_libraries/out.test.toml +++ b/acceptance/bundle/artifacts/unique_name_libraries/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/artifacts/upload_multiple_libraries/out.test.toml b/acceptance/bundle/artifacts/upload_multiple_libraries/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/artifacts/upload_multiple_libraries/out.test.toml +++ b/acceptance/bundle/artifacts/upload_multiple_libraries/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/artifacts/whl_change_version/out.test.toml b/acceptance/bundle/artifacts/whl_change_version/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/artifacts/whl_change_version/out.test.toml +++ b/acceptance/bundle/artifacts/whl_change_version/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/artifacts/whl_dbfs/out.test.toml b/acceptance/bundle/artifacts/whl_dbfs/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/artifacts/whl_dbfs/out.test.toml +++ b/acceptance/bundle/artifacts/whl_dbfs/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/artifacts/whl_dynamic/out.test.toml b/acceptance/bundle/artifacts/whl_dynamic/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/artifacts/whl_dynamic/out.test.toml +++ b/acceptance/bundle/artifacts/whl_dynamic/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/artifacts/whl_explicit/out.test.toml b/acceptance/bundle/artifacts/whl_explicit/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/artifacts/whl_explicit/out.test.toml +++ b/acceptance/bundle/artifacts/whl_explicit/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/artifacts/whl_implicit/out.test.toml b/acceptance/bundle/artifacts/whl_implicit/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/artifacts/whl_implicit/out.test.toml +++ b/acceptance/bundle/artifacts/whl_implicit/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/artifacts/whl_implicit_custom_path/out.test.toml b/acceptance/bundle/artifacts/whl_implicit_custom_path/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/artifacts/whl_implicit_custom_path/out.test.toml +++ b/acceptance/bundle/artifacts/whl_implicit_custom_path/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/artifacts/whl_implicit_notebook/out.test.toml b/acceptance/bundle/artifacts/whl_implicit_notebook/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/artifacts/whl_implicit_notebook/out.test.toml +++ b/acceptance/bundle/artifacts/whl_implicit_notebook/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/artifacts/whl_multiple/out.test.toml b/acceptance/bundle/artifacts/whl_multiple/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/artifacts/whl_multiple/out.test.toml +++ b/acceptance/bundle/artifacts/whl_multiple/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/artifacts/whl_no_cleanup/out.test.toml b/acceptance/bundle/artifacts/whl_no_cleanup/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/artifacts/whl_no_cleanup/out.test.toml +++ b/acceptance/bundle/artifacts/whl_no_cleanup/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/artifacts/whl_prebuilt_multiple/out.test.toml b/acceptance/bundle/artifacts/whl_prebuilt_multiple/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/artifacts/whl_prebuilt_multiple/out.test.toml +++ b/acceptance/bundle/artifacts/whl_prebuilt_multiple/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/artifacts/whl_prebuilt_outside/out.test.toml b/acceptance/bundle/artifacts/whl_prebuilt_outside/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/artifacts/whl_prebuilt_outside/out.test.toml +++ b/acceptance/bundle/artifacts/whl_prebuilt_outside/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/artifacts/whl_prebuilt_outside_dynamic/out.test.toml b/acceptance/bundle/artifacts/whl_prebuilt_outside_dynamic/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/artifacts/whl_prebuilt_outside_dynamic/out.test.toml +++ b/acceptance/bundle/artifacts/whl_prebuilt_outside_dynamic/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/artifacts/whl_via_environment_key/out.test.toml b/acceptance/bundle/artifacts/whl_via_environment_key/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/artifacts/whl_via_environment_key/out.test.toml +++ b/acceptance/bundle/artifacts/whl_via_environment_key/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/benchmarks/deploy/out.test.toml b/acceptance/bundle/benchmarks/deploy/out.test.toml index 1baaa898c5b..53d56511b36 100644 --- a/acceptance/bundle/benchmarks/deploy/out.test.toml +++ b/acceptance/bundle/benchmarks/deploy/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/benchmarks/plan/out.test.toml b/acceptance/bundle/benchmarks/plan/out.test.toml index 1baaa898c5b..53d56511b36 100644 --- a/acceptance/bundle/benchmarks/plan/out.test.toml +++ b/acceptance/bundle/benchmarks/plan/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/benchmarks/validate/out.test.toml b/acceptance/bundle/benchmarks/validate/out.test.toml index 1baaa898c5b..53d56511b36 100644 --- a/acceptance/bundle/benchmarks/validate/out.test.toml +++ b/acceptance/bundle/benchmarks/validate/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/bundle_tag/id/out.test.toml b/acceptance/bundle/bundle_tag/id/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/bundle_tag/id/out.test.toml +++ b/acceptance/bundle/bundle_tag/id/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/bundle_tag/url/out.test.toml b/acceptance/bundle/bundle_tag/url/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/bundle_tag/url/out.test.toml +++ b/acceptance/bundle/bundle_tag/url/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/bundle_tag/url_ref/out.test.toml b/acceptance/bundle/bundle_tag/url_ref/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/bundle_tag/url_ref/out.test.toml +++ b/acceptance/bundle/bundle_tag/url_ref/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/config-remote-sync/cli_defaults/out.test.toml b/acceptance/bundle/config-remote-sync/cli_defaults/out.test.toml index 579b1e4a3c9..2a02860b8b1 100644 --- a/acceptance/bundle/config-remote-sync/cli_defaults/out.test.toml +++ b/acceptance/bundle/config-remote-sync/cli_defaults/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/config-remote-sync/config_edits/out.test.toml b/acceptance/bundle/config-remote-sync/config_edits/out.test.toml index 579b1e4a3c9..2a02860b8b1 100644 --- a/acceptance/bundle/config-remote-sync/config_edits/out.test.toml +++ b/acceptance/bundle/config-remote-sync/config_edits/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/config-remote-sync/dashboard_etag/out.test.toml b/acceptance/bundle/config-remote-sync/dashboard_etag/out.test.toml index 4c2be3166c4..dc7c5353574 100644 --- a/acceptance/bundle/config-remote-sync/dashboard_etag/out.test.toml +++ b/acceptance/bundle/config-remote-sync/dashboard_etag/out.test.toml @@ -2,4 +2,5 @@ Local = true Cloud = true RequiresWarehouse = true GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/config-remote-sync/flushed_cache/out.test.toml b/acceptance/bundle/config-remote-sync/flushed_cache/out.test.toml index 579b1e4a3c9..2a02860b8b1 100644 --- a/acceptance/bundle/config-remote-sync/flushed_cache/out.test.toml +++ b/acceptance/bundle/config-remote-sync/flushed_cache/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/config-remote-sync/formatting_preserved/out.test.toml b/acceptance/bundle/config-remote-sync/formatting_preserved/out.test.toml index 579b1e4a3c9..2a02860b8b1 100644 --- a/acceptance/bundle/config-remote-sync/formatting_preserved/out.test.toml +++ b/acceptance/bundle/config-remote-sync/formatting_preserved/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/config-remote-sync/job_fields/out.test.toml b/acceptance/bundle/config-remote-sync/job_fields/out.test.toml index 1773f7accf5..879c46c7bac 100644 --- a/acceptance/bundle/config-remote-sync/job_fields/out.test.toml +++ b/acceptance/bundle/config-remote-sync/job_fields/out.test.toml @@ -2,4 +2,5 @@ Local = true Cloud = true RequiresUnityCatalog = true GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/config-remote-sync/job_multiple_tasks/out.test.toml b/acceptance/bundle/config-remote-sync/job_multiple_tasks/out.test.toml index 579b1e4a3c9..2a02860b8b1 100644 --- a/acceptance/bundle/config-remote-sync/job_multiple_tasks/out.test.toml +++ b/acceptance/bundle/config-remote-sync/job_multiple_tasks/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/config-remote-sync/job_params_variables/out.test.toml b/acceptance/bundle/config-remote-sync/job_params_variables/out.test.toml index 1773f7accf5..879c46c7bac 100644 --- a/acceptance/bundle/config-remote-sync/job_params_variables/out.test.toml +++ b/acceptance/bundle/config-remote-sync/job_params_variables/out.test.toml @@ -2,4 +2,5 @@ Local = true Cloud = true RequiresUnityCatalog = true GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/config-remote-sync/job_pipeline_task/out.test.toml b/acceptance/bundle/config-remote-sync/job_pipeline_task/out.test.toml index 579b1e4a3c9..2a02860b8b1 100644 --- a/acceptance/bundle/config-remote-sync/job_pipeline_task/out.test.toml +++ b/acceptance/bundle/config-remote-sync/job_pipeline_task/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/config-remote-sync/multiple_files/out.test.toml b/acceptance/bundle/config-remote-sync/multiple_files/out.test.toml index 579b1e4a3c9..2a02860b8b1 100644 --- a/acceptance/bundle/config-remote-sync/multiple_files/out.test.toml +++ b/acceptance/bundle/config-remote-sync/multiple_files/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/config-remote-sync/multiple_resources/out.test.toml b/acceptance/bundle/config-remote-sync/multiple_resources/out.test.toml index 1773f7accf5..879c46c7bac 100644 --- a/acceptance/bundle/config-remote-sync/multiple_resources/out.test.toml +++ b/acceptance/bundle/config-remote-sync/multiple_resources/out.test.toml @@ -2,4 +2,5 @@ Local = true Cloud = true RequiresUnityCatalog = true GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/config-remote-sync/output_json/out.test.toml b/acceptance/bundle/config-remote-sync/output_json/out.test.toml index 579b1e4a3c9..2a02860b8b1 100644 --- a/acceptance/bundle/config-remote-sync/output_json/out.test.toml +++ b/acceptance/bundle/config-remote-sync/output_json/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/config-remote-sync/output_no_changes/out.test.toml b/acceptance/bundle/config-remote-sync/output_no_changes/out.test.toml index 579b1e4a3c9..2a02860b8b1 100644 --- a/acceptance/bundle/config-remote-sync/output_no_changes/out.test.toml +++ b/acceptance/bundle/config-remote-sync/output_no_changes/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/config-remote-sync/pipeline_fields/out.test.toml b/acceptance/bundle/config-remote-sync/pipeline_fields/out.test.toml index 1773f7accf5..879c46c7bac 100644 --- a/acceptance/bundle/config-remote-sync/pipeline_fields/out.test.toml +++ b/acceptance/bundle/config-remote-sync/pipeline_fields/out.test.toml @@ -2,4 +2,5 @@ Local = true Cloud = true RequiresUnityCatalog = true GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/config-remote-sync/policy_injected_cluster_fields/out.test.toml b/acceptance/bundle/config-remote-sync/policy_injected_cluster_fields/out.test.toml index 579b1e4a3c9..2a02860b8b1 100644 --- a/acceptance/bundle/config-remote-sync/policy_injected_cluster_fields/out.test.toml +++ b/acceptance/bundle/config-remote-sync/policy_injected_cluster_fields/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/config-remote-sync/resolve_variables/out.test.toml b/acceptance/bundle/config-remote-sync/resolve_variables/out.test.toml index 1773f7accf5..879c46c7bac 100644 --- a/acceptance/bundle/config-remote-sync/resolve_variables/out.test.toml +++ b/acceptance/bundle/config-remote-sync/resolve_variables/out.test.toml @@ -2,4 +2,5 @@ Local = true Cloud = true RequiresUnityCatalog = true GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/config-remote-sync/select_basic/out.test.toml b/acceptance/bundle/config-remote-sync/select_basic/out.test.toml index 579b1e4a3c9..2a02860b8b1 100644 --- a/acceptance/bundle/config-remote-sync/select_basic/out.test.toml +++ b/acceptance/bundle/config-remote-sync/select_basic/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/config-remote-sync/select_multiple/out.test.toml b/acceptance/bundle/config-remote-sync/select_multiple/out.test.toml index 579b1e4a3c9..2a02860b8b1 100644 --- a/acceptance/bundle/config-remote-sync/select_multiple/out.test.toml +++ b/acceptance/bundle/config-remote-sync/select_multiple/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/config-remote-sync/skip_permissions/out.test.toml b/acceptance/bundle/config-remote-sync/skip_permissions/out.test.toml index 579b1e4a3c9..2a02860b8b1 100644 --- a/acceptance/bundle/config-remote-sync/skip_permissions/out.test.toml +++ b/acceptance/bundle/config-remote-sync/skip_permissions/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/config-remote-sync/split/cli_default_split_element/out.test.toml b/acceptance/bundle/config-remote-sync/split/cli_default_split_element/out.test.toml index 4b5914daa2c..8c75ebba86a 100644 --- a/acceptance/bundle/config-remote-sync/split/cli_default_split_element/out.test.toml +++ b/acceptance/bundle/config-remote-sync/split/cli_default_split_element/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/config-remote-sync/split/dotted_target/out.test.toml b/acceptance/bundle/config-remote-sync/split/dotted_target/out.test.toml index 4b5914daa2c..8c75ebba86a 100644 --- a/acceptance/bundle/config-remote-sync/split/dotted_target/out.test.toml +++ b/acceptance/bundle/config-remote-sync/split/dotted_target/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/config-remote-sync/split/isolation/out.test.toml b/acceptance/bundle/config-remote-sync/split/isolation/out.test.toml index 4b5914daa2c..8c75ebba86a 100644 --- a/acceptance/bundle/config-remote-sync/split/isolation/out.test.toml +++ b/acceptance/bundle/config-remote-sync/split/isolation/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/config-remote-sync/split/keyed_edit/out.test.toml b/acceptance/bundle/config-remote-sync/split/keyed_edit/out.test.toml index 4b5914daa2c..8c75ebba86a 100644 --- a/acceptance/bundle/config-remote-sync/split/keyed_edit/out.test.toml +++ b/acceptance/bundle/config-remote-sync/split/keyed_edit/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/config-remote-sync/split/keyed_remove/out.test.toml b/acceptance/bundle/config-remote-sync/split/keyed_remove/out.test.toml index 4b5914daa2c..8c75ebba86a 100644 --- a/acceptance/bundle/config-remote-sync/split/keyed_remove/out.test.toml +++ b/acceptance/bundle/config-remote-sync/split/keyed_remove/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/config-remote-sync/split/keyed_rename/out.test.toml b/acceptance/bundle/config-remote-sync/split/keyed_rename/out.test.toml index 4b5914daa2c..8c75ebba86a 100644 --- a/acceptance/bundle/config-remote-sync/split/keyed_rename/out.test.toml +++ b/acceptance/bundle/config-remote-sync/split/keyed_rename/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/config-remote-sync/split/keyed_twoblock/out.test.toml b/acceptance/bundle/config-remote-sync/split/keyed_twoblock/out.test.toml index 4b5914daa2c..8c75ebba86a 100644 --- a/acceptance/bundle/config-remote-sync/split/keyed_twoblock/out.test.toml +++ b/acceptance/bundle/config-remote-sync/split/keyed_twoblock/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/config-remote-sync/split/multifile/out.test.toml b/acceptance/bundle/config-remote-sync/split/multifile/out.test.toml index 4b5914daa2c..8c75ebba86a 100644 --- a/acceptance/bundle/config-remote-sync/split/multifile/out.test.toml +++ b/acceptance/bundle/config-remote-sync/split/multifile/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/config-remote-sync/split/nested_add_split_parent/out.test.toml b/acceptance/bundle/config-remote-sync/split/nested_add_split_parent/out.test.toml index 4b5914daa2c..8c75ebba86a 100644 --- a/acceptance/bundle/config-remote-sync/split/nested_add_split_parent/out.test.toml +++ b/acceptance/bundle/config-remote-sync/split/nested_add_split_parent/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/config-remote-sync/split/nested_sequence/out.test.toml b/acceptance/bundle/config-remote-sync/split/nested_sequence/out.test.toml index 4b5914daa2c..8c75ebba86a 100644 --- a/acceptance/bundle/config-remote-sync/split/nested_sequence/out.test.toml +++ b/acceptance/bundle/config-remote-sync/split/nested_sequence/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/config-remote-sync/split/positional/out.test.toml b/acceptance/bundle/config-remote-sync/split/positional/out.test.toml index 4b5914daa2c..8c75ebba86a 100644 --- a/acceptance/bundle/config-remote-sync/split/positional/out.test.toml +++ b/acceptance/bundle/config-remote-sync/split/positional/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/config-remote-sync/split/remove_field_both_blocks/out.test.toml b/acceptance/bundle/config-remote-sync/split/remove_field_both_blocks/out.test.toml index 4b5914daa2c..8c75ebba86a 100644 --- a/acceptance/bundle/config-remote-sync/split/remove_field_both_blocks/out.test.toml +++ b/acceptance/bundle/config-remote-sync/split/remove_field_both_blocks/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/config-remote-sync/split/remove_with_unrelated_add/out.test.toml b/acceptance/bundle/config-remote-sync/split/remove_with_unrelated_add/out.test.toml index 4b5914daa2c..8c75ebba86a 100644 --- a/acceptance/bundle/config-remote-sync/split/remove_with_unrelated_add/out.test.toml +++ b/acceptance/bundle/config-remote-sync/split/remove_with_unrelated_add/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/config-remote-sync/split/rename_ambiguous_pairing/out.test.toml b/acceptance/bundle/config-remote-sync/split/rename_ambiguous_pairing/out.test.toml index 4b5914daa2c..8c75ebba86a 100644 --- a/acceptance/bundle/config-remote-sync/split/rename_ambiguous_pairing/out.test.toml +++ b/acceptance/bundle/config-remote-sync/split/rename_ambiguous_pairing/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/config-remote-sync/split/rename_ambiguous_single_block/out.test.toml b/acceptance/bundle/config-remote-sync/split/rename_ambiguous_single_block/out.test.toml index 4b5914daa2c..8c75ebba86a 100644 --- a/acceptance/bundle/config-remote-sync/split/rename_ambiguous_single_block/out.test.toml +++ b/acceptance/bundle/config-remote-sync/split/rename_ambiguous_single_block/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/config-remote-sync/split/rename_two_removes_one_add/out.test.toml b/acceptance/bundle/config-remote-sync/split/rename_two_removes_one_add/out.test.toml index 4b5914daa2c..8c75ebba86a 100644 --- a/acceptance/bundle/config-remote-sync/split/rename_two_removes_one_add/out.test.toml +++ b/acceptance/bundle/config-remote-sync/split/rename_two_removes_one_add/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/config-remote-sync/split/target_variable/out.test.toml b/acceptance/bundle/config-remote-sync/split/target_variable/out.test.toml index 4b5914daa2c..8c75ebba86a 100644 --- a/acceptance/bundle/config-remote-sync/split/target_variable/out.test.toml +++ b/acceptance/bundle/config-remote-sync/split/target_variable/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/config-remote-sync/split/variable_file_order/out.test.toml b/acceptance/bundle/config-remote-sync/split/variable_file_order/out.test.toml index 4b5914daa2c..8c75ebba86a 100644 --- a/acceptance/bundle/config-remote-sync/split/variable_file_order/out.test.toml +++ b/acceptance/bundle/config-remote-sync/split/variable_file_order/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/config-remote-sync/target_override/out.test.toml b/acceptance/bundle/config-remote-sync/target_override/out.test.toml index 579b1e4a3c9..2a02860b8b1 100644 --- a/acceptance/bundle/config-remote-sync/target_override/out.test.toml +++ b/acceptance/bundle/config-remote-sync/target_override/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/config-remote-sync/task_rename_revert/out.test.toml b/acceptance/bundle/config-remote-sync/task_rename_revert/out.test.toml index 579b1e4a3c9..2a02860b8b1 100644 --- a/acceptance/bundle/config-remote-sync/task_rename_revert/out.test.toml +++ b/acceptance/bundle/config-remote-sync/task_rename_revert/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/config-remote-sync/validation_errors/out.test.toml b/acceptance/bundle/config-remote-sync/validation_errors/out.test.toml index 4b5914daa2c..8c75ebba86a 100644 --- a/acceptance/bundle/config-remote-sync/validation_errors/out.test.toml +++ b/acceptance/bundle/config-remote-sync/validation_errors/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/debug/list-targets/out.test.toml b/acceptance/bundle/debug/list-targets/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/debug/list-targets/out.test.toml +++ b/acceptance/bundle/debug/list-targets/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/debug/out.test.toml b/acceptance/bundle/debug/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/debug/out.test.toml +++ b/acceptance/bundle/debug/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deploy/empty-bundle/out.test.toml b/acceptance/bundle/deploy/empty-bundle/out.test.toml index 72e8a7a4dfe..3054de89706 100644 --- a/acceptance/bundle/deploy/empty-bundle/out.test.toml +++ b/acceptance/bundle/deploy/empty-bundle/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENABLE_EXPERIMENTAL_YAML_SYNC = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deploy/experimental-python/out.test.toml b/acceptance/bundle/deploy/experimental-python/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/deploy/experimental-python/out.test.toml +++ b/acceptance/bundle/deploy/experimental-python/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deploy/fail-on-active-runs/out.test.toml b/acceptance/bundle/deploy/fail-on-active-runs/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/deploy/fail-on-active-runs/out.test.toml +++ b/acceptance/bundle/deploy/fail-on-active-runs/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deploy/files/no-snapshot-sync/out.test.toml b/acceptance/bundle/deploy/files/no-snapshot-sync/out.test.toml index bbc7fcfd1bd..78c615e06e5 100644 --- a/acceptance/bundle/deploy/files/no-snapshot-sync/out.test.toml +++ b/acceptance/bundle/deploy/files/no-snapshot-sync/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deploy/files/out-of-band-delete/out.test.toml b/acceptance/bundle/deploy/files/out-of-band-delete/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/deploy/files/out-of-band-delete/out.test.toml +++ b/acceptance/bundle/deploy/files/out-of-band-delete/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deploy/force-lock-config/out.test.toml b/acceptance/bundle/deploy/force-lock-config/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/deploy/force-lock-config/out.test.toml +++ b/acceptance/bundle/deploy/force-lock-config/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deploy/immutable-no-artifacts/out.test.toml b/acceptance/bundle/deploy/immutable-no-artifacts/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/deploy/immutable-no-artifacts/out.test.toml +++ b/acceptance/bundle/deploy/immutable-no-artifacts/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deploy/immutable-permissions-change/out.test.toml b/acceptance/bundle/deploy/immutable-permissions-change/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/deploy/immutable-permissions-change/out.test.toml +++ b/acceptance/bundle/deploy/immutable-permissions-change/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deploy/immutable/out.test.toml b/acceptance/bundle/deploy/immutable/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/deploy/immutable/out.test.toml +++ b/acceptance/bundle/deploy/immutable/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deploy/mlops-stacks/out.test.toml b/acceptance/bundle/deploy/mlops-stacks/out.test.toml index e849ec85ace..fdd9f954e6c 100644 --- a/acceptance/bundle/deploy/mlops-stacks/out.test.toml +++ b/acceptance/bundle/deploy/mlops-stacks/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deploy/pipeline-config-dots/out.test.toml b/acceptance/bundle/deploy/pipeline-config-dots/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/deploy/pipeline-config-dots/out.test.toml +++ b/acceptance/bundle/deploy/pipeline-config-dots/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deploy/python-notebook/out.test.toml b/acceptance/bundle/deploy/python-notebook/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/deploy/python-notebook/out.test.toml +++ b/acceptance/bundle/deploy/python-notebook/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deploy/readplan/basic/out.test.toml b/acceptance/bundle/deploy/readplan/basic/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/deploy/readplan/basic/out.test.toml +++ b/acceptance/bundle/deploy/readplan/basic/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deploy/readplan/cli-version-mismatch/out.test.toml b/acceptance/bundle/deploy/readplan/cli-version-mismatch/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/deploy/readplan/cli-version-mismatch/out.test.toml +++ b/acceptance/bundle/deploy/readplan/cli-version-mismatch/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deploy/readplan/grants-remove-principal/out.test.toml b/acceptance/bundle/deploy/readplan/grants-remove-principal/out.test.toml index 71970b719d4..2962c9963cc 100644 --- a/acceptance/bundle/deploy/readplan/grants-remove-principal/out.test.toml +++ b/acceptance/bundle/deploy/readplan/grants-remove-principal/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/deploy/readplan/invalid-plan/out.test.toml b/acceptance/bundle/deploy/readplan/invalid-plan/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/deploy/readplan/invalid-plan/out.test.toml +++ b/acceptance/bundle/deploy/readplan/invalid-plan/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deploy/readplan/lineage-mismatch/out.test.toml b/acceptance/bundle/deploy/readplan/lineage-mismatch/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/deploy/readplan/lineage-mismatch/out.test.toml +++ b/acceptance/bundle/deploy/readplan/lineage-mismatch/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deploy/readplan/plan-not-found/out.test.toml b/acceptance/bundle/deploy/readplan/plan-not-found/out.test.toml index f7c4cf648a9..a3d9e265a64 100644 --- a/acceptance/bundle/deploy/readplan/plan-not-found/out.test.toml +++ b/acceptance/bundle/deploy/readplan/plan-not-found/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deploy/readplan/plan-version-mismatch/out.test.toml b/acceptance/bundle/deploy/readplan/plan-version-mismatch/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/deploy/readplan/plan-version-mismatch/out.test.toml +++ b/acceptance/bundle/deploy/readplan/plan-version-mismatch/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deploy/readplan/postgres_role/out.test.toml b/acceptance/bundle/deploy/readplan/postgres_role/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/deploy/readplan/postgres_role/out.test.toml +++ b/acceptance/bundle/deploy/readplan/postgres_role/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deploy/readplan/serial-mismatch/out.test.toml b/acceptance/bundle/deploy/readplan/serial-mismatch/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/deploy/readplan/serial-mismatch/out.test.toml +++ b/acceptance/bundle/deploy/readplan/serial-mismatch/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deploy/readplan/terraform-error/out.test.toml b/acceptance/bundle/deploy/readplan/terraform-error/out.test.toml index 65156e0457c..76ce926fd59 100644 --- a/acceptance/bundle/deploy/readplan/terraform-error/out.test.toml +++ b/acceptance/bundle/deploy/readplan/terraform-error/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform"] diff --git a/acceptance/bundle/deploy/readplan/unknown-field/out.test.toml b/acceptance/bundle/deploy/readplan/unknown-field/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/deploy/readplan/unknown-field/out.test.toml +++ b/acceptance/bundle/deploy/readplan/unknown-field/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deploy/snapshot-comparison/out.test.toml b/acceptance/bundle/deploy/snapshot-comparison/out.test.toml index 42c0997090a..7674f9c196c 100644 --- a/acceptance/bundle/deploy/snapshot-comparison/out.test.toml +++ b/acceptance/bundle/deploy/snapshot-comparison/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform"] diff --git a/acceptance/bundle/deploy/spark-jar-task/out.test.toml b/acceptance/bundle/deploy/spark-jar-task/out.test.toml index bbc7fcfd1bd..78c615e06e5 100644 --- a/acceptance/bundle/deploy/spark-jar-task/out.test.toml +++ b/acceptance/bundle/deploy/spark-jar-task/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deploy/wal/chain-3-jobs/out.test.toml b/acceptance/bundle/deploy/wal/chain-3-jobs/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/deploy/wal/chain-3-jobs/out.test.toml +++ b/acceptance/bundle/deploy/wal/chain-3-jobs/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deploy/wal/corrupted-wal-entry/out.test.toml b/acceptance/bundle/deploy/wal/corrupted-wal-entry/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/deploy/wal/corrupted-wal-entry/out.test.toml +++ b/acceptance/bundle/deploy/wal/corrupted-wal-entry/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deploy/wal/crash-after-create/out.test.toml b/acceptance/bundle/deploy/wal/crash-after-create/out.test.toml index 1d895a16c96..426690291a0 100644 --- a/acceptance/bundle/deploy/wal/crash-after-create/out.test.toml +++ b/acceptance/bundle/deploy/wal/crash-after-create/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false EnvMatrix.COMMAND = ["plan", "deploy --force-lock", "summary"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deploy/wal/empty-wal/out.test.toml b/acceptance/bundle/deploy/wal/empty-wal/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/deploy/wal/empty-wal/out.test.toml +++ b/acceptance/bundle/deploy/wal/empty-wal/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deploy/wal/failed-plan-no-wal/out.test.toml b/acceptance/bundle/deploy/wal/failed-plan-no-wal/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/deploy/wal/failed-plan-no-wal/out.test.toml +++ b/acceptance/bundle/deploy/wal/failed-plan-no-wal/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deploy/wal/future-serial-wal/out.test.toml b/acceptance/bundle/deploy/wal/future-serial-wal/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/deploy/wal/future-serial-wal/out.test.toml +++ b/acceptance/bundle/deploy/wal/future-serial-wal/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deploy/wal/header-only-wal/out.test.toml b/acceptance/bundle/deploy/wal/header-only-wal/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/deploy/wal/header-only-wal/out.test.toml +++ b/acceptance/bundle/deploy/wal/header-only-wal/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deploy/wal/lineage-mismatch/out.test.toml b/acceptance/bundle/deploy/wal/lineage-mismatch/out.test.toml index 9448f875df7..84742e9cd0a 100644 --- a/acceptance/bundle/deploy/wal/lineage-mismatch/out.test.toml +++ b/acceptance/bundle/deploy/wal/lineage-mismatch/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false EnvMatrix.COMMAND = ["deploy", "plan", "summary"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deploy/wal/stale-wal/out.test.toml b/acceptance/bundle/deploy/wal/stale-wal/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/deploy/wal/stale-wal/out.test.toml +++ b/acceptance/bundle/deploy/wal/stale-wal/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deploy/wal/wal-with-delete/out.test.toml b/acceptance/bundle/deploy/wal/wal-with-delete/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/deploy/wal/wal-with-delete/out.test.toml +++ b/acceptance/bundle/deploy/wal/wal-with-delete/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deploy/yaml-sync-empty-grants/out.test.toml b/acceptance/bundle/deploy/yaml-sync-empty-grants/out.test.toml index 65156e0457c..76ce926fd59 100644 --- a/acceptance/bundle/deploy/yaml-sync-empty-grants/out.test.toml +++ b/acceptance/bundle/deploy/yaml-sync-empty-grants/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform"] diff --git a/acceptance/bundle/deployment/bind/alert/out.test.toml b/acceptance/bundle/deployment/bind/alert/out.test.toml index bbc7fcfd1bd..78c615e06e5 100644 --- a/acceptance/bundle/deployment/bind/alert/out.test.toml +++ b/acceptance/bundle/deployment/bind/alert/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/catalog/out.test.toml b/acceptance/bundle/deployment/bind/catalog/out.test.toml index fe4076cdf9b..ce8dec17c30 100644 --- a/acceptance/bundle/deployment/bind/catalog/out.test.toml +++ b/acceptance/bundle/deployment/bind/catalog/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deployment/bind/cluster/out.test.toml b/acceptance/bundle/deployment/bind/cluster/out.test.toml index f61486ff080..3f6826cd945 100644 --- a/acceptance/bundle/deployment/bind/cluster/out.test.toml +++ b/acceptance/bundle/deployment/bind/cluster/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RequiresCluster = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/dashboard/out.test.toml b/acceptance/bundle/deployment/bind/dashboard/out.test.toml index bcadf671a38..c35c189b0af 100644 --- a/acceptance/bundle/deployment/bind/dashboard/out.test.toml +++ b/acceptance/bundle/deployment/bind/dashboard/out.test.toml @@ -1,5 +1,6 @@ Local = true Cloud = true RequiresWarehouse = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/deployment/bind/dashboard/recreation/out.test.toml b/acceptance/bundle/deployment/bind/dashboard/recreation/out.test.toml index bcadf671a38..c35c189b0af 100644 --- a/acceptance/bundle/deployment/bind/dashboard/recreation/out.test.toml +++ b/acceptance/bundle/deployment/bind/dashboard/recreation/out.test.toml @@ -1,5 +1,6 @@ Local = true Cloud = true RequiresWarehouse = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/deployment/bind/database_instance/out.test.toml b/acceptance/bundle/deployment/bind/database_instance/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/deployment/bind/database_instance/out.test.toml +++ b/acceptance/bundle/deployment/bind/database_instance/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/experiment/out.test.toml b/acceptance/bundle/deployment/bind/experiment/out.test.toml index bbc7fcfd1bd..78c615e06e5 100644 --- a/acceptance/bundle/deployment/bind/experiment/out.test.toml +++ b/acceptance/bundle/deployment/bind/experiment/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/external_location/out.test.toml b/acceptance/bundle/deployment/bind/external_location/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/deployment/bind/external_location/out.test.toml +++ b/acceptance/bundle/deployment/bind/external_location/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deployment/bind/genie_space/out.test.toml b/acceptance/bundle/deployment/bind/genie_space/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/deployment/bind/genie_space/out.test.toml +++ b/acceptance/bundle/deployment/bind/genie_space/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deployment/bind/job/already-managed-different/out.test.toml b/acceptance/bundle/deployment/bind/job/already-managed-different/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/deployment/bind/job/already-managed-different/out.test.toml +++ b/acceptance/bundle/deployment/bind/job/already-managed-different/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/job/already-managed-same/out.test.toml b/acceptance/bundle/deployment/bind/job/already-managed-same/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/deployment/bind/job/already-managed-same/out.test.toml +++ b/acceptance/bundle/deployment/bind/job/already-managed-same/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/job/engine-from-config/out.test.toml b/acceptance/bundle/deployment/bind/job/engine-from-config/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/deployment/bind/job/engine-from-config/out.test.toml +++ b/acceptance/bundle/deployment/bind/job/engine-from-config/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deployment/bind/job/generate-and-bind/out.test.toml b/acceptance/bundle/deployment/bind/job/generate-and-bind/out.test.toml index bbc7fcfd1bd..78c615e06e5 100644 --- a/acceptance/bundle/deployment/bind/job/generate-and-bind/out.test.toml +++ b/acceptance/bundle/deployment/bind/job/generate-and-bind/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/job/job-abort-bind/out.test.toml b/acceptance/bundle/deployment/bind/job/job-abort-bind/out.test.toml index bbc7fcfd1bd..78c615e06e5 100644 --- a/acceptance/bundle/deployment/bind/job/job-abort-bind/out.test.toml +++ b/acceptance/bundle/deployment/bind/job/job-abort-bind/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/job/job-spark-python-task/out.test.toml b/acceptance/bundle/deployment/bind/job/job-spark-python-task/out.test.toml index bbc7fcfd1bd..78c615e06e5 100644 --- a/acceptance/bundle/deployment/bind/job/job-spark-python-task/out.test.toml +++ b/acceptance/bundle/deployment/bind/job/job-spark-python-task/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/job/noop-job/out.test.toml b/acceptance/bundle/deployment/bind/job/noop-job/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/deployment/bind/job/noop-job/out.test.toml +++ b/acceptance/bundle/deployment/bind/job/noop-job/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/job/python-job/out.test.toml b/acceptance/bundle/deployment/bind/job/python-job/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/deployment/bind/job/python-job/out.test.toml +++ b/acceptance/bundle/deployment/bind/job/python-job/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/job/stale-state/out.test.toml b/acceptance/bundle/deployment/bind/job/stale-state/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/deployment/bind/job/stale-state/out.test.toml +++ b/acceptance/bundle/deployment/bind/job/stale-state/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/model-serving-endpoint/out.test.toml b/acceptance/bundle/deployment/bind/model-serving-endpoint/out.test.toml index bbc7fcfd1bd..78c615e06e5 100644 --- a/acceptance/bundle/deployment/bind/model-serving-endpoint/out.test.toml +++ b/acceptance/bundle/deployment/bind/model-serving-endpoint/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/pipelines/recreate/out.test.toml b/acceptance/bundle/deployment/bind/pipelines/recreate/out.test.toml index bbc7fcfd1bd..78c615e06e5 100644 --- a/acceptance/bundle/deployment/bind/pipelines/recreate/out.test.toml +++ b/acceptance/bundle/deployment/bind/pipelines/recreate/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/pipelines/update/out.test.toml b/acceptance/bundle/deployment/bind/pipelines/update/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/deployment/bind/pipelines/update/out.test.toml +++ b/acceptance/bundle/deployment/bind/pipelines/update/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/postgres_database/out.test.toml b/acceptance/bundle/deployment/bind/postgres_database/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/deployment/bind/postgres_database/out.test.toml +++ b/acceptance/bundle/deployment/bind/postgres_database/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/postgres_role/out.test.toml b/acceptance/bundle/deployment/bind/postgres_role/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/deployment/bind/postgres_role/out.test.toml +++ b/acceptance/bundle/deployment/bind/postgres_role/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/quality-monitor/out.test.toml b/acceptance/bundle/deployment/bind/quality-monitor/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/deployment/bind/quality-monitor/out.test.toml +++ b/acceptance/bundle/deployment/bind/quality-monitor/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/registered-model/out.test.toml b/acceptance/bundle/deployment/bind/registered-model/out.test.toml index e849ec85ace..fdd9f954e6c 100644 --- a/acceptance/bundle/deployment/bind/registered-model/out.test.toml +++ b/acceptance/bundle/deployment/bind/registered-model/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/schema/out.test.toml b/acceptance/bundle/deployment/bind/schema/out.test.toml index e849ec85ace..fdd9f954e6c 100644 --- a/acceptance/bundle/deployment/bind/schema/out.test.toml +++ b/acceptance/bundle/deployment/bind/schema/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/secret-scope/out.test.toml b/acceptance/bundle/deployment/bind/secret-scope/out.test.toml index e849ec85ace..fdd9f954e6c 100644 --- a/acceptance/bundle/deployment/bind/secret-scope/out.test.toml +++ b/acceptance/bundle/deployment/bind/secret-scope/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/sql_warehouse/out.test.toml b/acceptance/bundle/deployment/bind/sql_warehouse/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/deployment/bind/sql_warehouse/out.test.toml +++ b/acceptance/bundle/deployment/bind/sql_warehouse/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/vector_search_endpoint/out.test.toml b/acceptance/bundle/deployment/bind/vector_search_endpoint/out.test.toml index fe4076cdf9b..ce8dec17c30 100644 --- a/acceptance/bundle/deployment/bind/vector_search_endpoint/out.test.toml +++ b/acceptance/bundle/deployment/bind/vector_search_endpoint/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deployment/bind/vector_search_index/out.test.toml b/acceptance/bundle/deployment/bind/vector_search_index/out.test.toml index 48203e833cd..8c71b922b55 100644 --- a/acceptance/bundle/deployment/bind/vector_search_index/out.test.toml +++ b/acceptance/bundle/deployment/bind/vector_search_index/out.test.toml @@ -2,4 +2,5 @@ Local = true Cloud = true CloudSlow = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deployment/bind/volume/out.test.toml b/acceptance/bundle/deployment/bind/volume/out.test.toml index e849ec85ace..fdd9f954e6c 100644 --- a/acceptance/bundle/deployment/bind/volume/out.test.toml +++ b/acceptance/bundle/deployment/bind/volume/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/unbind/engine-from-config/out.test.toml b/acceptance/bundle/deployment/unbind/engine-from-config/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/deployment/unbind/engine-from-config/out.test.toml +++ b/acceptance/bundle/deployment/unbind/engine-from-config/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deployment/unbind/grants/out.test.toml b/acceptance/bundle/deployment/unbind/grants/out.test.toml index e849ec85ace..fdd9f954e6c 100644 --- a/acceptance/bundle/deployment/unbind/grants/out.test.toml +++ b/acceptance/bundle/deployment/unbind/grants/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/unbind/job/out.test.toml b/acceptance/bundle/deployment/unbind/job/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/deployment/unbind/job/out.test.toml +++ b/acceptance/bundle/deployment/unbind/job/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/unbind/permissions/out.test.toml b/acceptance/bundle/deployment/unbind/permissions/out.test.toml index bbc7fcfd1bd..78c615e06e5 100644 --- a/acceptance/bundle/deployment/unbind/permissions/out.test.toml +++ b/acceptance/bundle/deployment/unbind/permissions/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/unbind/python-job/out.test.toml b/acceptance/bundle/deployment/unbind/python-job/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/deployment/unbind/python-job/out.test.toml +++ b/acceptance/bundle/deployment/unbind/python-job/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/destroy/all-resources/out.test.toml b/acceptance/bundle/destroy/all-resources/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/destroy/all-resources/out.test.toml +++ b/acceptance/bundle/destroy/all-resources/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/destroy/force-lock-node-limit/out.test.toml b/acceptance/bundle/destroy/force-lock-node-limit/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/destroy/force-lock-node-limit/out.test.toml +++ b/acceptance/bundle/destroy/force-lock-node-limit/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/destroy/jobs-and-pipeline/out.test.toml b/acceptance/bundle/destroy/jobs-and-pipeline/out.test.toml index bbc7fcfd1bd..78c615e06e5 100644 --- a/acceptance/bundle/destroy/jobs-and-pipeline/out.test.toml +++ b/acceptance/bundle/destroy/jobs-and-pipeline/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/dms/depends-on/output.txt b/acceptance/bundle/dms/depends-on/output.txt index 5b9ca490227..952dcb00a86 100644 --- a/acceptance/bundle/dms/depends-on/output.txt +++ b/acceptance/bundle/dms/depends-on/output.txt @@ -5,7 +5,6 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-depends-on/def Deploying resources... Updating deployment state... Deployment complete! -Deployment history: [DATABRICKS_URL]/deployments/[NUMID]?version=1 >>> print_requests.py //versions/1/operations --sort { diff --git a/acceptance/bundle/dms/existing-state/output.txt b/acceptance/bundle/dms/existing-state/output.txt index 3b5da470cfa..6233f158974 100644 --- a/acceptance/bundle/dms/existing-state/output.txt +++ b/acceptance/bundle/dms/existing-state/output.txt @@ -58,7 +58,6 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-existing-state Deploying resources... Updating deployment state... Deployment complete! -Deployment history: [DATABRICKS_URL]/deployments/[NUMID]?version=1 >>> print_requests.py //api/2.0/bundle --oneline {"method": "POST", "path": "/api/2.0/bundle/deployments", "body": {"initial_parent_path": "/Workspace/Users/[USERNAME]/.bundle/dms-existing-state/default/state", "target_name": "default"}} diff --git a/acceptance/bundle/dms/multiple-resources/output.txt b/acceptance/bundle/dms/multiple-resources/output.txt index 7e81865e11a..b593998dbd1 100644 --- a/acceptance/bundle/dms/multiple-resources/output.txt +++ b/acceptance/bundle/dms/multiple-resources/output.txt @@ -5,7 +5,6 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-multiple-resou Deploying resources... Updating deployment state... Deployment complete! -Deployment history: [DATABRICKS_URL]/deployments/[NUMID]?version=1 >>> print_requests.py //versions/1/operations --sort --del-body state --oneline {"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations", "q": {"resource_key": "jobs.five"}, "body": {"action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_id": "[NUMID]", "resource_key": "jobs.five", "status": "OPERATION_STATUS_SUCCEEDED"}} @@ -20,7 +19,6 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-multiple-resou Deploying resources... Updating deployment state... Deployment complete! -Deployment history: [DATABRICKS_URL]/deployments/[NUMID]?version=2 >>> print_requests.py //api/2.0/bundle --sort --oneline {"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": {"version_id": "2"}, "body": {"cli_version": "[CLI_VERSION]", "version_type": "VERSION_TYPE_DEPLOY", "target_name": "default", "display_name": "dms-multiple-resources", "previous_version_id": "1", "workspace_info": {"file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources/default/files", "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources/default"}}} diff --git a/acceptance/bundle/dms/no-drift/output.txt b/acceptance/bundle/dms/no-drift/output.txt index 0307d57910b..47ffe89cb73 100644 --- a/acceptance/bundle/dms/no-drift/output.txt +++ b/acceptance/bundle/dms/no-drift/output.txt @@ -5,7 +5,6 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-no-drift/defau Deploying resources... Updating deployment state... Deployment complete! -Deployment history: [DATABRICKS_URL]/deployments/[NUMID]?version=1 >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged @@ -16,7 +15,6 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-no-drift/defau Deploying resources... Updating deployment state... Deployment complete! -Deployment history: [DATABRICKS_URL]/deployments/[NUMID]?version=2 >>> print_requests.py //api/2.2/jobs //api/2.0/pipelines --sort { diff --git a/acceptance/bundle/dms/no-resources/output.txt b/acceptance/bundle/dms/no-resources/output.txt index e4c0341934b..8801d4179b1 100644 --- a/acceptance/bundle/dms/no-resources/output.txt +++ b/acceptance/bundle/dms/no-resources/output.txt @@ -4,7 +4,6 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-no-resources/default/files... Deploying resources... Deployment complete! -Deployment history: [DATABRICKS_URL]/deployments/[NUMID]?version=1 >>> print_requests.py //api/2.0/bundle --get { @@ -51,7 +50,6 @@ Deployment history: [DATABRICKS_URL]/deployments/[NUMID]?version=1 Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-no-resources/default/files... Deploying resources... Deployment complete! -Deployment history: [DATABRICKS_URL]/deployments/[NUMID]?version=2 >>> print_requests.py //api/2.0/bundle --get { diff --git a/acceptance/bundle/dms/partial-update/output.txt b/acceptance/bundle/dms/partial-update/output.txt index 8c36883a9db..f6984c72258 100644 --- a/acceptance/bundle/dms/partial-update/output.txt +++ b/acceptance/bundle/dms/partial-update/output.txt @@ -5,7 +5,6 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-partial-update Deploying resources... Updating deployment state... Deployment complete! -Deployment history: [DATABRICKS_URL]/deployments/[NUMID]?version=1 >>> print_requests.py //api/2.0/bundle { @@ -66,7 +65,6 @@ This action will result in the deletion or recreation of the following UC schema Deploying resources... Updating deployment state... Deployment complete! -Deployment history: [DATABRICKS_URL]/deployments/[NUMID]?version=2 >>> print_requests.py //api/2.0/bundle { diff --git a/acceptance/bundle/dms/provenance/output.txt b/acceptance/bundle/dms/provenance/output.txt index a4084f82728..c567a12d66c 100644 --- a/acceptance/bundle/dms/provenance/output.txt +++ b/acceptance/bundle/dms/provenance/output.txt @@ -5,7 +5,6 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-provenance/dev Deploying resources... Updating deployment state... Deployment complete! -Deployment history: [DATABRICKS_URL]/deployments/[NUMID]?version=1 >>> print_requests.py //versions --sort { diff --git a/acceptance/bundle/dms/record/output.txt b/acceptance/bundle/dms/record/output.txt index 9a51c82e60e..a707e5e46d7 100644 --- a/acceptance/bundle/dms/record/output.txt +++ b/acceptance/bundle/dms/record/output.txt @@ -5,7 +5,6 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-record/default Deploying resources... Updating deployment state... Deployment complete! -Deployment history: [DATABRICKS_URL]/deployments/[NUMID]?version=1 >>> print_requests.py //api/2.0/bundle { @@ -71,7 +70,6 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-record/default Deploying resources... Updating deployment state... Deployment complete! -Deployment history: [DATABRICKS_URL]/deployments/[NUMID]?version=2 >>> print_requests.py //api/2.0/bundle { diff --git a/acceptance/bundle/dms/redeploy-after-destroy/output.txt b/acceptance/bundle/dms/redeploy-after-destroy/output.txt index 64c4e6b8c4f..12ce1a298c4 100644 --- a/acceptance/bundle/dms/redeploy-after-destroy/output.txt +++ b/acceptance/bundle/dms/redeploy-after-destroy/output.txt @@ -5,7 +5,6 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-redeploy-after Deploying resources... Updating deployment state... Deployment complete! -Deployment history: [DATABRICKS_URL]/deployments/[NUMID]?version=1 >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: @@ -25,7 +24,6 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-redeploy-after Deploying resources... Updating deployment state... Deployment complete! -Deployment history: [DATABRICKS_URL]/deployments/[NUMID]?version=1 >>> MSYS_NO_PATHCONV=1 [CLI] workspace get-status /Workspace/Users/[USERNAME]/.bundle/dms-redeploy-after-destroy/default/state/resources.deployment.json { diff --git a/acceptance/bundle/dms/summary/output.txt b/acceptance/bundle/dms/summary/output.txt index 13a86eb3a21..c792a60fcfa 100644 --- a/acceptance/bundle/dms/summary/output.txt +++ b/acceptance/bundle/dms/summary/output.txt @@ -5,7 +5,6 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-summary/defaul Deploying resources... Updating deployment state... Deployment complete! -Deployment history: [DATABRICKS_URL]/deployments/[NUMID]?version=1 >>> [CLI] bundle summary -o json { @@ -19,7 +18,6 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-summary/defaul Deploying resources... Updating deployment state... Deployment complete! -Deployment history: [DATABRICKS_URL]/deployments/[NUMID]?version=2 >>> [CLI] bundle summary -o json { diff --git a/acceptance/bundle/empty_string_dropped/out.test.toml b/acceptance/bundle/empty_string_dropped/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/empty_string_dropped/out.test.toml +++ b/acceptance/bundle/empty_string_dropped/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/empty_string_variable/out.test.toml b/acceptance/bundle/empty_string_variable/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/empty_string_variable/out.test.toml +++ b/acceptance/bundle/empty_string_variable/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/environments/dependencies/out.test.toml b/acceptance/bundle/environments/dependencies/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/environments/dependencies/out.test.toml +++ b/acceptance/bundle/environments/dependencies/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/experimental/skip_name_prefix_for_schema/out.test.toml b/acceptance/bundle/experimental/skip_name_prefix_for_schema/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/experimental/skip_name_prefix_for_schema/out.test.toml +++ b/acceptance/bundle/experimental/skip_name_prefix_for_schema/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/generate/alert/out.test.toml b/acceptance/bundle/generate/alert/out.test.toml index bbc7fcfd1bd..78c615e06e5 100644 --- a/acceptance/bundle/generate/alert/out.test.toml +++ b/acceptance/bundle/generate/alert/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/generate/alert_existing_id_not_found/out.test.toml b/acceptance/bundle/generate/alert_existing_id_not_found/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/generate/alert_existing_id_not_found/out.test.toml +++ b/acceptance/bundle/generate/alert_existing_id_not_found/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/generate/app_not_yet_deployed/out.test.toml b/acceptance/bundle/generate/app_not_yet_deployed/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/generate/app_not_yet_deployed/out.test.toml +++ b/acceptance/bundle/generate/app_not_yet_deployed/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/generate/app_subfolders/out.test.toml b/acceptance/bundle/generate/app_subfolders/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/generate/app_subfolders/out.test.toml +++ b/acceptance/bundle/generate/app_subfolders/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/generate/auto-bind/out.test.toml b/acceptance/bundle/generate/auto-bind/out.test.toml index 42c0997090a..7674f9c196c 100644 --- a/acceptance/bundle/generate/auto-bind/out.test.toml +++ b/acceptance/bundle/generate/auto-bind/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform"] diff --git a/acceptance/bundle/generate/dashboard-inplace/out.test.toml b/acceptance/bundle/generate/dashboard-inplace/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/generate/dashboard-inplace/out.test.toml +++ b/acceptance/bundle/generate/dashboard-inplace/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/generate/dashboard/out.test.toml b/acceptance/bundle/generate/dashboard/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/generate/dashboard/out.test.toml +++ b/acceptance/bundle/generate/dashboard/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/generate/dashboard_existing_id_not_found/out.test.toml b/acceptance/bundle/generate/dashboard_existing_id_not_found/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/generate/dashboard_existing_id_not_found/out.test.toml +++ b/acceptance/bundle/generate/dashboard_existing_id_not_found/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/generate/dashboard_existing_path_nominal/out.test.toml b/acceptance/bundle/generate/dashboard_existing_path_nominal/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/generate/dashboard_existing_path_nominal/out.test.toml +++ b/acceptance/bundle/generate/dashboard_existing_path_nominal/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/generate/dashboard_existing_path_not_found/out.test.toml b/acceptance/bundle/generate/dashboard_existing_path_not_found/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/generate/dashboard_existing_path_not_found/out.test.toml +++ b/acceptance/bundle/generate/dashboard_existing_path_not_found/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/generate/designer_job/out.test.toml b/acceptance/bundle/generate/designer_job/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/generate/designer_job/out.test.toml +++ b/acceptance/bundle/generate/designer_job/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/generate/genie_space/out.test.toml b/acceptance/bundle/generate/genie_space/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/generate/genie_space/out.test.toml +++ b/acceptance/bundle/generate/genie_space/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/generate/genie_space_existing_id_not_found/out.test.toml b/acceptance/bundle/generate/genie_space_existing_id_not_found/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/generate/genie_space_existing_id_not_found/out.test.toml +++ b/acceptance/bundle/generate/genie_space_existing_id_not_found/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/generate/genie_space_inplace/out.test.toml b/acceptance/bundle/generate/genie_space_inplace/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/generate/genie_space_inplace/out.test.toml +++ b/acceptance/bundle/generate/genie_space_inplace/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/generate/git_job/out.test.toml b/acceptance/bundle/generate/git_job/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/generate/git_job/out.test.toml +++ b/acceptance/bundle/generate/git_job/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/generate/include_warning/out.test.toml b/acceptance/bundle/generate/include_warning/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/generate/include_warning/out.test.toml +++ b/acceptance/bundle/generate/include_warning/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/generate/ipynb_job/out.test.toml b/acceptance/bundle/generate/ipynb_job/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/generate/ipynb_job/out.test.toml +++ b/acceptance/bundle/generate/ipynb_job/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/generate/job_nested_notebooks/out.test.toml b/acceptance/bundle/generate/job_nested_notebooks/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/generate/job_nested_notebooks/out.test.toml +++ b/acceptance/bundle/generate/job_nested_notebooks/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/generate/lakeflow_pipelines/out.test.toml b/acceptance/bundle/generate/lakeflow_pipelines/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/generate/lakeflow_pipelines/out.test.toml +++ b/acceptance/bundle/generate/lakeflow_pipelines/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/generate/pipeline/out.test.toml b/acceptance/bundle/generate/pipeline/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/generate/pipeline/out.test.toml +++ b/acceptance/bundle/generate/pipeline/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/generate/pipeline_and_deploy/out.test.toml b/acceptance/bundle/generate/pipeline_and_deploy/out.test.toml index bbc7fcfd1bd..78c615e06e5 100644 --- a/acceptance/bundle/generate/pipeline_and_deploy/out.test.toml +++ b/acceptance/bundle/generate/pipeline_and_deploy/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/generate/pipeline_with_glob/out.test.toml b/acceptance/bundle/generate/pipeline_with_glob/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/generate/pipeline_with_glob/out.test.toml +++ b/acceptance/bundle/generate/pipeline_with_glob/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/generate/pipeline_with_sql/out.test.toml b/acceptance/bundle/generate/pipeline_with_sql/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/generate/pipeline_with_sql/out.test.toml +++ b/acceptance/bundle/generate/pipeline_with_sql/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/generate/python_job/out.test.toml b/acceptance/bundle/generate/python_job/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/generate/python_job/out.test.toml +++ b/acceptance/bundle/generate/python_job/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/generate/python_job_and_deploy/out.test.toml b/acceptance/bundle/generate/python_job_and_deploy/out.test.toml index bbc7fcfd1bd..78c615e06e5 100644 --- a/acceptance/bundle/generate/python_job_and_deploy/out.test.toml +++ b/acceptance/bundle/generate/python_job_and_deploy/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/generate/spark_python_task_job/out.test.toml b/acceptance/bundle/generate/spark_python_task_job/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/generate/spark_python_task_job/out.test.toml +++ b/acceptance/bundle/generate/spark_python_task_job/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/git-permerror/out.test.toml b/acceptance/bundle/git-permerror/out.test.toml index 1baaa898c5b..53d56511b36 100644 --- a/acceptance/bundle/git-permerror/out.test.toml +++ b/acceptance/bundle/git-permerror/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/help/bundle-deploy/out.test.toml b/acceptance/bundle/help/bundle-deploy/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/help/bundle-deploy/out.test.toml +++ b/acceptance/bundle/help/bundle-deploy/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/help/bundle-deployment-migrate/out.test.toml b/acceptance/bundle/help/bundle-deployment-migrate/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/help/bundle-deployment-migrate/out.test.toml +++ b/acceptance/bundle/help/bundle-deployment-migrate/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/help/bundle-deployment/out.test.toml b/acceptance/bundle/help/bundle-deployment/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/help/bundle-deployment/out.test.toml +++ b/acceptance/bundle/help/bundle-deployment/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/help/bundle-destroy/out.test.toml b/acceptance/bundle/help/bundle-destroy/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/help/bundle-destroy/out.test.toml +++ b/acceptance/bundle/help/bundle-destroy/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/help/bundle-generate-dashboard/out.test.toml b/acceptance/bundle/help/bundle-generate-dashboard/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/help/bundle-generate-dashboard/out.test.toml +++ b/acceptance/bundle/help/bundle-generate-dashboard/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/help/bundle-generate-job/out.test.toml b/acceptance/bundle/help/bundle-generate-job/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/help/bundle-generate-job/out.test.toml +++ b/acceptance/bundle/help/bundle-generate-job/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/help/bundle-generate-pipeline/out.test.toml b/acceptance/bundle/help/bundle-generate-pipeline/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/help/bundle-generate-pipeline/out.test.toml +++ b/acceptance/bundle/help/bundle-generate-pipeline/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/help/bundle-generate/out.test.toml b/acceptance/bundle/help/bundle-generate/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/help/bundle-generate/out.test.toml +++ b/acceptance/bundle/help/bundle-generate/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/help/bundle-init/out.test.toml b/acceptance/bundle/help/bundle-init/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/help/bundle-init/out.test.toml +++ b/acceptance/bundle/help/bundle-init/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/help/bundle-open/out.test.toml b/acceptance/bundle/help/bundle-open/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/help/bundle-open/out.test.toml +++ b/acceptance/bundle/help/bundle-open/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/help/bundle-run/out.test.toml b/acceptance/bundle/help/bundle-run/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/help/bundle-run/out.test.toml +++ b/acceptance/bundle/help/bundle-run/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/help/bundle-schema/out.test.toml b/acceptance/bundle/help/bundle-schema/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/help/bundle-schema/out.test.toml +++ b/acceptance/bundle/help/bundle-schema/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/help/bundle-summary/out.test.toml b/acceptance/bundle/help/bundle-summary/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/help/bundle-summary/out.test.toml +++ b/acceptance/bundle/help/bundle-summary/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/help/bundle-sync/out.test.toml b/acceptance/bundle/help/bundle-sync/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/help/bundle-sync/out.test.toml +++ b/acceptance/bundle/help/bundle-sync/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/help/bundle-validate/out.test.toml b/acceptance/bundle/help/bundle-validate/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/help/bundle-validate/out.test.toml +++ b/acceptance/bundle/help/bundle-validate/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/help/bundle/out.test.toml b/acceptance/bundle/help/bundle/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/help/bundle/out.test.toml +++ b/acceptance/bundle/help/bundle/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/includes/glob_in_root_path/out.test.toml b/acceptance/bundle/includes/glob_in_root_path/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/includes/glob_in_root_path/out.test.toml +++ b/acceptance/bundle/includes/glob_in_root_path/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/includes/include_outside_root/out.test.toml b/acceptance/bundle/includes/include_outside_root/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/includes/include_outside_root/out.test.toml +++ b/acceptance/bundle/includes/include_outside_root/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/includes/non_yaml_in_include/out.test.toml b/acceptance/bundle/includes/non_yaml_in_include/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/includes/non_yaml_in_include/out.test.toml +++ b/acceptance/bundle/includes/non_yaml_in_include/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/includes/yml_outside_root/out.test.toml b/acceptance/bundle/includes/yml_outside_root/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/includes/yml_outside_root/out.test.toml +++ b/acceptance/bundle/includes/yml_outside_root/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/integration_whl/base/out.test.toml b/acceptance/bundle/integration_whl/base/out.test.toml index 880431d5a9f..95d684f1aba 100644 --- a/acceptance/bundle/integration_whl/base/out.test.toml +++ b/acceptance/bundle/integration_whl/base/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true CloudSlow = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/integration_whl/custom_params/out.test.toml b/acceptance/bundle/integration_whl/custom_params/out.test.toml index 880431d5a9f..95d684f1aba 100644 --- a/acceptance/bundle/integration_whl/custom_params/out.test.toml +++ b/acceptance/bundle/integration_whl/custom_params/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true CloudSlow = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/integration_whl/interactive_cluster/out.test.toml b/acceptance/bundle/integration_whl/interactive_cluster/out.test.toml index 880431d5a9f..95d684f1aba 100644 --- a/acceptance/bundle/integration_whl/interactive_cluster/out.test.toml +++ b/acceptance/bundle/integration_whl/interactive_cluster/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true CloudSlow = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/integration_whl/interactive_cluster_dynamic_version/out.test.toml b/acceptance/bundle/integration_whl/interactive_cluster_dynamic_version/out.test.toml index 19068d43e0a..5d56f06c3a2 100644 --- a/acceptance/bundle/integration_whl/interactive_cluster_dynamic_version/out.test.toml +++ b/acceptance/bundle/integration_whl/interactive_cluster_dynamic_version/out.test.toml @@ -1,5 +1,6 @@ Local = true Cloud = true CloudSlow = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] EnvMatrix.DATA_SECURITY_MODE = ["USER_ISOLATION", "SINGLE_USER"] diff --git a/acceptance/bundle/integration_whl/interactive_single_user/out.test.toml b/acceptance/bundle/integration_whl/interactive_single_user/out.test.toml index 880431d5a9f..95d684f1aba 100644 --- a/acceptance/bundle/integration_whl/interactive_single_user/out.test.toml +++ b/acceptance/bundle/integration_whl/interactive_single_user/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true CloudSlow = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/integration_whl/serverless/out.test.toml b/acceptance/bundle/integration_whl/serverless/out.test.toml index 68b04957a4c..7a09cba84bb 100644 --- a/acceptance/bundle/integration_whl/serverless/out.test.toml +++ b/acceptance/bundle/integration_whl/serverless/out.test.toml @@ -2,4 +2,5 @@ Local = true Cloud = true CloudSlow = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/integration_whl/serverless_custom_params/out.test.toml b/acceptance/bundle/integration_whl/serverless_custom_params/out.test.toml index 68b04957a4c..7a09cba84bb 100644 --- a/acceptance/bundle/integration_whl/serverless_custom_params/out.test.toml +++ b/acceptance/bundle/integration_whl/serverless_custom_params/out.test.toml @@ -2,4 +2,5 @@ Local = true Cloud = true CloudSlow = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/integration_whl/serverless_dynamic_version/out.test.toml b/acceptance/bundle/integration_whl/serverless_dynamic_version/out.test.toml index 68b04957a4c..7a09cba84bb 100644 --- a/acceptance/bundle/integration_whl/serverless_dynamic_version/out.test.toml +++ b/acceptance/bundle/integration_whl/serverless_dynamic_version/out.test.toml @@ -2,4 +2,5 @@ Local = true Cloud = true CloudSlow = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/integration_whl/wrapper/out.test.toml b/acceptance/bundle/integration_whl/wrapper/out.test.toml index 44a1a2186a1..6342b9a4af7 100644 --- a/acceptance/bundle/integration_whl/wrapper/out.test.toml +++ b/acceptance/bundle/integration_whl/wrapper/out.test.toml @@ -3,4 +3,5 @@ Cloud = true CloudSlow = true CloudEnvs.aws = false CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/integration_whl/wrapper_custom_params/out.test.toml b/acceptance/bundle/integration_whl/wrapper_custom_params/out.test.toml index 44a1a2186a1..6342b9a4af7 100644 --- a/acceptance/bundle/integration_whl/wrapper_custom_params/out.test.toml +++ b/acceptance/bundle/integration_whl/wrapper_custom_params/out.test.toml @@ -3,4 +3,5 @@ Cloud = true CloudSlow = true CloudEnvs.aws = false CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/invariant/continue_293/out.test.toml b/acceptance/bundle/invariant/continue_293/out.test.toml index c294b244621..c9d202227e3 100644 --- a/acceptance/bundle/invariant/continue_293/out.test.toml +++ b/acceptance/bundle/invariant/continue_293/out.test.toml @@ -1,6 +1,7 @@ Local = true Cloud = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] EnvMatrix.INPUT_CONFIG = [ "alert.yml.tmpl", diff --git a/acceptance/bundle/invariant/delete_idempotent/out.test.toml b/acceptance/bundle/invariant/delete_idempotent/out.test.toml index f65b1680aa1..3ce69ae1a49 100644 --- a/acceptance/bundle/invariant/delete_idempotent/out.test.toml +++ b/acceptance/bundle/invariant/delete_idempotent/out.test.toml @@ -1,6 +1,7 @@ Local = true Cloud = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] EnvMatrix.INPUT_CONFIG = [ "alert.yml.tmpl", diff --git a/acceptance/bundle/invariant/destroy_idempotent/out.test.toml b/acceptance/bundle/invariant/destroy_idempotent/out.test.toml index f65b1680aa1..3ce69ae1a49 100644 --- a/acceptance/bundle/invariant/destroy_idempotent/out.test.toml +++ b/acceptance/bundle/invariant/destroy_idempotent/out.test.toml @@ -1,6 +1,7 @@ Local = true Cloud = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] EnvMatrix.INPUT_CONFIG = [ "alert.yml.tmpl", diff --git a/acceptance/bundle/invariant/migrate/out.test.toml b/acceptance/bundle/invariant/migrate/out.test.toml index 8560caa0ee5..cf188cbc54b 100644 --- a/acceptance/bundle/invariant/migrate/out.test.toml +++ b/acceptance/bundle/invariant/migrate/out.test.toml @@ -1,6 +1,7 @@ Local = true Cloud = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] EnvMatrix.INPUT_CONFIG = [ "alert.yml.tmpl", diff --git a/acceptance/bundle/invariant/no_drift/out.test.toml b/acceptance/bundle/invariant/no_drift/out.test.toml index f65b1680aa1..3ce69ae1a49 100644 --- a/acceptance/bundle/invariant/no_drift/out.test.toml +++ b/acceptance/bundle/invariant/no_drift/out.test.toml @@ -1,6 +1,7 @@ Local = true Cloud = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] EnvMatrix.INPUT_CONFIG = [ "alert.yml.tmpl", diff --git a/acceptance/bundle/libraries/maven/out.test.toml b/acceptance/bundle/libraries/maven/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/libraries/maven/out.test.toml +++ b/acceptance/bundle/libraries/maven/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/libraries/outside_of_bundle_root/out.test.toml b/acceptance/bundle/libraries/outside_of_bundle_root/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/libraries/outside_of_bundle_root/out.test.toml +++ b/acceptance/bundle/libraries/outside_of_bundle_root/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/libraries/pypi/out.test.toml b/acceptance/bundle/libraries/pypi/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/libraries/pypi/out.test.toml +++ b/acceptance/bundle/libraries/pypi/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/lifecycle/prevent-destroy/out.test.toml b/acceptance/bundle/lifecycle/prevent-destroy/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/lifecycle/prevent-destroy/out.test.toml +++ b/acceptance/bundle/lifecycle/prevent-destroy/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/lifecycle/started-validation/out.test.toml b/acceptance/bundle/lifecycle/started-validation/out.test.toml index b37ee45aed6..931153dacc9 100644 --- a/acceptance/bundle/lifecycle/started-validation/out.test.toml +++ b/acceptance/bundle/lifecycle/started-validation/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/lifecycle/started/out.test.toml b/acceptance/bundle/lifecycle/started/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/lifecycle/started/out.test.toml +++ b/acceptance/bundle/lifecycle/started/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/local_state_staleness/out.test.toml b/acceptance/bundle/local_state_staleness/out.test.toml index bbc7fcfd1bd..78c615e06e5 100644 --- a/acceptance/bundle/local_state_staleness/out.test.toml +++ b/acceptance/bundle/local_state_staleness/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/migrate/added/out.test.toml b/acceptance/bundle/migrate/added/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/migrate/added/out.test.toml +++ b/acceptance/bundle/migrate/added/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/migrate/auto-migrate-clean/out.test.toml b/acceptance/bundle/migrate/auto-migrate-clean/out.test.toml index 71970b719d4..2962c9963cc 100644 --- a/acceptance/bundle/migrate/auto-migrate-clean/out.test.toml +++ b/acceptance/bundle/migrate/auto-migrate-clean/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/migrate/auto-migrate-empty-tfstate/out.test.toml b/acceptance/bundle/migrate/auto-migrate-empty-tfstate/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/migrate/auto-migrate-empty-tfstate/out.test.toml +++ b/acceptance/bundle/migrate/auto-migrate-empty-tfstate/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/migrate/auto-migrate-envvar/out.test.toml b/acceptance/bundle/migrate/auto-migrate-envvar/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/migrate/auto-migrate-envvar/out.test.toml +++ b/acceptance/bundle/migrate/auto-migrate-envvar/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/migrate/auto-migrate-push-failure/out.test.toml b/acceptance/bundle/migrate/auto-migrate-push-failure/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/migrate/auto-migrate-push-failure/out.test.toml +++ b/acceptance/bundle/migrate/auto-migrate-push-failure/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/migrate/auto-migrate-tfbackup-failure/out.test.toml b/acceptance/bundle/migrate/auto-migrate-tfbackup-failure/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/migrate/auto-migrate-tfbackup-failure/out.test.toml +++ b/acceptance/bundle/migrate/auto-migrate-tfbackup-failure/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/migrate/basic/out.test.toml b/acceptance/bundle/migrate/basic/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/migrate/basic/out.test.toml +++ b/acceptance/bundle/migrate/basic/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/migrate/dashboards/out.test.toml b/acceptance/bundle/migrate/dashboards/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/migrate/dashboards/out.test.toml +++ b/acceptance/bundle/migrate/dashboards/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/migrate/default-python/out.test.toml b/acceptance/bundle/migrate/default-python/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/migrate/default-python/out.test.toml +++ b/acceptance/bundle/migrate/default-python/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/migrate/engine-config-direct/out.test.toml b/acceptance/bundle/migrate/engine-config-direct/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/migrate/engine-config-direct/out.test.toml +++ b/acceptance/bundle/migrate/engine-config-direct/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/migrate/engine-config-terraform/out.test.toml b/acceptance/bundle/migrate/engine-config-terraform/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/migrate/engine-config-terraform/out.test.toml +++ b/acceptance/bundle/migrate/engine-config-terraform/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/migrate/grants/out.test.toml b/acceptance/bundle/migrate/grants/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/migrate/grants/out.test.toml +++ b/acceptance/bundle/migrate/grants/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/migrate/permissions/out.test.toml b/acceptance/bundle/migrate/permissions/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/migrate/permissions/out.test.toml +++ b/acceptance/bundle/migrate/permissions/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/migrate/profile_arg/out.test.toml b/acceptance/bundle/migrate/profile_arg/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/migrate/profile_arg/out.test.toml +++ b/acceptance/bundle/migrate/profile_arg/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/migrate/removed/out.test.toml b/acceptance/bundle/migrate/removed/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/migrate/removed/out.test.toml +++ b/acceptance/bundle/migrate/removed/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/migrate/runas/out.test.toml b/acceptance/bundle/migrate/runas/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/migrate/runas/out.test.toml +++ b/acceptance/bundle/migrate/runas/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/migrate/var_arg/out.test.toml b/acceptance/bundle/migrate/var_arg/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/migrate/var_arg/out.test.toml +++ b/acceptance/bundle/migrate/var_arg/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/multi_profile/auto_select/out.test.toml b/acceptance/bundle/multi_profile/auto_select/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/multi_profile/auto_select/out.test.toml +++ b/acceptance/bundle/multi_profile/auto_select/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/multi_profile/env_auth_skip/out.test.toml b/acceptance/bundle/multi_profile/env_auth_skip/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/multi_profile/env_auth_skip/out.test.toml +++ b/acceptance/bundle/multi_profile/env_auth_skip/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/multi_profile/no_workspace_profiles/out.test.toml b/acceptance/bundle/multi_profile/no_workspace_profiles/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/multi_profile/no_workspace_profiles/out.test.toml +++ b/acceptance/bundle/multi_profile/no_workspace_profiles/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/multi_profile/non_interactive_error/out.test.toml b/acceptance/bundle/multi_profile/non_interactive_error/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/multi_profile/non_interactive_error/out.test.toml +++ b/acceptance/bundle/multi_profile/non_interactive_error/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/open/out.test.toml b/acceptance/bundle/open/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/open/out.test.toml +++ b/acceptance/bundle/open/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/override/clusters/out.test.toml b/acceptance/bundle/override/clusters/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/override/clusters/out.test.toml +++ b/acceptance/bundle/override/clusters/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/override/job_cluster/out.test.toml b/acceptance/bundle/override/job_cluster/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/override/job_cluster/out.test.toml +++ b/acceptance/bundle/override/job_cluster/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/override/job_cluster_var/out.test.toml b/acceptance/bundle/override/job_cluster_var/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/override/job_cluster_var/out.test.toml +++ b/acceptance/bundle/override/job_cluster_var/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/override/job_tasks/out.test.toml b/acceptance/bundle/override/job_tasks/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/override/job_tasks/out.test.toml +++ b/acceptance/bundle/override/job_tasks/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/override/merge-string-map/out.test.toml b/acceptance/bundle/override/merge-string-map/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/override/merge-string-map/out.test.toml +++ b/acceptance/bundle/override/merge-string-map/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/override/pipeline_cluster/out.test.toml b/acceptance/bundle/override/pipeline_cluster/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/override/pipeline_cluster/out.test.toml +++ b/acceptance/bundle/override/pipeline_cluster/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/paths/designer_notebook/out.test.toml b/acceptance/bundle/paths/designer_notebook/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/paths/designer_notebook/out.test.toml +++ b/acceptance/bundle/paths/designer_notebook/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/paths/fallback/out.test.toml b/acceptance/bundle/paths/fallback/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/paths/fallback/out.test.toml +++ b/acceptance/bundle/paths/fallback/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/paths/git_source_jobs/out.test.toml b/acceptance/bundle/paths/git_source_jobs/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/paths/git_source_jobs/out.test.toml +++ b/acceptance/bundle/paths/git_source_jobs/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/paths/invalid_pipeline_globs/out.test.toml b/acceptance/bundle/paths/invalid_pipeline_globs/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/paths/invalid_pipeline_globs/out.test.toml +++ b/acceptance/bundle/paths/invalid_pipeline_globs/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/paths/nominal/out.test.toml b/acceptance/bundle/paths/nominal/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/paths/nominal/out.test.toml +++ b/acceptance/bundle/paths/nominal/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/paths/outside_root_no_sync/out.test.toml b/acceptance/bundle/paths/outside_root_no_sync/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/paths/outside_root_no_sync/out.test.toml +++ b/acceptance/bundle/paths/outside_root_no_sync/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/paths/pipeline_expected_file_got_notebook/out.test.toml b/acceptance/bundle/paths/pipeline_expected_file_got_notebook/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/paths/pipeline_expected_file_got_notebook/out.test.toml +++ b/acceptance/bundle/paths/pipeline_expected_file_got_notebook/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/paths/pipeline_globs/out.test.toml b/acceptance/bundle/paths/pipeline_globs/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/paths/pipeline_globs/out.test.toml +++ b/acceptance/bundle/paths/pipeline_globs/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/paths/pipeline_root_path_doesnotexist/out.test.toml b/acceptance/bundle/paths/pipeline_root_path_doesnotexist/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/paths/pipeline_root_path_doesnotexist/out.test.toml +++ b/acceptance/bundle/paths/pipeline_root_path_doesnotexist/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/paths/pipelines_glob_include_and_root_path/out.test.toml b/acceptance/bundle/paths/pipelines_glob_include_and_root_path/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/paths/pipelines_glob_include_and_root_path/out.test.toml +++ b/acceptance/bundle/paths/pipelines_glob_include_and_root_path/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/paths/pipelines_root_path_outside_sync_root/out.test.toml b/acceptance/bundle/paths/pipelines_root_path_outside_sync_root/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/paths/pipelines_root_path_outside_sync_root/out.test.toml +++ b/acceptance/bundle/paths/pipelines_root_path_outside_sync_root/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/paths/relative_path_outside_root/out.test.toml b/acceptance/bundle/paths/relative_path_outside_root/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/paths/relative_path_outside_root/out.test.toml +++ b/acceptance/bundle/paths/relative_path_outside_root/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/paths/relative_path_translation/out.test.toml b/acceptance/bundle/paths/relative_path_translation/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/paths/relative_path_translation/out.test.toml +++ b/acceptance/bundle/paths/relative_path_translation/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/plan/no_upload/out.test.toml b/acceptance/bundle/plan/no_upload/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/plan/no_upload/out.test.toml +++ b/acceptance/bundle/plan/no_upload/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/presets/preset_vs_dev_mode/out.test.toml b/acceptance/bundle/presets/preset_vs_dev_mode/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/presets/preset_vs_dev_mode/out.test.toml +++ b/acceptance/bundle/presets/preset_vs_dev_mode/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/python/experimental-compatibility-both-equal/out.test.toml b/acceptance/bundle/python/experimental-compatibility-both-equal/out.test.toml index 0969b3f3733..62e13e707c8 100644 --- a/acceptance/bundle/python/experimental-compatibility-both-equal/out.test.toml +++ b/acceptance/bundle/python/experimental-compatibility-both-equal/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] EnvMatrix.PYDAB_VERSION = ["0.266.0", "current"] diff --git a/acceptance/bundle/python/experimental-compatibility-both-error/out.test.toml b/acceptance/bundle/python/experimental-compatibility-both-error/out.test.toml index 0969b3f3733..62e13e707c8 100644 --- a/acceptance/bundle/python/experimental-compatibility-both-error/out.test.toml +++ b/acceptance/bundle/python/experimental-compatibility-both-error/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] EnvMatrix.PYDAB_VERSION = ["0.266.0", "current"] diff --git a/acceptance/bundle/python/experimental-compatibility/out.test.toml b/acceptance/bundle/python/experimental-compatibility/out.test.toml index 0969b3f3733..62e13e707c8 100644 --- a/acceptance/bundle/python/experimental-compatibility/out.test.toml +++ b/acceptance/bundle/python/experimental-compatibility/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] EnvMatrix.PYDAB_VERSION = ["0.266.0", "current"] diff --git a/acceptance/bundle/python/grants-aliases/out.test.toml b/acceptance/bundle/python/grants-aliases/out.test.toml index 98d084e3bb9..c806f1e3811 100644 --- a/acceptance/bundle/python/grants-aliases/out.test.toml +++ b/acceptance/bundle/python/grants-aliases/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] EnvMatrix.PYDAB_VERSION = ["current"] diff --git a/acceptance/bundle/python/mutator-ordering/out.test.toml b/acceptance/bundle/python/mutator-ordering/out.test.toml index 0969b3f3733..62e13e707c8 100644 --- a/acceptance/bundle/python/mutator-ordering/out.test.toml +++ b/acceptance/bundle/python/mutator-ordering/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] EnvMatrix.PYDAB_VERSION = ["0.266.0", "current"] diff --git a/acceptance/bundle/python/mutator-permissions-owner-5682/out.test.toml b/acceptance/bundle/python/mutator-permissions-owner-5682/out.test.toml index 4c48a83f25b..256d0941b25 100644 --- a/acceptance/bundle/python/mutator-permissions-owner-5682/out.test.toml +++ b/acceptance/bundle/python/mutator-permissions-owner-5682/out.test.toml @@ -1,5 +1,6 @@ Local = true Cloud = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] EnvMatrix.PYDAB_VERSION = ["0.266.0", "current"] diff --git a/acceptance/bundle/python/pipelines-support/out.test.toml b/acceptance/bundle/python/pipelines-support/out.test.toml index 0969b3f3733..62e13e707c8 100644 --- a/acceptance/bundle/python/pipelines-support/out.test.toml +++ b/acceptance/bundle/python/pipelines-support/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] EnvMatrix.PYDAB_VERSION = ["0.266.0", "current"] diff --git a/acceptance/bundle/python/propagates-auth-env/out.test.toml b/acceptance/bundle/python/propagates-auth-env/out.test.toml index 0969b3f3733..62e13e707c8 100644 --- a/acceptance/bundle/python/propagates-auth-env/out.test.toml +++ b/acceptance/bundle/python/propagates-auth-env/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] EnvMatrix.PYDAB_VERSION = ["0.266.0", "current"] diff --git a/acceptance/bundle/python/resolve-variable/out.test.toml b/acceptance/bundle/python/resolve-variable/out.test.toml index 0969b3f3733..62e13e707c8 100644 --- a/acceptance/bundle/python/resolve-variable/out.test.toml +++ b/acceptance/bundle/python/resolve-variable/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] EnvMatrix.PYDAB_VERSION = ["0.266.0", "current"] diff --git a/acceptance/bundle/python/resource-loading/out.test.toml b/acceptance/bundle/python/resource-loading/out.test.toml index 0969b3f3733..62e13e707c8 100644 --- a/acceptance/bundle/python/resource-loading/out.test.toml +++ b/acceptance/bundle/python/resource-loading/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] EnvMatrix.PYDAB_VERSION = ["0.266.0", "current"] diff --git a/acceptance/bundle/python/restricted-execution/out.test.toml b/acceptance/bundle/python/restricted-execution/out.test.toml index 0969b3f3733..62e13e707c8 100644 --- a/acceptance/bundle/python/restricted-execution/out.test.toml +++ b/acceptance/bundle/python/restricted-execution/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] EnvMatrix.PYDAB_VERSION = ["0.266.0", "current"] diff --git a/acceptance/bundle/python/schemas-support/out.test.toml b/acceptance/bundle/python/schemas-support/out.test.toml index 98d084e3bb9..c806f1e3811 100644 --- a/acceptance/bundle/python/schemas-support/out.test.toml +++ b/acceptance/bundle/python/schemas-support/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] EnvMatrix.PYDAB_VERSION = ["current"] diff --git a/acceptance/bundle/python/unicode-support/out.test.toml b/acceptance/bundle/python/unicode-support/out.test.toml index 0969b3f3733..62e13e707c8 100644 --- a/acceptance/bundle/python/unicode-support/out.test.toml +++ b/acceptance/bundle/python/unicode-support/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] EnvMatrix.PYDAB_VERSION = ["0.266.0", "current"] diff --git a/acceptance/bundle/python/volumes-support/out.test.toml b/acceptance/bundle/python/volumes-support/out.test.toml index 0969b3f3733..62e13e707c8 100644 --- a/acceptance/bundle/python/volumes-support/out.test.toml +++ b/acceptance/bundle/python/volumes-support/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] EnvMatrix.PYDAB_VERSION = ["0.266.0", "current"] diff --git a/acceptance/bundle/quality_monitor/out.test.toml b/acceptance/bundle/quality_monitor/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/quality_monitor/out.test.toml +++ b/acceptance/bundle/quality_monitor/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/refschema/out.test.toml b/acceptance/bundle/refschema/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/refschema/out.test.toml +++ b/acceptance/bundle/refschema/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/bad_ref_string_to_int/out.test.toml b/acceptance/bundle/resource_deps/bad_ref_string_to_int/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resource_deps/bad_ref_string_to_int/out.test.toml +++ b/acceptance/bundle/resource_deps/bad_ref_string_to_int/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/bad_syntax/out.test.toml b/acceptance/bundle/resource_deps/bad_syntax/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resource_deps/bad_syntax/out.test.toml +++ b/acceptance/bundle/resource_deps/bad_syntax/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/computed_volume_path/out.test.toml b/acceptance/bundle/resource_deps/computed_volume_path/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resource_deps/computed_volume_path/out.test.toml +++ b/acceptance/bundle/resource_deps/computed_volume_path/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/create_error/out.test.toml b/acceptance/bundle/resource_deps/create_error/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resource_deps/create_error/out.test.toml +++ b/acceptance/bundle/resource_deps/create_error/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/duplicate_ref/out.test.toml b/acceptance/bundle/resource_deps/duplicate_ref/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resource_deps/duplicate_ref/out.test.toml +++ b/acceptance/bundle/resource_deps/duplicate_ref/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/grant_ref/out.test.toml b/acceptance/bundle/resource_deps/grant_ref/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/resource_deps/grant_ref/out.test.toml +++ b/acceptance/bundle/resource_deps/grant_ref/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resource_deps/id_chain/out.test.toml b/acceptance/bundle/resource_deps/id_chain/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resource_deps/id_chain/out.test.toml +++ b/acceptance/bundle/resource_deps/id_chain/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/id_star/out.test.toml b/acceptance/bundle/resource_deps/id_star/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resource_deps/id_star/out.test.toml +++ b/acceptance/bundle/resource_deps/id_star/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/immutable_field_ref/out.test.toml b/acceptance/bundle/resource_deps/immutable_field_ref/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/resource_deps/immutable_field_ref/out.test.toml +++ b/acceptance/bundle/resource_deps/immutable_field_ref/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resource_deps/implicit_deps_model_serving_endpoint/out.test.toml b/acceptance/bundle/resource_deps/implicit_deps_model_serving_endpoint/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resource_deps/implicit_deps_model_serving_endpoint/out.test.toml +++ b/acceptance/bundle/resource_deps/implicit_deps_model_serving_endpoint/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/implicit_deps_quality_monitor/out.test.toml b/acceptance/bundle/resource_deps/implicit_deps_quality_monitor/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resource_deps/implicit_deps_quality_monitor/out.test.toml +++ b/acceptance/bundle/resource_deps/implicit_deps_quality_monitor/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/implicit_deps_registered_model/out.test.toml b/acceptance/bundle/resource_deps/implicit_deps_registered_model/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resource_deps/implicit_deps_registered_model/out.test.toml +++ b/acceptance/bundle/resource_deps/implicit_deps_registered_model/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/implicit_deps_volume/out.test.toml b/acceptance/bundle/resource_deps/implicit_deps_volume/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resource_deps/implicit_deps_volume/out.test.toml +++ b/acceptance/bundle/resource_deps/implicit_deps_volume/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/job_id/out.test.toml b/acceptance/bundle/resource_deps/job_id/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resource_deps/job_id/out.test.toml +++ b/acceptance/bundle/resource_deps/job_id/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/job_id_big_graph/delete_all/out.test.toml b/acceptance/bundle/resource_deps/job_id_big_graph/delete_all/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resource_deps/job_id_big_graph/delete_all/out.test.toml +++ b/acceptance/bundle/resource_deps/job_id_big_graph/delete_all/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/job_id_big_graph/destroy/out.test.toml b/acceptance/bundle/resource_deps/job_id_big_graph/destroy/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resource_deps/job_id_big_graph/destroy/out.test.toml +++ b/acceptance/bundle/resource_deps/job_id_big_graph/destroy/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/job_id_delete_bar/out.test.toml b/acceptance/bundle/resource_deps/job_id_delete_bar/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resource_deps/job_id_delete_bar/out.test.toml +++ b/acceptance/bundle/resource_deps/job_id_delete_bar/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/job_id_delete_foo/out.test.toml b/acceptance/bundle/resource_deps/job_id_delete_foo/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resource_deps/job_id_delete_foo/out.test.toml +++ b/acceptance/bundle/resource_deps/job_id_delete_foo/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/job_tasks/out.test.toml b/acceptance/bundle/resource_deps/job_tasks/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resource_deps/job_tasks/out.test.toml +++ b/acceptance/bundle/resource_deps/job_tasks/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/jobs_update/out.test.toml b/acceptance/bundle/resource_deps/jobs_update/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resource_deps/jobs_update/out.test.toml +++ b/acceptance/bundle/resource_deps/jobs_update/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/jobs_update_remote/out.test.toml b/acceptance/bundle/resource_deps/jobs_update_remote/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resource_deps/jobs_update_remote/out.test.toml +++ b/acceptance/bundle/resource_deps/jobs_update_remote/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/loop_jobs/out.test.toml b/acceptance/bundle/resource_deps/loop_jobs/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resource_deps/loop_jobs/out.test.toml +++ b/acceptance/bundle/resource_deps/loop_jobs/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/loop_self/out.test.toml b/acceptance/bundle/resource_deps/loop_self/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resource_deps/loop_self/out.test.toml +++ b/acceptance/bundle/resource_deps/loop_self/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/missing_ingestion_definition/out.test.toml b/acceptance/bundle/resource_deps/missing_ingestion_definition/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resource_deps/missing_ingestion_definition/out.test.toml +++ b/acceptance/bundle/resource_deps/missing_ingestion_definition/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/missing_map_key/out.test.toml b/acceptance/bundle/resource_deps/missing_map_key/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resource_deps/missing_map_key/out.test.toml +++ b/acceptance/bundle/resource_deps/missing_map_key/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/missing_string_field/out.test.toml b/acceptance/bundle/resource_deps/missing_string_field/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resource_deps/missing_string_field/out.test.toml +++ b/acceptance/bundle/resource_deps/missing_string_field/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/model_id_ref/out.test.toml b/acceptance/bundle/resource_deps/model_id_ref/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resource_deps/model_id_ref/out.test.toml +++ b/acceptance/bundle/resource_deps/model_id_ref/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/non_existent_field/out.test.toml b/acceptance/bundle/resource_deps/non_existent_field/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resource_deps/non_existent_field/out.test.toml +++ b/acceptance/bundle/resource_deps/non_existent_field/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/permission_ref/out.test.toml b/acceptance/bundle/resource_deps/permission_ref/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/resource_deps/permission_ref/out.test.toml +++ b/acceptance/bundle/resource_deps/permission_ref/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resource_deps/pipelines_recreate/out.test.toml b/acceptance/bundle/resource_deps/pipelines_recreate/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resource_deps/pipelines_recreate/out.test.toml +++ b/acceptance/bundle/resource_deps/pipelines_recreate/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/present_ingestion_definition/out.test.toml b/acceptance/bundle/resource_deps/present_ingestion_definition/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resource_deps/present_ingestion_definition/out.test.toml +++ b/acceptance/bundle/resource_deps/present_ingestion_definition/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/remote_app_url/out.test.toml b/acceptance/bundle/resource_deps/remote_app_url/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resource_deps/remote_app_url/out.test.toml +++ b/acceptance/bundle/resource_deps/remote_app_url/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/remote_field_storage_location/out.test.toml b/acceptance/bundle/resource_deps/remote_field_storage_location/out.test.toml index 8c738f635ac..4ba1c38c46b 100644 --- a/acceptance/bundle/resource_deps/remote_field_storage_location/out.test.toml +++ b/acceptance/bundle/resource_deps/remote_field_storage_location/out.test.toml @@ -3,4 +3,5 @@ Cloud = true RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/remote_pipeline/out.test.toml b/acceptance/bundle/resource_deps/remote_pipeline/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resource_deps/remote_pipeline/out.test.toml +++ b/acceptance/bundle/resource_deps/remote_pipeline/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/resources_var/out.test.toml b/acceptance/bundle/resource_deps/resources_var/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resource_deps/resources_var/out.test.toml +++ b/acceptance/bundle/resource_deps/resources_var/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/resources_var_presets/out.test.toml b/acceptance/bundle/resource_deps/resources_var_presets/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resource_deps/resources_var_presets/out.test.toml +++ b/acceptance/bundle/resource_deps/resources_var_presets/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/resources_var_presets_implicit_deps/out.test.toml b/acceptance/bundle/resource_deps/resources_var_presets_implicit_deps/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resource_deps/resources_var_presets_implicit_deps/out.test.toml +++ b/acceptance/bundle/resource_deps/resources_var_presets_implicit_deps/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/tf_path_only_error/out.test.toml b/acceptance/bundle/resource_deps/tf_path_only_error/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resource_deps/tf_path_only_error/out.test.toml +++ b/acceptance/bundle/resource_deps/tf_path_only_error/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/tf_path_renames/out.test.toml b/acceptance/bundle/resource_deps/tf_path_renames/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resource_deps/tf_path_renames/out.test.toml +++ b/acceptance/bundle/resource_deps/tf_path_renames/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/unicode_reference/out.test.toml b/acceptance/bundle/resource_deps/unicode_reference/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resource_deps/unicode_reference/out.test.toml +++ b/acceptance/bundle/resource_deps/unicode_reference/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/volume_path_contains_id/out.test.toml b/acceptance/bundle/resource_deps/volume_path_contains_id/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resource_deps/volume_path_contains_id/out.test.toml +++ b/acceptance/bundle/resource_deps/volume_path_contains_id/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resource_deps/volume_path_job_ref/out.test.toml b/acceptance/bundle/resource_deps/volume_path_job_ref/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/resource_deps/volume_path_job_ref/out.test.toml +++ b/acceptance/bundle/resource_deps/volume_path_job_ref/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/alerts/basic/out.test.toml b/acceptance/bundle/resources/alerts/basic/out.test.toml index c45d8e76a8e..eaf2e7a7966 100644 --- a/acceptance/bundle/resources/alerts/basic/out.test.toml +++ b/acceptance/bundle/resources/alerts/basic/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RunsOnDbr = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/alerts/with_file/out.test.toml b/acceptance/bundle/resources/alerts/with_file/out.test.toml index bbc7fcfd1bd..78c615e06e5 100644 --- a/acceptance/bundle/resources/alerts/with_file/out.test.toml +++ b/acceptance/bundle/resources/alerts/with_file/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/alerts/with_file_not_allowed_field_error/out.test.toml b/acceptance/bundle/resources/alerts/with_file_not_allowed_field_error/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/alerts/with_file_not_allowed_field_error/out.test.toml +++ b/acceptance/bundle/resources/alerts/with_file_not_allowed_field_error/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/alerts/with_file_run_from_subdir/out.test.toml b/acceptance/bundle/resources/alerts/with_file_run_from_subdir/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/alerts/with_file_run_from_subdir/out.test.toml +++ b/acceptance/bundle/resources/alerts/with_file_run_from_subdir/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/alerts/with_file_variable_interpolation_error/out.test.toml b/acceptance/bundle/resources/alerts/with_file_variable_interpolation_error/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/alerts/with_file_variable_interpolation_error/out.test.toml +++ b/acceptance/bundle/resources/alerts/with_file_variable_interpolation_error/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/apps/config-drift-stopped/out.test.toml b/acceptance/bundle/resources/apps/config-drift-stopped/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/resources/apps/config-drift-stopped/out.test.toml +++ b/acceptance/bundle/resources/apps/config-drift-stopped/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/apps/config-drift/out.test.toml b/acceptance/bundle/resources/apps/config-drift/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/resources/apps/config-drift/out.test.toml +++ b/acceptance/bundle/resources/apps/config-drift/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/apps/config-no-deployment/out.test.toml b/acceptance/bundle/resources/apps/config-no-deployment/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/resources/apps/config-no-deployment/out.test.toml +++ b/acceptance/bundle/resources/apps/config-no-deployment/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/apps/create_already_exists/out.test.toml b/acceptance/bundle/resources/apps/create_already_exists/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/resources/apps/create_already_exists/out.test.toml +++ b/acceptance/bundle/resources/apps/create_already_exists/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/apps/default_description/out.test.toml b/acceptance/bundle/resources/apps/default_description/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/apps/default_description/out.test.toml +++ b/acceptance/bundle/resources/apps/default_description/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/apps/git-source-no-deployment/out.test.toml b/acceptance/bundle/resources/apps/git-source-no-deployment/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/resources/apps/git-source-no-deployment/out.test.toml +++ b/acceptance/bundle/resources/apps/git-source-no-deployment/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/apps/immutable/out.test.toml b/acceptance/bundle/resources/apps/immutable/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/resources/apps/immutable/out.test.toml +++ b/acceptance/bundle/resources/apps/immutable/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/apps/inline_config/out.test.toml b/acceptance/bundle/resources/apps/inline_config/out.test.toml index bbc7fcfd1bd..78c615e06e5 100644 --- a/acceptance/bundle/resources/apps/inline_config/out.test.toml +++ b/acceptance/bundle/resources/apps/inline_config/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/apps/lifecycle-started-omitted/out.test.toml b/acceptance/bundle/resources/apps/lifecycle-started-omitted/out.test.toml index 9cfad3fb0d5..db8b4387cfe 100644 --- a/acceptance/bundle/resources/apps/lifecycle-started-omitted/out.test.toml +++ b/acceptance/bundle/resources/apps/lifecycle-started-omitted/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/apps/lifecycle-started-terraform-error/out.test.toml b/acceptance/bundle/resources/apps/lifecycle-started-terraform-error/out.test.toml index 42c0997090a..7674f9c196c 100644 --- a/acceptance/bundle/resources/apps/lifecycle-started-terraform-error/out.test.toml +++ b/acceptance/bundle/resources/apps/lifecycle-started-terraform-error/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform"] diff --git a/acceptance/bundle/resources/apps/lifecycle-started-toggle/out.test.toml b/acceptance/bundle/resources/apps/lifecycle-started-toggle/out.test.toml index 9cfad3fb0d5..db8b4387cfe 100644 --- a/acceptance/bundle/resources/apps/lifecycle-started-toggle/out.test.toml +++ b/acceptance/bundle/resources/apps/lifecycle-started-toggle/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/apps/lifecycle-started/out.test.toml b/acceptance/bundle/resources/apps/lifecycle-started/out.test.toml index 9cfad3fb0d5..db8b4387cfe 100644 --- a/acceptance/bundle/resources/apps/lifecycle-started/out.test.toml +++ b/acceptance/bundle/resources/apps/lifecycle-started/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/apps/readplan-lifecycle/out.test.toml b/acceptance/bundle/resources/apps/readplan-lifecycle/out.test.toml index 71970b719d4..2962c9963cc 100644 --- a/acceptance/bundle/resources/apps/readplan-lifecycle/out.test.toml +++ b/acceptance/bundle/resources/apps/readplan-lifecycle/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/resources/apps/resource-refs/out.test.toml b/acceptance/bundle/resources/apps/resource-refs/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/apps/resource-refs/out.test.toml +++ b/acceptance/bundle/resources/apps/resource-refs/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/apps/update/out.test.toml b/acceptance/bundle/resources/apps/update/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/apps/update/out.test.toml +++ b/acceptance/bundle/resources/apps/update/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/catalogs/auto-approve/out.test.toml b/acceptance/bundle/resources/catalogs/auto-approve/out.test.toml index fe4076cdf9b..ce8dec17c30 100644 --- a/acceptance/bundle/resources/catalogs/auto-approve/out.test.toml +++ b/acceptance/bundle/resources/catalogs/auto-approve/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/catalogs/basic/out.test.toml b/acceptance/bundle/resources/catalogs/basic/out.test.toml index fe4076cdf9b..ce8dec17c30 100644 --- a/acceptance/bundle/resources/catalogs/basic/out.test.toml +++ b/acceptance/bundle/resources/catalogs/basic/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/catalogs/drift/managed_properties/out.test.toml b/acceptance/bundle/resources/catalogs/drift/managed_properties/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/resources/catalogs/drift/managed_properties/out.test.toml +++ b/acceptance/bundle/resources/catalogs/drift/managed_properties/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/catalogs/empty-name/out.test.toml b/acceptance/bundle/resources/catalogs/empty-name/out.test.toml index fe4076cdf9b..ce8dec17c30 100644 --- a/acceptance/bundle/resources/catalogs/empty-name/out.test.toml +++ b/acceptance/bundle/resources/catalogs/empty-name/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/catalogs/with-schemas/out.test.toml b/acceptance/bundle/resources/catalogs/with-schemas/out.test.toml index fe4076cdf9b..ce8dec17c30 100644 --- a/acceptance/bundle/resources/catalogs/with-schemas/out.test.toml +++ b/acceptance/bundle/resources/catalogs/with-schemas/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/clusters/deploy/data_security_mode/out.test.toml b/acceptance/bundle/resources/clusters/deploy/data_security_mode/out.test.toml index 5ad0addb75e..c1cae9fcc3e 100644 --- a/acceptance/bundle/resources/clusters/deploy/data_security_mode/out.test.toml +++ b/acceptance/bundle/resources/clusters/deploy/data_security_mode/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RunsOnDbr = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/clusters/deploy/instance_pool/out.test.toml b/acceptance/bundle/resources/clusters/deploy/instance_pool/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/clusters/deploy/instance_pool/out.test.toml +++ b/acceptance/bundle/resources/clusters/deploy/instance_pool/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/clusters/deploy/instance_pool_and_node_type/out.test.toml b/acceptance/bundle/resources/clusters/deploy/instance_pool_and_node_type/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/clusters/deploy/instance_pool_and_node_type/out.test.toml +++ b/acceptance/bundle/resources/clusters/deploy/instance_pool_and_node_type/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/clusters/deploy/local_ssd_count/out.test.toml b/acceptance/bundle/resources/clusters/deploy/local_ssd_count/out.test.toml index 5a821e39edc..813b3a187d4 100644 --- a/acceptance/bundle/resources/clusters/deploy/local_ssd_count/out.test.toml +++ b/acceptance/bundle/resources/clusters/deploy/local_ssd_count/out.test.toml @@ -3,4 +3,5 @@ Cloud = true CloudEnvs.aws = false CloudEnvs.azure = false CloudEnvs.gcp = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/clusters/deploy/num_workers_absent/out.test.toml b/acceptance/bundle/resources/clusters/deploy/num_workers_absent/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/clusters/deploy/num_workers_absent/out.test.toml +++ b/acceptance/bundle/resources/clusters/deploy/num_workers_absent/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/clusters/deploy/simple/out.test.toml b/acceptance/bundle/resources/clusters/deploy/simple/out.test.toml index 5ad0addb75e..c1cae9fcc3e 100644 --- a/acceptance/bundle/resources/clusters/deploy/simple/out.test.toml +++ b/acceptance/bundle/resources/clusters/deploy/simple/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RunsOnDbr = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/clusters/deploy/update-after-create/out.test.toml b/acceptance/bundle/resources/clusters/deploy/update-after-create/out.test.toml index bbc7fcfd1bd..78c615e06e5 100644 --- a/acceptance/bundle/resources/clusters/deploy/update-after-create/out.test.toml +++ b/acceptance/bundle/resources/clusters/deploy/update-after-create/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/out.test.toml b/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/out.test.toml +++ b/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/clusters/deploy/update-and-resize/out.test.toml b/acceptance/bundle/resources/clusters/deploy/update-and-resize/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/clusters/deploy/update-and-resize/out.test.toml +++ b/acceptance/bundle/resources/clusters/deploy/update-and-resize/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/clusters/deploy/workload_type/out.test.toml b/acceptance/bundle/resources/clusters/deploy/workload_type/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/clusters/deploy/workload_type/out.test.toml +++ b/acceptance/bundle/resources/clusters/deploy/workload_type/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/clusters/lifecycle-started-terraform-error/out.test.toml b/acceptance/bundle/resources/clusters/lifecycle-started-terraform-error/out.test.toml index 42c0997090a..7674f9c196c 100644 --- a/acceptance/bundle/resources/clusters/lifecycle-started-terraform-error/out.test.toml +++ b/acceptance/bundle/resources/clusters/lifecycle-started-terraform-error/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform"] diff --git a/acceptance/bundle/resources/clusters/lifecycle-started-toggle/out.test.toml b/acceptance/bundle/resources/clusters/lifecycle-started-toggle/out.test.toml index 9cfad3fb0d5..db8b4387cfe 100644 --- a/acceptance/bundle/resources/clusters/lifecycle-started-toggle/out.test.toml +++ b/acceptance/bundle/resources/clusters/lifecycle-started-toggle/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/clusters/lifecycle-started/out.test.toml b/acceptance/bundle/resources/clusters/lifecycle-started/out.test.toml index 9cfad3fb0d5..db8b4387cfe 100644 --- a/acceptance/bundle/resources/clusters/lifecycle-started/out.test.toml +++ b/acceptance/bundle/resources/clusters/lifecycle-started/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/clusters/readplan-lifecycle/out.test.toml b/acceptance/bundle/resources/clusters/readplan-lifecycle/out.test.toml index 71970b719d4..2962c9963cc 100644 --- a/acceptance/bundle/resources/clusters/readplan-lifecycle/out.test.toml +++ b/acceptance/bundle/resources/clusters/readplan-lifecycle/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/resources/clusters/resize-terminated-fallback/out.test.toml b/acceptance/bundle/resources/clusters/resize-terminated-fallback/out.test.toml index 9cfad3fb0d5..db8b4387cfe 100644 --- a/acceptance/bundle/resources/clusters/resize-terminated-fallback/out.test.toml +++ b/acceptance/bundle/resources/clusters/resize-terminated-fallback/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/clusters/run/spark_python_task/out.test.toml b/acceptance/bundle/resources/clusters/run/spark_python_task/out.test.toml index 475b179caed..ebb5db02455 100644 --- a/acceptance/bundle/resources/clusters/run/spark_python_task/out.test.toml +++ b/acceptance/bundle/resources/clusters/run/spark_python_task/out.test.toml @@ -2,4 +2,5 @@ Local = true Cloud = true CloudSlow = true RunsOnDbr = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/dashboards/change-embed-credentials/out.test.toml b/acceptance/bundle/resources/dashboards/change-embed-credentials/out.test.toml index 96be4fdfe9d..ec63f1740c2 100644 --- a/acceptance/bundle/resources/dashboards/change-embed-credentials/out.test.toml +++ b/acceptance/bundle/resources/dashboards/change-embed-credentials/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RequiresWarehouse = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/dashboards/change-name/out.test.toml b/acceptance/bundle/resources/dashboards/change-name/out.test.toml index 96be4fdfe9d..ec63f1740c2 100644 --- a/acceptance/bundle/resources/dashboards/change-name/out.test.toml +++ b/acceptance/bundle/resources/dashboards/change-name/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RequiresWarehouse = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/dashboards/change-parent-path/out.test.toml b/acceptance/bundle/resources/dashboards/change-parent-path/out.test.toml index 96be4fdfe9d..ec63f1740c2 100644 --- a/acceptance/bundle/resources/dashboards/change-parent-path/out.test.toml +++ b/acceptance/bundle/resources/dashboards/change-parent-path/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RequiresWarehouse = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/dashboards/change-serialized-dashboard/out.test.toml b/acceptance/bundle/resources/dashboards/change-serialized-dashboard/out.test.toml index bcadf671a38..c35c189b0af 100644 --- a/acceptance/bundle/resources/dashboards/change-serialized-dashboard/out.test.toml +++ b/acceptance/bundle/resources/dashboards/change-serialized-dashboard/out.test.toml @@ -1,5 +1,6 @@ Local = true Cloud = true RequiresWarehouse = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/resources/dashboards/dataset-catalog-schema/out.test.toml b/acceptance/bundle/resources/dashboards/dataset-catalog-schema/out.test.toml index c2ac722e76a..ec9bbdf7292 100644 --- a/acceptance/bundle/resources/dashboards/dataset-catalog-schema/out.test.toml +++ b/acceptance/bundle/resources/dashboards/dataset-catalog-schema/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RequiresWarehouse = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/resources/dashboards/delete-trashed-out-of-band/out.test.toml b/acceptance/bundle/resources/dashboards/delete-trashed-out-of-band/out.test.toml index 7edd52865f7..dce76f03654 100644 --- a/acceptance/bundle/resources/dashboards/delete-trashed-out-of-band/out.test.toml +++ b/acceptance/bundle/resources/dashboards/delete-trashed-out-of-band/out.test.toml @@ -2,4 +2,5 @@ Local = true Cloud = true RequiresWarehouse = true RunsOnDbr = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/dashboards/destroy/out.test.toml b/acceptance/bundle/resources/dashboards/destroy/out.test.toml index 7edd52865f7..dce76f03654 100644 --- a/acceptance/bundle/resources/dashboards/destroy/out.test.toml +++ b/acceptance/bundle/resources/dashboards/destroy/out.test.toml @@ -2,4 +2,5 @@ Local = true Cloud = true RequiresWarehouse = true RunsOnDbr = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/dashboards/detect-change/out.test.toml b/acceptance/bundle/resources/dashboards/detect-change/out.test.toml index 96be4fdfe9d..ec63f1740c2 100644 --- a/acceptance/bundle/resources/dashboards/detect-change/out.test.toml +++ b/acceptance/bundle/resources/dashboards/detect-change/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RequiresWarehouse = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/dashboards/generate_inplace/out.test.toml b/acceptance/bundle/resources/dashboards/generate_inplace/out.test.toml index 7edd52865f7..dce76f03654 100644 --- a/acceptance/bundle/resources/dashboards/generate_inplace/out.test.toml +++ b/acceptance/bundle/resources/dashboards/generate_inplace/out.test.toml @@ -2,4 +2,5 @@ Local = true Cloud = true RequiresWarehouse = true RunsOnDbr = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/dashboards/nested-folders/out.test.toml b/acceptance/bundle/resources/dashboards/nested-folders/out.test.toml index 1976bc173ca..65ad5749140 100644 --- a/acceptance/bundle/resources/dashboards/nested-folders/out.test.toml +++ b/acceptance/bundle/resources/dashboards/nested-folders/out.test.toml @@ -2,5 +2,6 @@ Local = true Cloud = true RequiresWarehouse = true RunsOnDbr = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/resources/dashboards/publish-failure-cleans-up-dashboard/out.test.toml b/acceptance/bundle/resources/dashboards/publish-failure-cleans-up-dashboard/out.test.toml index b5ce19512e3..2e27b38ff97 100644 --- a/acceptance/bundle/resources/dashboards/publish-failure-cleans-up-dashboard/out.test.toml +++ b/acceptance/bundle/resources/dashboards/publish-failure-cleans-up-dashboard/out.test.toml @@ -1,5 +1,6 @@ Local = true Cloud = false RequiresWarehouse = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/resources/dashboards/publish-failure-stale-content/out.test.toml b/acceptance/bundle/resources/dashboards/publish-failure-stale-content/out.test.toml index a29f11b9ab2..88d62d0ff13 100644 --- a/acceptance/bundle/resources/dashboards/publish-failure-stale-content/out.test.toml +++ b/acceptance/bundle/resources/dashboards/publish-failure-stale-content/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false RequiresWarehouse = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/dashboards/republish-after-draft-update/out.test.toml b/acceptance/bundle/resources/dashboards/republish-after-draft-update/out.test.toml index a29f11b9ab2..88d62d0ff13 100644 --- a/acceptance/bundle/resources/dashboards/republish-after-draft-update/out.test.toml +++ b/acceptance/bundle/resources/dashboards/republish-after-draft-update/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false RequiresWarehouse = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/dashboards/simple/out.test.toml b/acceptance/bundle/resources/dashboards/simple/out.test.toml index 7edd52865f7..dce76f03654 100644 --- a/acceptance/bundle/resources/dashboards/simple/out.test.toml +++ b/acceptance/bundle/resources/dashboards/simple/out.test.toml @@ -2,4 +2,5 @@ Local = true Cloud = true RequiresWarehouse = true RunsOnDbr = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/dashboards/simple_outside_bundle_root/out.test.toml b/acceptance/bundle/resources/dashboards/simple_outside_bundle_root/out.test.toml index 7edd52865f7..dce76f03654 100644 --- a/acceptance/bundle/resources/dashboards/simple_outside_bundle_root/out.test.toml +++ b/acceptance/bundle/resources/dashboards/simple_outside_bundle_root/out.test.toml @@ -2,4 +2,5 @@ Local = true Cloud = true RequiresWarehouse = true RunsOnDbr = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/dashboards/simple_syncroot/out.test.toml b/acceptance/bundle/resources/dashboards/simple_syncroot/out.test.toml index 7edd52865f7..dce76f03654 100644 --- a/acceptance/bundle/resources/dashboards/simple_syncroot/out.test.toml +++ b/acceptance/bundle/resources/dashboards/simple_syncroot/out.test.toml @@ -2,4 +2,5 @@ Local = true Cloud = true RequiresWarehouse = true RunsOnDbr = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/dashboards/unpublish-out-of-band/out.test.toml b/acceptance/bundle/resources/dashboards/unpublish-out-of-band/out.test.toml index 7edd52865f7..dce76f03654 100644 --- a/acceptance/bundle/resources/dashboards/unpublish-out-of-band/out.test.toml +++ b/acceptance/bundle/resources/dashboards/unpublish-out-of-band/out.test.toml @@ -2,4 +2,5 @@ Local = true Cloud = true RequiresWarehouse = true RunsOnDbr = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/database_catalogs/basic/out.test.toml b/acceptance/bundle/resources/database_catalogs/basic/out.test.toml index 1f9d1fc1b75..281190c64eb 100644 --- a/acceptance/bundle/resources/database_catalogs/basic/out.test.toml +++ b/acceptance/bundle/resources/database_catalogs/basic/out.test.toml @@ -4,4 +4,5 @@ CloudSlow = true RequiresUnityCatalog = true RunsOnDbr = true CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/database_catalogs/recreate/out.test.toml b/acceptance/bundle/resources/database_catalogs/recreate/out.test.toml index c777e3ce206..9e11601cb89 100644 --- a/acceptance/bundle/resources/database_catalogs/recreate/out.test.toml +++ b/acceptance/bundle/resources/database_catalogs/recreate/out.test.toml @@ -2,4 +2,5 @@ Local = true Cloud = false RequiresUnityCatalog = true CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/database_instances/recreate/out.test.toml b/acceptance/bundle/resources/database_instances/recreate/out.test.toml index c777e3ce206..9e11601cb89 100644 --- a/acceptance/bundle/resources/database_instances/recreate/out.test.toml +++ b/acceptance/bundle/resources/database_instances/recreate/out.test.toml @@ -2,4 +2,5 @@ Local = true Cloud = false RequiresUnityCatalog = true CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/database_instances/single-instance/out.test.toml b/acceptance/bundle/resources/database_instances/single-instance/out.test.toml index 12ab4ea7f78..d093a69af64 100644 --- a/acceptance/bundle/resources/database_instances/single-instance/out.test.toml +++ b/acceptance/bundle/resources/database_instances/single-instance/out.test.toml @@ -3,4 +3,5 @@ Cloud = true CloudSlow = true RequiresUnityCatalog = true CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/experiments/basic/out.test.toml b/acceptance/bundle/resources/experiments/basic/out.test.toml index 5ad0addb75e..c1cae9fcc3e 100644 --- a/acceptance/bundle/resources/experiments/basic/out.test.toml +++ b/acceptance/bundle/resources/experiments/basic/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RunsOnDbr = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/external_locations/out.test.toml b/acceptance/bundle/resources/external_locations/out.test.toml index 88423408186..e3153c50fff 100644 --- a/acceptance/bundle/resources/external_locations/out.test.toml +++ b/acceptance/bundle/resources/external_locations/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/genie_spaces/delete_warning/out.test.toml b/acceptance/bundle/resources/genie_spaces/delete_warning/out.test.toml index a29f11b9ab2..88d62d0ff13 100644 --- a/acceptance/bundle/resources/genie_spaces/delete_warning/out.test.toml +++ b/acceptance/bundle/resources/genie_spaces/delete_warning/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false RequiresWarehouse = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/genie_spaces/inline/out.test.toml b/acceptance/bundle/resources/genie_spaces/inline/out.test.toml index a29f11b9ab2..88d62d0ff13 100644 --- a/acceptance/bundle/resources/genie_spaces/inline/out.test.toml +++ b/acceptance/bundle/resources/genie_spaces/inline/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false RequiresWarehouse = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/genie_spaces/parent_path_update/out.test.toml b/acceptance/bundle/resources/genie_spaces/parent_path_update/out.test.toml index 6072bc71acd..a1c0b99f373 100644 --- a/acceptance/bundle/resources/genie_spaces/parent_path_update/out.test.toml +++ b/acceptance/bundle/resources/genie_spaces/parent_path_update/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RequiresWarehouse = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/genie_spaces/recreate_when_gone/out.test.toml b/acceptance/bundle/resources/genie_spaces/recreate_when_gone/out.test.toml index 6072bc71acd..a1c0b99f373 100644 --- a/acceptance/bundle/resources/genie_spaces/recreate_when_gone/out.test.toml +++ b/acceptance/bundle/resources/genie_spaces/recreate_when_gone/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RequiresWarehouse = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/genie_spaces/serialized_space/out.test.toml b/acceptance/bundle/resources/genie_spaces/serialized_space/out.test.toml index a29f11b9ab2..88d62d0ff13 100644 --- a/acceptance/bundle/resources/genie_spaces/serialized_space/out.test.toml +++ b/acceptance/bundle/resources/genie_spaces/serialized_space/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false RequiresWarehouse = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/genie_spaces/simple/out.test.toml b/acceptance/bundle/resources/genie_spaces/simple/out.test.toml index 6072bc71acd..a1c0b99f373 100644 --- a/acceptance/bundle/resources/genie_spaces/simple/out.test.toml +++ b/acceptance/bundle/resources/genie_spaces/simple/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RequiresWarehouse = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/genie_spaces/version_migration/out.test.toml b/acceptance/bundle/resources/genie_spaces/version_migration/out.test.toml index 6072bc71acd..a1c0b99f373 100644 --- a/acceptance/bundle/resources/genie_spaces/version_migration/out.test.toml +++ b/acceptance/bundle/resources/genie_spaces/version_migration/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RequiresWarehouse = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/grants/catalogs/out.test.toml b/acceptance/bundle/resources/grants/catalogs/out.test.toml index fe4076cdf9b..ce8dec17c30 100644 --- a/acceptance/bundle/resources/grants/catalogs/out.test.toml +++ b/acceptance/bundle/resources/grants/catalogs/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/grants/registered_models/out.test.toml b/acceptance/bundle/resources/grants/registered_models/out.test.toml index e849ec85ace..fdd9f954e6c 100644 --- a/acceptance/bundle/resources/grants/registered_models/out.test.toml +++ b/acceptance/bundle/resources/grants/registered_models/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/grants/schemas/all_privileges/out.test.toml b/acceptance/bundle/resources/grants/schemas/all_privileges/out.test.toml index e849ec85ace..fdd9f954e6c 100644 --- a/acceptance/bundle/resources/grants/schemas/all_privileges/out.test.toml +++ b/acceptance/bundle/resources/grants/schemas/all_privileges/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/grants/schemas/all_privileges_coexist/out.test.toml b/acceptance/bundle/resources/grants/schemas/all_privileges_coexist/out.test.toml index e849ec85ace..fdd9f954e6c 100644 --- a/acceptance/bundle/resources/grants/schemas/all_privileges_coexist/out.test.toml +++ b/acceptance/bundle/resources/grants/schemas/all_privileges_coexist/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/grants/schemas/change_privilege/out.test.toml b/acceptance/bundle/resources/grants/schemas/change_privilege/out.test.toml index e849ec85ace..fdd9f954e6c 100644 --- a/acceptance/bundle/resources/grants/schemas/change_privilege/out.test.toml +++ b/acceptance/bundle/resources/grants/schemas/change_privilege/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/grants/schemas/duplicate_principals/out.test.toml b/acceptance/bundle/resources/grants/schemas/duplicate_principals/out.test.toml index e849ec85ace..fdd9f954e6c 100644 --- a/acceptance/bundle/resources/grants/schemas/duplicate_principals/out.test.toml +++ b/acceptance/bundle/resources/grants/schemas/duplicate_principals/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/grants/schemas/duplicate_privileges/out.test.toml b/acceptance/bundle/resources/grants/schemas/duplicate_privileges/out.test.toml index e849ec85ace..fdd9f954e6c 100644 --- a/acceptance/bundle/resources/grants/schemas/duplicate_privileges/out.test.toml +++ b/acceptance/bundle/resources/grants/schemas/duplicate_privileges/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/grants/schemas/empty_array/out.test.toml b/acceptance/bundle/resources/grants/schemas/empty_array/out.test.toml index e849ec85ace..fdd9f954e6c 100644 --- a/acceptance/bundle/resources/grants/schemas/empty_array/out.test.toml +++ b/acceptance/bundle/resources/grants/schemas/empty_array/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/grants/schemas/out_of_band_principal/out.test.toml b/acceptance/bundle/resources/grants/schemas/out_of_band_principal/out.test.toml index e849ec85ace..fdd9f954e6c 100644 --- a/acceptance/bundle/resources/grants/schemas/out_of_band_principal/out.test.toml +++ b/acceptance/bundle/resources/grants/schemas/out_of_band_principal/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/grants/schemas/remove_all/out.test.toml b/acceptance/bundle/resources/grants/schemas/remove_all/out.test.toml index e849ec85ace..fdd9f954e6c 100644 --- a/acceptance/bundle/resources/grants/schemas/remove_all/out.test.toml +++ b/acceptance/bundle/resources/grants/schemas/remove_all/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/grants/schemas/remove_principal/out.test.toml b/acceptance/bundle/resources/grants/schemas/remove_principal/out.test.toml index e849ec85ace..fdd9f954e6c 100644 --- a/acceptance/bundle/resources/grants/schemas/remove_principal/out.test.toml +++ b/acceptance/bundle/resources/grants/schemas/remove_principal/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/grants/volumes/out.test.toml b/acceptance/bundle/resources/grants/volumes/out.test.toml index e849ec85ace..fdd9f954e6c 100644 --- a/acceptance/bundle/resources/grants/volumes/out.test.toml +++ b/acceptance/bundle/resources/grants/volumes/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/independent/out.test.toml b/acceptance/bundle/resources/independent/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/independent/out.test.toml +++ b/acceptance/bundle/resources/independent/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/instance_pools/out.test.toml b/acceptance/bundle/resources/instance_pools/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/resources/instance_pools/out.test.toml +++ b/acceptance/bundle/resources/instance_pools/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/job_runs/basic/out.test.toml b/acceptance/bundle/resources/job_runs/basic/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/resources/job_runs/basic/out.test.toml +++ b/acceptance/bundle/resources/job_runs/basic/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/job_runs/job_parameters/out.test.toml b/acceptance/bundle/resources/job_runs/job_parameters/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/resources/job_runs/job_parameters/out.test.toml +++ b/acceptance/bundle/resources/job_runs/job_parameters/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/job_runs/redeploy/out.test.toml b/acceptance/bundle/resources/job_runs/redeploy/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/resources/job_runs/redeploy/out.test.toml +++ b/acceptance/bundle/resources/job_runs/redeploy/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/jobs/alert-task/out.test.toml b/acceptance/bundle/resources/jobs/alert-task/out.test.toml index bbc7fcfd1bd..78c615e06e5 100644 --- a/acceptance/bundle/resources/jobs/alert-task/out.test.toml +++ b/acceptance/bundle/resources/jobs/alert-task/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/jobs/big_id/out.test.toml b/acceptance/bundle/resources/jobs/big_id/out.test.toml index 71970b719d4..310be221793 100644 --- a/acceptance/bundle/resources/jobs/big_id/out.test.toml +++ b/acceptance/bundle/resources/jobs/big_id/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/resources/jobs/big_id/output.txt b/acceptance/bundle/resources/jobs/big_id/output.txt index 82ec469ca83..c539d037f79 100644 --- a/acceptance/bundle/resources/jobs/big_id/output.txt +++ b/acceptance/bundle/resources/jobs/big_id/output.txt @@ -7,7 +7,7 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> print_requests.py //jobs +>>> print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id,new_settings.deployment.deployment_id,new_settings.deployment.version_id { "method": "POST", "path": "/api/2.2/jobs/create", @@ -70,7 +70,7 @@ All files and directories at the following location will be deleted: /Workspace/ Deleting files... Destroy complete! ->>> print_requests.py //jobs +>>> print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id,new_settings.deployment.deployment_id,new_settings.deployment.version_id { "method": "POST", "path": "/api/2.2/jobs/delete", diff --git a/acceptance/bundle/resources/jobs/big_id/script b/acceptance/bundle/resources/jobs/big_id/script index 0b803cb1b8d..6f0e1215c6a 100644 --- a/acceptance/bundle/resources/jobs/big_id/script +++ b/acceptance/bundle/resources/jobs/big_id/script @@ -1,8 +1,8 @@ trace $CLI bundle validate -o json | jq .resources > out.validate.json trace $CLI bundle plan -o json > out.plan.$DATABRICKS_BUNDLE_ENGINE.json $CLI bundle deploy $(readplanarg out.plan.direct.json) -trace print_requests.py //jobs +trace print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id,new_settings.deployment.deployment_id,new_settings.deployment.version_id print_state.py > out.state.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle plan | contains.py '0 to add, 0 to change, 0 to delete' trace $CLI bundle destroy --auto-approve -trace print_requests.py //jobs +trace print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id,new_settings.deployment.deployment_id,new_settings.deployment.version_id diff --git a/acceptance/bundle/resources/jobs/big_id/test.toml b/acceptance/bundle/resources/jobs/big_id/test.toml index 6dd4efd85d3..467d0855fa4 100644 --- a/acceptance/bundle/resources/jobs/big_id/test.toml +++ b/acceptance/bundle/resources/jobs/big_id/test.toml @@ -3,6 +3,12 @@ EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ['direct'] EnvMatrix.READPLAN = ["", "1"] +# `bundle deploy --plan` applies the resource state the plan was saved with, and the plan +# is written before the deployment version exists, so the deployment stamp never reaches +# the applied resource and the next plan reports it as a change. Recording is skipped here +# until the stamp is written into the saved plan too. +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] + [[Repls]] Old = '9223372036854775807' New = '[MAX_INT_64]' diff --git a/acceptance/bundle/resources/jobs/check-metadata/out.test.toml b/acceptance/bundle/resources/jobs/check-metadata/out.test.toml index bbc7fcfd1bd..78c615e06e5 100644 --- a/acceptance/bundle/resources/jobs/check-metadata/out.test.toml +++ b/acceptance/bundle/resources/jobs/check-metadata/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/jobs/create-error/out.test.toml b/acceptance/bundle/resources/jobs/create-error/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/resources/jobs/create-error/out.test.toml +++ b/acceptance/bundle/resources/jobs/create-error/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/jobs/delete_job/out.test.toml b/acceptance/bundle/resources/jobs/delete_job/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/jobs/delete_job/out.test.toml +++ b/acceptance/bundle/resources/jobs/delete_job/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/jobs/delete_job/script b/acceptance/bundle/resources/jobs/delete_job/script index c9242b9e209..c3ce3c66802 100644 --- a/acceptance/bundle/resources/jobs/delete_job/script +++ b/acceptance/bundle/resources/jobs/delete_job/script @@ -1,5 +1,5 @@ trace $CLI bundle deploy cp empty.yml databricks.yml -$CLI bundle plan -o json > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | jq 'del(.. | objects | .deployment_id?, .version_id?)' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy -print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id +print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id,new_settings.deployment.deployment_id,new_settings.deployment.version_id diff --git a/acceptance/bundle/resources/jobs/delete_task/out.test.toml b/acceptance/bundle/resources/jobs/delete_task/out.test.toml index 8ffbd40f24c..57abc73cb92 100644 --- a/acceptance/bundle/resources/jobs/delete_task/out.test.toml +++ b/acceptance/bundle/resources/jobs/delete_task/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/resources/jobs/delete_task/test.toml b/acceptance/bundle/resources/jobs/delete_task/test.toml index d4bbeb7e7ef..cca4fd1ac9c 100644 --- a/acceptance/bundle/resources/jobs/delete_task/test.toml +++ b/acceptance/bundle/resources/jobs/delete_task/test.toml @@ -1,2 +1,8 @@ RecordRequests = false EnvMatrix.READPLAN = ["", "1"] +# `bundle deploy --plan` applies the resource state the plan was saved with, and the plan +# is written before the deployment version exists, so the deployment stamp never reaches +# the applied resource and the next plan reports it as a change. Recording is skipped here +# until the stamp is written into the saved plan too. +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] + diff --git a/acceptance/bundle/resources/jobs/double-underscore-keys/out.test.toml b/acceptance/bundle/resources/jobs/double-underscore-keys/out.test.toml index 5ad0addb75e..c1cae9fcc3e 100644 --- a/acceptance/bundle/resources/jobs/double-underscore-keys/out.test.toml +++ b/acceptance/bundle/resources/jobs/double-underscore-keys/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RunsOnDbr = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/jobs/fail-on-active-runs/out.test.toml b/acceptance/bundle/resources/jobs/fail-on-active-runs/out.test.toml index 5ad0addb75e..c1cae9fcc3e 100644 --- a/acceptance/bundle/resources/jobs/fail-on-active-runs/out.test.toml +++ b/acceptance/bundle/resources/jobs/fail-on-active-runs/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RunsOnDbr = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/jobs/instance_pool_and_node_type/out.test.toml b/acceptance/bundle/resources/jobs/instance_pool_and_node_type/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/jobs/instance_pool_and_node_type/out.test.toml +++ b/acceptance/bundle/resources/jobs/instance_pool_and_node_type/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/jobs/no-git-provider/out.test.toml b/acceptance/bundle/resources/jobs/no-git-provider/out.test.toml index 5ad0addb75e..c1cae9fcc3e 100644 --- a/acceptance/bundle/resources/jobs/no-git-provider/out.test.toml +++ b/acceptance/bundle/resources/jobs/no-git-provider/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RunsOnDbr = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/jobs/num_workers/out.test.toml b/acceptance/bundle/resources/jobs/num_workers/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/jobs/num_workers/out.test.toml +++ b/acceptance/bundle/resources/jobs/num_workers/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/jobs/num_workers/output.txt b/acceptance/bundle/resources/jobs/num_workers/output.txt index f61ab034296..702558444e9 100644 --- a/acceptance/bundle/resources/jobs/num_workers/output.txt +++ b/acceptance/bundle/resources/jobs/num_workers/output.txt @@ -21,7 +21,7 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id +>>> print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id,new_settings.deployment.deployment_id,new_settings.deployment.version_id { "method": "POST", "path": "/api/2.2/jobs/create", diff --git a/acceptance/bundle/resources/jobs/num_workers/script b/acceptance/bundle/resources/jobs/num_workers/script index 83d9321bcc4..674a820061c 100644 --- a/acceptance/bundle/resources/jobs/num_workers/script +++ b/acceptance/bundle/resources/jobs/num_workers/script @@ -1,6 +1,6 @@ envsubst < databricks.yml.tmpl > databricks.yml trace $CLI bundle deploy -trace print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id +trace print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id,new_settings.deployment.deployment_id,new_settings.deployment.version_id trace $CLI bundle plan rm out.requests.txt diff --git a/acceptance/bundle/resources/jobs/on_failure_empty_slice/out.test.toml b/acceptance/bundle/resources/jobs/on_failure_empty_slice/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/jobs/on_failure_empty_slice/out.test.toml +++ b/acceptance/bundle/resources/jobs/on_failure_empty_slice/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/jobs/remote_add_tag/out.test.toml b/acceptance/bundle/resources/jobs/remote_add_tag/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/jobs/remote_add_tag/out.test.toml +++ b/acceptance/bundle/resources/jobs/remote_add_tag/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/jobs/remote_add_tag/script b/acceptance/bundle/resources/jobs/remote_add_tag/script index d7593e7b8f7..37a37b0059f 100644 --- a/acceptance/bundle/resources/jobs/remote_add_tag/script +++ b/acceptance/bundle/resources/jobs/remote_add_tag/script @@ -8,4 +8,4 @@ r["tags"]["new_tag"] = "new_value" EOF $CLI bundle plan -$CLI bundle plan -o json > out.plan_post_update.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | jq 'del(.. | objects | .deployment_id?, .version_id?)' > out.plan_post_update.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/jobs/remote_delete/deploy/out.test.toml b/acceptance/bundle/resources/jobs/remote_delete/deploy/out.test.toml index 8ffbd40f24c..57abc73cb92 100644 --- a/acceptance/bundle/resources/jobs/remote_delete/deploy/out.test.toml +++ b/acceptance/bundle/resources/jobs/remote_delete/deploy/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/resources/jobs/remote_delete/deploy/test.toml b/acceptance/bundle/resources/jobs/remote_delete/deploy/test.toml index d4bbeb7e7ef..cca4fd1ac9c 100644 --- a/acceptance/bundle/resources/jobs/remote_delete/deploy/test.toml +++ b/acceptance/bundle/resources/jobs/remote_delete/deploy/test.toml @@ -1,2 +1,8 @@ RecordRequests = false EnvMatrix.READPLAN = ["", "1"] +# `bundle deploy --plan` applies the resource state the plan was saved with, and the plan +# is written before the deployment version exists, so the deployment stamp never reaches +# the applied resource and the next plan reports it as a change. Recording is skipped here +# until the stamp is written into the saved plan too. +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] + diff --git a/acceptance/bundle/resources/jobs/remote_delete/destroy/out.test.toml b/acceptance/bundle/resources/jobs/remote_delete/destroy/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/jobs/remote_delete/destroy/out.test.toml +++ b/acceptance/bundle/resources/jobs/remote_delete/destroy/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/jobs/remote_delete/removed_from_config/out.test.toml b/acceptance/bundle/resources/jobs/remote_delete/removed_from_config/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/jobs/remote_delete/removed_from_config/out.test.toml +++ b/acceptance/bundle/resources/jobs/remote_delete/removed_from_config/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/jobs/remote_delete/removed_from_config/output.txt b/acceptance/bundle/resources/jobs/remote_delete/removed_from_config/output.txt index c97c93273cb..45a761af80e 100644 --- a/acceptance/bundle/resources/jobs/remote_delete/removed_from_config/output.txt +++ b/acceptance/bundle/resources/jobs/remote_delete/removed_from_config/output.txt @@ -21,4 +21,4 @@ Updating deployment state... Deployment complete! === No delete API calls for resources that are already gone remotely ->>> print_requests.py //jobs/delete //pipelines/ +>>> print_requests.py //jobs/delete //pipelines/ --del-body deployment.deployment_id,deployment.version_id,new_settings.deployment.deployment_id,new_settings.deployment.version_id diff --git a/acceptance/bundle/resources/jobs/remote_delete/removed_from_config/script b/acceptance/bundle/resources/jobs/remote_delete/removed_from_config/script index 2b032406a00..45098c8a690 100644 --- a/acceptance/bundle/resources/jobs/remote_delete/removed_from_config/script +++ b/acceptance/bundle/resources/jobs/remote_delete/removed_from_config/script @@ -18,4 +18,4 @@ trace $CLI bundle deploy trace $CLI bundle summary &> out.summary.$DATABRICKS_BUNDLE_ENGINE.txt title "No delete API calls for resources that are already gone remotely" -trace print_requests.py //jobs/delete //pipelines/ +trace print_requests.py //jobs/delete //pipelines/ --del-body deployment.deployment_id,deployment.version_id,new_settings.deployment.deployment_id,new_settings.deployment.version_id diff --git a/acceptance/bundle/resources/jobs/remote_matches_config/out.test.toml b/acceptance/bundle/resources/jobs/remote_matches_config/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/jobs/remote_matches_config/out.test.toml +++ b/acceptance/bundle/resources/jobs/remote_matches_config/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/jobs/remote_matches_config/output.txt b/acceptance/bundle/resources/jobs/remote_matches_config/output.txt index 9ce9dba13c1..d1f2a5cf7d5 100644 --- a/acceptance/bundle/resources/jobs/remote_matches_config/output.txt +++ b/acceptance/bundle/resources/jobs/remote_matches_config/output.txt @@ -23,4 +23,4 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id +>>> print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id,new_settings.deployment.deployment_id,new_settings.deployment.version_id diff --git a/acceptance/bundle/resources/jobs/remote_matches_config/script b/acceptance/bundle/resources/jobs/remote_matches_config/script index 972181ba497..b528539bf92 100755 --- a/acceptance/bundle/resources/jobs/remote_matches_config/script +++ b/acceptance/bundle/resources/jobs/remote_matches_config/script @@ -13,9 +13,9 @@ r["max_concurrent_runs"] = 2 EOF trace $CLI bundle plan -$CLI bundle plan -o json > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | jq 'del(.. | objects | .deployment_id?, .version_id?)' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json rm out.requests.txt # XXX READPLAN trace $CLI bundle deploy -trace print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id +trace print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id,new_settings.deployment.deployment_id,new_settings.deployment.version_id diff --git a/acceptance/bundle/resources/jobs/shared-root-path/out.test.toml b/acceptance/bundle/resources/jobs/shared-root-path/out.test.toml index 5ad0addb75e..c1cae9fcc3e 100644 --- a/acceptance/bundle/resources/jobs/shared-root-path/out.test.toml +++ b/acceptance/bundle/resources/jobs/shared-root-path/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RunsOnDbr = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/jobs/tags_empty_map/out.test.toml b/acceptance/bundle/resources/jobs/tags_empty_map/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/jobs/tags_empty_map/out.test.toml +++ b/acceptance/bundle/resources/jobs/tags_empty_map/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/jobs/task-source/out.test.toml b/acceptance/bundle/resources/jobs/task-source/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/jobs/task-source/out.test.toml +++ b/acceptance/bundle/resources/jobs/task-source/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/jobs/tasks-reorder-locally/out.test.toml b/acceptance/bundle/resources/jobs/tasks-reorder-locally/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/jobs/tasks-reorder-locally/out.test.toml +++ b/acceptance/bundle/resources/jobs/tasks-reorder-locally/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/jobs/unknown-terraform-field/out.test.toml b/acceptance/bundle/resources/jobs/unknown-terraform-field/out.test.toml index 65156e0457c..76ce926fd59 100644 --- a/acceptance/bundle/resources/jobs/unknown-terraform-field/out.test.toml +++ b/acceptance/bundle/resources/jobs/unknown-terraform-field/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform"] diff --git a/acceptance/bundle/resources/jobs/update/out.test.toml b/acceptance/bundle/resources/jobs/update/out.test.toml index 8ffbd40f24c..57abc73cb92 100644 --- a/acceptance/bundle/resources/jobs/update/out.test.toml +++ b/acceptance/bundle/resources/jobs/update/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/resources/jobs/update/output.txt b/acceptance/bundle/resources/jobs/update/output.txt index eea83d272d5..f6a44af17c6 100644 --- a/acceptance/bundle/resources/jobs/update/output.txt +++ b/acceptance/bundle/resources/jobs/update/output.txt @@ -8,7 +8,7 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> print_requests.py //jobs +>>> print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id,new_settings.deployment.deployment_id,new_settings.deployment.version_id >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged @@ -19,7 +19,7 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> print_requests.py //jobs +>>> print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id,new_settings.deployment.deployment_id,new_settings.deployment.version_id === Update trigger.periodic.unit and re-deploy >>> update_file.py databricks.yml DAYS HOURS @@ -33,7 +33,7 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> print_requests.py //jobs +>>> print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id,new_settings.deployment.deployment_id,new_settings.deployment.version_id >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged @@ -89,7 +89,7 @@ All files and directories at the following location will be deleted: /Workspace/ Deleting files... Destroy complete! ->>> print_requests.py //jobs +>>> print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id,new_settings.deployment.deployment_id,new_settings.deployment.version_id { "method": "POST", "path": "/api/2.2/jobs/delete", diff --git a/acceptance/bundle/resources/jobs/update/script b/acceptance/bundle/resources/jobs/update/script index 270efcaea6d..c15b425741c 100644 --- a/acceptance/bundle/resources/jobs/update/script +++ b/acceptance/bundle/resources/jobs/update/script @@ -2,14 +2,14 @@ echo "*" > .gitignore trace $CLI bundle plan $CLI bundle plan -o json > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json $CLI bundle deploy $(readplanarg out.plan_create.direct.json) -trace print_requests.py //jobs > out.create.requests.json +trace print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id,new_settings.deployment.deployment_id,new_settings.deployment.version_id > out.create.requests.json print_state.py > out.state.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle plan trace $CLI bundle plan -o json > out.plan_skip.$DATABRICKS_BUNDLE_ENGINE.json $CLI bundle deploy $(readplanarg out.plan_skip.direct.json) -trace print_requests.py //jobs +trace print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id,new_settings.deployment.deployment_id,new_settings.deployment.version_id title "Update trigger.periodic.unit and re-deploy" trace update_file.py databricks.yml DAYS HOURS @@ -17,7 +17,7 @@ trace $CLI bundle plan $CLI bundle plan -o json > out.plan_update.$DATABRICKS_BUNDLE_ENGINE.json $CLI bundle deploy $(readplanarg out.plan_update.direct.json) -trace print_requests.py //jobs | jq 'del(.body.new_settings.run_as, .body.new_settings.webhook_notifications, .body.new_settings.email_notifications)' > out.update.requests.json +trace print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id,new_settings.deployment.deployment_id,new_settings.deployment.version_id | jq 'del(.body.new_settings.run_as, .body.new_settings.webhook_notifications, .body.new_settings.email_notifications)' > out.update.requests.json trace $CLI bundle plan @@ -30,7 +30,7 @@ rm out.requests.txt title "Destroy the job and verify that it's removed from the state and from remote" trace $CLI bundle destroy --auto-approve -trace print_requests.py //jobs +trace print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id,new_settings.deployment.deployment_id,new_settings.deployment.version_id trace musterr $CLI jobs get $ppid rm out.requests.txt diff --git a/acceptance/bundle/resources/jobs/update/test.toml b/acceptance/bundle/resources/jobs/update/test.toml index ff8a66c196e..a8a9a1e90b9 100644 --- a/acceptance/bundle/resources/jobs/update/test.toml +++ b/acceptance/bundle/resources/jobs/update/test.toml @@ -1 +1,7 @@ EnvMatrix.READPLAN = ["", "1"] +# `bundle deploy --plan` applies the resource state the plan was saved with, and the plan +# is written before the deployment version exists, so the deployment stamp never reaches +# the applied resource and the next plan reports it as a change. Recording is skipped here +# until the stamp is written into the saved plan too. +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] + diff --git a/acceptance/bundle/resources/jobs/update_single_node/out.test.toml b/acceptance/bundle/resources/jobs/update_single_node/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/jobs/update_single_node/out.test.toml +++ b/acceptance/bundle/resources/jobs/update_single_node/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/jobs/update_single_node/output.txt b/acceptance/bundle/resources/jobs/update_single_node/output.txt index aba6e239b86..ec195ebc965 100644 --- a/acceptance/bundle/resources/jobs/update_single_node/output.txt +++ b/acceptance/bundle/resources/jobs/update_single_node/output.txt @@ -10,7 +10,7 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id +>>> print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id,new_settings.deployment.deployment_id,new_settings.deployment.version_id === Update trigger.periodic.unit and re-deploy >>> update_file.py databricks.yml DAYS HOURS @@ -26,7 +26,7 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id +>>> print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id,new_settings.deployment.deployment_id,new_settings.deployment.version_id >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged @@ -36,7 +36,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged { "created_time": [UNIX_TIME_MILLIS], "creator_user_name": "[USERNAME]", - "job_id": [NUMID], + "job_id": [FOO_ID], "run_as_user_name": "[USERNAME]", "settings": { "deployment": { @@ -89,7 +89,7 @@ All files and directories at the following location will be deleted: /Workspace/ Deleting files... Destroy complete! ->>> print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id +>>> print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id,new_settings.deployment.deployment_id,new_settings.deployment.version_id { "method": "POST", "path": "/api/2.2/jobs/delete", diff --git a/acceptance/bundle/resources/jobs/update_single_node/script b/acceptance/bundle/resources/jobs/update_single_node/script index 55ce937b978..d822881fe20 100644 --- a/acceptance/bundle/resources/jobs/update_single_node/script +++ b/acceptance/bundle/resources/jobs/update_single_node/script @@ -1,17 +1,17 @@ echo "*" > .gitignore trace $CLI bundle plan -$CLI bundle plan -o json > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | jq 'del(.. | objects | .deployment_id?, .version_id?)' > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy -$CLI bundle plan -o json > out.plan_skip.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | jq 'del(.. | objects | .deployment_id?, .version_id?)' > out.plan_skip.$DATABRICKS_BUNDLE_ENGINE.json -trace print_requests.py //jobs > out.create.requests.txt --del-body deployment.deployment_id,deployment.version_id +trace print_requests.py //jobs > out.create.requests.txt --del-body deployment.deployment_id,deployment.version_id,new_settings.deployment.deployment_id,new_settings.deployment.version_id title "Update trigger.periodic.unit and re-deploy" trace update_file.py databricks.yml DAYS HOURS trace $CLI bundle plan -$CLI bundle plan -o json > out.plan_update.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | jq 'del(.. | objects | .deployment_id?, .version_id?)' > out.plan_update.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy -trace print_requests.py //jobs > out.update.requests.$DATABRICKS_BUNDLE_ENGINE.json --del-body deployment.deployment_id,deployment.version_id +trace print_requests.py //jobs > out.update.requests.$DATABRICKS_BUNDLE_ENGINE.json --del-body deployment.deployment_id,deployment.version_id,new_settings.deployment.deployment_id,new_settings.deployment.version_id trace $CLI bundle plan @@ -19,12 +19,12 @@ title "Fetch job ID and verify remote state" ppid=`read_id.py foo` -trace $CLI jobs get $ppid | jq 'del(.settings.run_as)' +trace $CLI jobs get $ppid | jq 'del(.settings.run_as, .settings.deployment.deployment_id, .settings.deployment.version_id)' rm out.requests.txt title "Destroy the job and verify that it's removed from the state and from remote" trace $CLI bundle destroy --auto-approve -trace print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id +trace print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id,new_settings.deployment.deployment_id,new_settings.deployment.version_id trace musterr $CLI jobs get $ppid rm out.requests.txt diff --git a/acceptance/bundle/resources/jobs/webhook-reorder-remote/out.test.toml b/acceptance/bundle/resources/jobs/webhook-reorder-remote/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/resources/jobs/webhook-reorder-remote/out.test.toml +++ b/acceptance/bundle/resources/jobs/webhook-reorder-remote/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/jobs/webhook-reorder-remote/output.txt b/acceptance/bundle/resources/jobs/webhook-reorder-remote/output.txt index 31a759d0c51..2185aedcef6 100644 --- a/acceptance/bundle/resources/jobs/webhook-reorder-remote/output.txt +++ b/acceptance/bundle/resources/jobs/webhook-reorder-remote/output.txt @@ -16,7 +16,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged ->>> print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id +>>> print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id,new_settings.deployment.deployment_id,new_settings.deployment.version_id { "method": "POST", "path": "/api/2.2/jobs/create", diff --git a/acceptance/bundle/resources/jobs/webhook-reorder-remote/script b/acceptance/bundle/resources/jobs/webhook-reorder-remote/script index fda553c3cea..7d9103aca52 100644 --- a/acceptance/bundle/resources/jobs/webhook-reorder-remote/script +++ b/acceptance/bundle/resources/jobs/webhook-reorder-remote/script @@ -14,6 +14,6 @@ EOF # The reordered remote must not produce a phantom diff: on_* lists are diffed by id. trace $CLI bundle plan -$CLI bundle plan -o json | jq '.plan."resources.jobs.my_job".changes' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | jq '.plan."resources.jobs.my_job".changes | del(.. | objects | .deployment_id?, .version_id?)' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json -trace print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id +trace print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id,new_settings.deployment.deployment_id,new_settings.deployment.version_id diff --git a/acceptance/bundle/resources/model_serving_endpoints/basic/out.test.toml b/acceptance/bundle/resources/model_serving_endpoints/basic/out.test.toml index e849ec85ace..fdd9f954e6c 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/basic/out.test.toml +++ b/acceptance/bundle/resources/model_serving_endpoints/basic/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/model_serving_endpoints/drift/write_only/out.test.toml b/acceptance/bundle/resources/model_serving_endpoints/drift/write_only/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/drift/write_only/out.test.toml +++ b/acceptance/bundle/resources/model_serving_endpoints/drift/write_only/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/model_serving_endpoints/recreate/catalog-name/out.test.toml b/acceptance/bundle/resources/model_serving_endpoints/recreate/catalog-name/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/recreate/catalog-name/out.test.toml +++ b/acceptance/bundle/resources/model_serving_endpoints/recreate/catalog-name/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/model_serving_endpoints/recreate/name-change/out.test.toml b/acceptance/bundle/resources/model_serving_endpoints/recreate/name-change/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/recreate/name-change/out.test.toml +++ b/acceptance/bundle/resources/model_serving_endpoints/recreate/name-change/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/model_serving_endpoints/recreate/route-optimized/out.test.toml b/acceptance/bundle/resources/model_serving_endpoints/recreate/route-optimized/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/recreate/route-optimized/out.test.toml +++ b/acceptance/bundle/resources/model_serving_endpoints/recreate/route-optimized/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/model_serving_endpoints/recreate/schema-name/out.test.toml b/acceptance/bundle/resources/model_serving_endpoints/recreate/schema-name/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/recreate/schema-name/out.test.toml +++ b/acceptance/bundle/resources/model_serving_endpoints/recreate/schema-name/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/model_serving_endpoints/recreate/table-prefix/out.test.toml b/acceptance/bundle/resources/model_serving_endpoints/recreate/table-prefix/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/recreate/table-prefix/out.test.toml +++ b/acceptance/bundle/resources/model_serving_endpoints/recreate/table-prefix/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/model_serving_endpoints/running-endpoint/out.test.toml b/acceptance/bundle/resources/model_serving_endpoints/running-endpoint/out.test.toml index 12ab4ea7f78..d093a69af64 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/running-endpoint/out.test.toml +++ b/acceptance/bundle/resources/model_serving_endpoints/running-endpoint/out.test.toml @@ -3,4 +3,5 @@ Cloud = true CloudSlow = true RequiresUnityCatalog = true CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/model_serving_endpoints/update/ai-gateway/out.test.toml b/acceptance/bundle/resources/model_serving_endpoints/update/ai-gateway/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/update/ai-gateway/out.test.toml +++ b/acceptance/bundle/resources/model_serving_endpoints/update/ai-gateway/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/model_serving_endpoints/update/both_gateway_and_tags/out.test.toml b/acceptance/bundle/resources/model_serving_endpoints/update/both_gateway_and_tags/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/update/both_gateway_and_tags/out.test.toml +++ b/acceptance/bundle/resources/model_serving_endpoints/update/both_gateway_and_tags/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/model_serving_endpoints/update/config/out.test.toml b/acceptance/bundle/resources/model_serving_endpoints/update/config/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/update/config/out.test.toml +++ b/acceptance/bundle/resources/model_serving_endpoints/update/config/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/model_serving_endpoints/update/email-notifications/out.test.toml b/acceptance/bundle/resources/model_serving_endpoints/update/email-notifications/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/update/email-notifications/out.test.toml +++ b/acceptance/bundle/resources/model_serving_endpoints/update/email-notifications/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/model_serving_endpoints/update/tags/out.test.toml b/acceptance/bundle/resources/model_serving_endpoints/update/tags/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/update/tags/out.test.toml +++ b/acceptance/bundle/resources/model_serving_endpoints/update/tags/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/models/basic/out.test.toml b/acceptance/bundle/resources/models/basic/out.test.toml index 5ad0addb75e..c1cae9fcc3e 100644 --- a/acceptance/bundle/resources/models/basic/out.test.toml +++ b/acceptance/bundle/resources/models/basic/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RunsOnDbr = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/models/empty-name/out.test.toml b/acceptance/bundle/resources/models/empty-name/out.test.toml index bbc7fcfd1bd..78c615e06e5 100644 --- a/acceptance/bundle/resources/models/empty-name/out.test.toml +++ b/acceptance/bundle/resources/models/empty-name/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/models/readplan-permissions/out.test.toml b/acceptance/bundle/resources/models/readplan-permissions/out.test.toml index 71970b719d4..2962c9963cc 100644 --- a/acceptance/bundle/resources/models/readplan-permissions/out.test.toml +++ b/acceptance/bundle/resources/models/readplan-permissions/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/resources/permissions/apps/current_can_manage/out.test.toml b/acceptance/bundle/resources/permissions/apps/current_can_manage/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/permissions/apps/current_can_manage/out.test.toml +++ b/acceptance/bundle/resources/permissions/apps/current_can_manage/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/permissions/apps/other_can_manage/out.test.toml b/acceptance/bundle/resources/permissions/apps/other_can_manage/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/permissions/apps/other_can_manage/out.test.toml +++ b/acceptance/bundle/resources/permissions/apps/other_can_manage/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/permissions/clusters/current_can_manage/out.test.toml b/acceptance/bundle/resources/permissions/clusters/current_can_manage/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/permissions/clusters/current_can_manage/out.test.toml +++ b/acceptance/bundle/resources/permissions/clusters/current_can_manage/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/permissions/clusters/target/out.test.toml b/acceptance/bundle/resources/permissions/clusters/target/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/permissions/clusters/target/out.test.toml +++ b/acceptance/bundle/resources/permissions/clusters/target/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/permissions/dashboards/create/out.test.toml b/acceptance/bundle/resources/permissions/dashboards/create/out.test.toml index 7845c49f70c..973ce7c68cf 100644 --- a/acceptance/bundle/resources/permissions/dashboards/create/out.test.toml +++ b/acceptance/bundle/resources/permissions/dashboards/create/out.test.toml @@ -2,4 +2,5 @@ Local = true Cloud = true RequiresWarehouse = true CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/permissions/database_instances/current_can_manage/out.test.toml b/acceptance/bundle/resources/permissions/database_instances/current_can_manage/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/permissions/database_instances/current_can_manage/out.test.toml +++ b/acceptance/bundle/resources/permissions/database_instances/current_can_manage/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/permissions/experiments/current_can_manage/out.test.toml b/acceptance/bundle/resources/permissions/experiments/current_can_manage/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/permissions/experiments/current_can_manage/out.test.toml +++ b/acceptance/bundle/resources/permissions/experiments/current_can_manage/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/permissions/factcheck/out.test.toml b/acceptance/bundle/resources/permissions/factcheck/out.test.toml index 581c975b773..64851222e54 100644 --- a/acceptance/bundle/resources/permissions/factcheck/out.test.toml +++ b/acceptance/bundle/resources/permissions/factcheck/out.test.toml @@ -3,4 +3,5 @@ Cloud = true CloudSlow = true RunsOnDbr = true CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/out.test.toml b/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/out.test.toml +++ b/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/permissions/genie_spaces/out_of_band_deletion/out.test.toml b/acceptance/bundle/resources/permissions/genie_spaces/out_of_band_deletion/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/resources/permissions/genie_spaces/out_of_band_deletion/out.test.toml +++ b/acceptance/bundle/resources/permissions/genie_spaces/out_of_band_deletion/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/permissions/jobs/added_remotely/out.test.toml b/acceptance/bundle/resources/permissions/jobs/added_remotely/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/permissions/jobs/added_remotely/out.test.toml +++ b/acceptance/bundle/resources/permissions/jobs/added_remotely/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/permissions/jobs/current_can_manage/out.test.toml b/acceptance/bundle/resources/permissions/jobs/current_can_manage/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/permissions/jobs/current_can_manage/out.test.toml +++ b/acceptance/bundle/resources/permissions/jobs/current_can_manage/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/permissions/jobs/current_can_manage_run/out.test.toml b/acceptance/bundle/resources/permissions/jobs/current_can_manage_run/out.test.toml index 887e4650a78..0b68a00d7ba 100644 --- a/acceptance/bundle/resources/permissions/jobs/current_can_manage_run/out.test.toml +++ b/acceptance/bundle/resources/permissions/jobs/current_can_manage_run/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/permissions/jobs/current_is_owner/out.test.toml b/acceptance/bundle/resources/permissions/jobs/current_is_owner/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/permissions/jobs/current_is_owner/out.test.toml +++ b/acceptance/bundle/resources/permissions/jobs/current_is_owner/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/permissions/jobs/delete_one/out.test.toml b/acceptance/bundle/resources/permissions/jobs/delete_one/out.test.toml index e849ec85ace..fdd9f954e6c 100644 --- a/acceptance/bundle/resources/permissions/jobs/delete_one/out.test.toml +++ b/acceptance/bundle/resources/permissions/jobs/delete_one/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/permissions/jobs/deleted_remotely/out.test.toml b/acceptance/bundle/resources/permissions/jobs/deleted_remotely/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/permissions/jobs/deleted_remotely/out.test.toml +++ b/acceptance/bundle/resources/permissions/jobs/deleted_remotely/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/permissions/jobs/destroy_without_mgmtperms/with_permissions/out.test.toml b/acceptance/bundle/resources/permissions/jobs/destroy_without_mgmtperms/with_permissions/out.test.toml index 9b877617211..aa99ae397ac 100644 --- a/acceptance/bundle/resources/permissions/jobs/destroy_without_mgmtperms/with_permissions/out.test.toml +++ b/acceptance/bundle/resources/permissions/jobs/destroy_without_mgmtperms/with_permissions/out.test.toml @@ -3,4 +3,5 @@ Cloud = true RunsOnDbr = false CloudEnvs.azure = false CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/permissions/jobs/destroy_without_mgmtperms/without_permissions/out.test.toml b/acceptance/bundle/resources/permissions/jobs/destroy_without_mgmtperms/without_permissions/out.test.toml index 9b877617211..aa99ae397ac 100644 --- a/acceptance/bundle/resources/permissions/jobs/destroy_without_mgmtperms/without_permissions/out.test.toml +++ b/acceptance/bundle/resources/permissions/jobs/destroy_without_mgmtperms/without_permissions/out.test.toml @@ -3,4 +3,5 @@ Cloud = true RunsOnDbr = false CloudEnvs.azure = false CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/permissions/jobs/empty_list/out.test.toml b/acceptance/bundle/resources/permissions/jobs/empty_list/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/permissions/jobs/empty_list/out.test.toml +++ b/acceptance/bundle/resources/permissions/jobs/empty_list/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/permissions/jobs/other_can_manage/out.test.toml b/acceptance/bundle/resources/permissions/jobs/other_can_manage/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/permissions/jobs/other_can_manage/out.test.toml +++ b/acceptance/bundle/resources/permissions/jobs/other_can_manage/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/permissions/jobs/other_can_manage_run/out.test.toml b/acceptance/bundle/resources/permissions/jobs/other_can_manage_run/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/permissions/jobs/other_can_manage_run/out.test.toml +++ b/acceptance/bundle/resources/permissions/jobs/other_can_manage_run/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/permissions/jobs/other_is_owner/out.test.toml b/acceptance/bundle/resources/permissions/jobs/other_is_owner/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/permissions/jobs/other_is_owner/out.test.toml +++ b/acceptance/bundle/resources/permissions/jobs/other_is_owner/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/permissions/jobs/reorder_locally/out.test.toml b/acceptance/bundle/resources/permissions/jobs/reorder_locally/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/permissions/jobs/reorder_locally/out.test.toml +++ b/acceptance/bundle/resources/permissions/jobs/reorder_locally/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/permissions/jobs/reorder_remotely/out.test.toml b/acceptance/bundle/resources/permissions/jobs/reorder_remotely/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/permissions/jobs/reorder_remotely/out.test.toml +++ b/acceptance/bundle/resources/permissions/jobs/reorder_remotely/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/permissions/jobs/update/out.test.toml b/acceptance/bundle/resources/permissions/jobs/update/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/permissions/jobs/update/out.test.toml +++ b/acceptance/bundle/resources/permissions/jobs/update/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/permissions/jobs/viewers/out.test.toml b/acceptance/bundle/resources/permissions/jobs/viewers/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/permissions/jobs/viewers/out.test.toml +++ b/acceptance/bundle/resources/permissions/jobs/viewers/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/permissions/models/current_can_manage/out.test.toml b/acceptance/bundle/resources/permissions/models/current_can_manage/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/permissions/models/current_can_manage/out.test.toml +++ b/acceptance/bundle/resources/permissions/models/current_can_manage/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/permissions/out.test.toml b/acceptance/bundle/resources/permissions/out.test.toml index be193812ec2..b827ff3f062 100644 --- a/acceptance/bundle/resources/permissions/out.test.toml +++ b/acceptance/bundle/resources/permissions/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false Phase = 1 +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/permissions/pipelines/504/create/out.test.toml b/acceptance/bundle/resources/permissions/pipelines/504/create/out.test.toml index 7ffa3d15391..c6eec1a9063 100644 --- a/acceptance/bundle/resources/permissions/pipelines/504/create/out.test.toml +++ b/acceptance/bundle/resources/permissions/pipelines/504/create/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] EnvMatrix.READPLAN = [] diff --git a/acceptance/bundle/resources/permissions/pipelines/504/plan/out.test.toml b/acceptance/bundle/resources/permissions/pipelines/504/plan/out.test.toml index 7ffa3d15391..c6eec1a9063 100644 --- a/acceptance/bundle/resources/permissions/pipelines/504/plan/out.test.toml +++ b/acceptance/bundle/resources/permissions/pipelines/504/plan/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] EnvMatrix.READPLAN = [] diff --git a/acceptance/bundle/resources/permissions/pipelines/504/update/out.test.toml b/acceptance/bundle/resources/permissions/pipelines/504/update/out.test.toml index 7ffa3d15391..c6eec1a9063 100644 --- a/acceptance/bundle/resources/permissions/pipelines/504/update/out.test.toml +++ b/acceptance/bundle/resources/permissions/pipelines/504/update/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] EnvMatrix.READPLAN = [] diff --git a/acceptance/bundle/resources/permissions/pipelines/current_can_manage/out.test.toml b/acceptance/bundle/resources/permissions/pipelines/current_can_manage/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/permissions/pipelines/current_can_manage/out.test.toml +++ b/acceptance/bundle/resources/permissions/pipelines/current_can_manage/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/permissions/pipelines/current_is_owner/out.test.toml b/acceptance/bundle/resources/permissions/pipelines/current_is_owner/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/permissions/pipelines/current_is_owner/out.test.toml +++ b/acceptance/bundle/resources/permissions/pipelines/current_is_owner/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/permissions/pipelines/empty_list/out.test.toml b/acceptance/bundle/resources/permissions/pipelines/empty_list/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/permissions/pipelines/empty_list/out.test.toml +++ b/acceptance/bundle/resources/permissions/pipelines/empty_list/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/permissions/pipelines/other_can_manage/out.test.toml b/acceptance/bundle/resources/permissions/pipelines/other_can_manage/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/permissions/pipelines/other_can_manage/out.test.toml +++ b/acceptance/bundle/resources/permissions/pipelines/other_can_manage/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/permissions/pipelines/other_is_owner/out.test.toml b/acceptance/bundle/resources/permissions/pipelines/other_is_owner/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/permissions/pipelines/other_is_owner/out.test.toml +++ b/acceptance/bundle/resources/permissions/pipelines/other_is_owner/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/permissions/pipelines/update/out.test.toml b/acceptance/bundle/resources/permissions/pipelines/update/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/permissions/pipelines/update/out.test.toml +++ b/acceptance/bundle/resources/permissions/pipelines/update/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/permissions/postgres_projects/current_can_manage/out.test.toml b/acceptance/bundle/resources/permissions/postgres_projects/current_can_manage/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/permissions/postgres_projects/current_can_manage/out.test.toml +++ b/acceptance/bundle/resources/permissions/postgres_projects/current_can_manage/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/permissions/sql_warehouses/current_can_manage/out.test.toml b/acceptance/bundle/resources/permissions/sql_warehouses/current_can_manage/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/permissions/sql_warehouses/current_can_manage/out.test.toml +++ b/acceptance/bundle/resources/permissions/sql_warehouses/current_can_manage/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/permissions/target_permissions/out.test.toml b/acceptance/bundle/resources/permissions/target_permissions/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/permissions/target_permissions/out.test.toml +++ b/acceptance/bundle/resources/permissions/target_permissions/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/permissions/vector_search_endpoints/current_can_manage/out.test.toml b/acceptance/bundle/resources/permissions/vector_search_endpoints/current_can_manage/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/resources/permissions/vector_search_endpoints/current_can_manage/out.test.toml +++ b/acceptance/bundle/resources/permissions/vector_search_endpoints/current_can_manage/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/pipelines/allow-duplicate-names/out.test.toml b/acceptance/bundle/resources/pipelines/allow-duplicate-names/out.test.toml index bbc7fcfd1bd..78c615e06e5 100644 --- a/acceptance/bundle/resources/pipelines/allow-duplicate-names/out.test.toml +++ b/acceptance/bundle/resources/pipelines/allow-duplicate-names/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/pipelines/auto-approve/out.test.toml b/acceptance/bundle/resources/pipelines/auto-approve/out.test.toml index 5ad0addb75e..c1cae9fcc3e 100644 --- a/acceptance/bundle/resources/pipelines/auto-approve/out.test.toml +++ b/acceptance/bundle/resources/pipelines/auto-approve/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RunsOnDbr = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/pipelines/drift/parameters/out.test.toml b/acceptance/bundle/resources/pipelines/drift/parameters/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/resources/pipelines/drift/parameters/out.test.toml +++ b/acceptance/bundle/resources/pipelines/drift/parameters/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/pipelines/lakeflow-pipeline/out.test.toml b/acceptance/bundle/resources/pipelines/lakeflow-pipeline/out.test.toml index bbc7fcfd1bd..78c615e06e5 100644 --- a/acceptance/bundle/resources/pipelines/lakeflow-pipeline/out.test.toml +++ b/acceptance/bundle/resources/pipelines/lakeflow-pipeline/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/pipelines/num-workers-zero/out.test.toml b/acceptance/bundle/resources/pipelines/num-workers-zero/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/pipelines/num-workers-zero/out.test.toml +++ b/acceptance/bundle/resources/pipelines/num-workers-zero/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/pipelines/photon-true/out.test.toml b/acceptance/bundle/resources/pipelines/photon-true/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/pipelines/photon-true/out.test.toml +++ b/acceptance/bundle/resources/pipelines/photon-true/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/out.test.toml b/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/out.test.toml +++ b/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/pipelines/recreate-keys/change-storage/out.test.toml b/acceptance/bundle/resources/pipelines/recreate-keys/change-storage/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/pipelines/recreate-keys/change-storage/out.test.toml +++ b/acceptance/bundle/resources/pipelines/recreate-keys/change-storage/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/pipelines/recreate/out.test.toml b/acceptance/bundle/resources/pipelines/recreate/out.test.toml index 6e3397efe53..36ec3116026 100644 --- a/acceptance/bundle/resources/pipelines/recreate/out.test.toml +++ b/acceptance/bundle/resources/pipelines/recreate/out.test.toml @@ -2,4 +2,5 @@ Local = true Cloud = true RequiresUnityCatalog = true RunsOnDbr = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/pipelines/remote_matches_config/out.test.toml b/acceptance/bundle/resources/pipelines/remote_matches_config/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/pipelines/remote_matches_config/out.test.toml +++ b/acceptance/bundle/resources/pipelines/remote_matches_config/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/pipelines/update/out.test.toml b/acceptance/bundle/resources/pipelines/update/out.test.toml index bbc7fcfd1bd..78c615e06e5 100644 --- a/acceptance/bundle/resources/pipelines/update/out.test.toml +++ b/acceptance/bundle/resources/pipelines/update/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/pipelines/zero-value-fields/out.test.toml b/acceptance/bundle/resources/pipelines/zero-value-fields/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/pipelines/zero-value-fields/out.test.toml +++ b/acceptance/bundle/resources/pipelines/zero-value-fields/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/postgres_branches/basic/out.test.toml b/acceptance/bundle/resources/postgres_branches/basic/out.test.toml index 110f841fa05..2adb592001c 100644 --- a/acceptance/bundle/resources/postgres_branches/basic/out.test.toml +++ b/acceptance/bundle/resources/postgres_branches/basic/out.test.toml @@ -3,4 +3,5 @@ Cloud = true RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/resources/postgres_branches/purge_on_delete/out.test.toml b/acceptance/bundle/resources/postgres_branches/purge_on_delete/out.test.toml index 110f841fa05..2adb592001c 100644 --- a/acceptance/bundle/resources/postgres_branches/purge_on_delete/out.test.toml +++ b/acceptance/bundle/resources/postgres_branches/purge_on_delete/out.test.toml @@ -3,4 +3,5 @@ Cloud = true RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/resources/postgres_branches/purge_on_delete_transitions/out.test.toml b/acceptance/bundle/resources/postgres_branches/purge_on_delete_transitions/out.test.toml index 4fe23e297fe..e9bd4504608 100644 --- a/acceptance/bundle/resources/postgres_branches/purge_on_delete_transitions/out.test.toml +++ b/acceptance/bundle/resources/postgres_branches/purge_on_delete_transitions/out.test.toml @@ -3,4 +3,5 @@ Cloud = true RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/postgres_branches/recreate/out.test.toml b/acceptance/bundle/resources/postgres_branches/recreate/out.test.toml index 110f841fa05..2adb592001c 100644 --- a/acceptance/bundle/resources/postgres_branches/recreate/out.test.toml +++ b/acceptance/bundle/resources/postgres_branches/recreate/out.test.toml @@ -3,4 +3,5 @@ Cloud = true RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/resources/postgres_branches/replace_existing/out.test.toml b/acceptance/bundle/resources/postgres_branches/replace_existing/out.test.toml index 110f841fa05..2adb592001c 100644 --- a/acceptance/bundle/resources/postgres_branches/replace_existing/out.test.toml +++ b/acceptance/bundle/resources/postgres_branches/replace_existing/out.test.toml @@ -3,4 +3,5 @@ Cloud = true RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/resources/postgres_branches/update_protected/out.test.toml b/acceptance/bundle/resources/postgres_branches/update_protected/out.test.toml index 110f841fa05..2adb592001c 100644 --- a/acceptance/bundle/resources/postgres_branches/update_protected/out.test.toml +++ b/acceptance/bundle/resources/postgres_branches/update_protected/out.test.toml @@ -3,4 +3,5 @@ Cloud = true RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/resources/postgres_branches/without_branch_id/out.test.toml b/acceptance/bundle/resources/postgres_branches/without_branch_id/out.test.toml index 110f841fa05..2adb592001c 100644 --- a/acceptance/bundle/resources/postgres_branches/without_branch_id/out.test.toml +++ b/acceptance/bundle/resources/postgres_branches/without_branch_id/out.test.toml @@ -3,4 +3,5 @@ Cloud = true RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/resources/postgres_catalogs/basic/out.test.toml b/acceptance/bundle/resources/postgres_catalogs/basic/out.test.toml index 110f841fa05..2adb592001c 100644 --- a/acceptance/bundle/resources/postgres_catalogs/basic/out.test.toml +++ b/acceptance/bundle/resources/postgres_catalogs/basic/out.test.toml @@ -3,4 +3,5 @@ Cloud = true RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/resources/postgres_catalogs/recreate/out.test.toml b/acceptance/bundle/resources/postgres_catalogs/recreate/out.test.toml index 110f841fa05..2adb592001c 100644 --- a/acceptance/bundle/resources/postgres_catalogs/recreate/out.test.toml +++ b/acceptance/bundle/resources/postgres_catalogs/recreate/out.test.toml @@ -3,4 +3,5 @@ Cloud = true RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/resources/postgres_databases/basic/out.test.toml b/acceptance/bundle/resources/postgres_databases/basic/out.test.toml index 110f841fa05..2adb592001c 100644 --- a/acceptance/bundle/resources/postgres_databases/basic/out.test.toml +++ b/acceptance/bundle/resources/postgres_databases/basic/out.test.toml @@ -3,4 +3,5 @@ Cloud = true RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/resources/postgres_databases/live_errors/bad_database_id/out.test.toml b/acceptance/bundle/resources/postgres_databases/live_errors/bad_database_id/out.test.toml index 4fe23e297fe..e9bd4504608 100644 --- a/acceptance/bundle/resources/postgres_databases/live_errors/bad_database_id/out.test.toml +++ b/acceptance/bundle/resources/postgres_databases/live_errors/bad_database_id/out.test.toml @@ -3,4 +3,5 @@ Cloud = true RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/postgres_databases/live_errors/bad_role_ref/out.test.toml b/acceptance/bundle/resources/postgres_databases/live_errors/bad_role_ref/out.test.toml index 4fe23e297fe..e9bd4504608 100644 --- a/acceptance/bundle/resources/postgres_databases/live_errors/bad_role_ref/out.test.toml +++ b/acceptance/bundle/resources/postgres_databases/live_errors/bad_role_ref/out.test.toml @@ -3,4 +3,5 @@ Cloud = true RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/postgres_databases/recreate/out.test.toml b/acceptance/bundle/resources/postgres_databases/recreate/out.test.toml index 110f841fa05..2adb592001c 100644 --- a/acceptance/bundle/resources/postgres_databases/recreate/out.test.toml +++ b/acceptance/bundle/resources/postgres_databases/recreate/out.test.toml @@ -3,4 +3,5 @@ Cloud = true RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/resources/postgres_databases/replace_existing/out.test.toml b/acceptance/bundle/resources/postgres_databases/replace_existing/out.test.toml index 110f841fa05..2adb592001c 100644 --- a/acceptance/bundle/resources/postgres_databases/replace_existing/out.test.toml +++ b/acceptance/bundle/resources/postgres_databases/replace_existing/out.test.toml @@ -3,4 +3,5 @@ Cloud = true RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/resources/postgres_databases/update/out.test.toml b/acceptance/bundle/resources/postgres_databases/update/out.test.toml index 110f841fa05..2adb592001c 100644 --- a/acceptance/bundle/resources/postgres_databases/update/out.test.toml +++ b/acceptance/bundle/resources/postgres_databases/update/out.test.toml @@ -3,4 +3,5 @@ Cloud = true RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/resources/postgres_endpoints/basic/out.test.toml b/acceptance/bundle/resources/postgres_endpoints/basic/out.test.toml index 110f841fa05..2adb592001c 100644 --- a/acceptance/bundle/resources/postgres_endpoints/basic/out.test.toml +++ b/acceptance/bundle/resources/postgres_endpoints/basic/out.test.toml @@ -3,4 +3,5 @@ Cloud = true RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/resources/postgres_endpoints/recreate/out.test.toml b/acceptance/bundle/resources/postgres_endpoints/recreate/out.test.toml index 4fe23e297fe..e9bd4504608 100644 --- a/acceptance/bundle/resources/postgres_endpoints/recreate/out.test.toml +++ b/acceptance/bundle/resources/postgres_endpoints/recreate/out.test.toml @@ -3,4 +3,5 @@ Cloud = true RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/postgres_endpoints/replace_existing/out.test.toml b/acceptance/bundle/resources/postgres_endpoints/replace_existing/out.test.toml index 110f841fa05..2adb592001c 100644 --- a/acceptance/bundle/resources/postgres_endpoints/replace_existing/out.test.toml +++ b/acceptance/bundle/resources/postgres_endpoints/replace_existing/out.test.toml @@ -3,4 +3,5 @@ Cloud = true RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/resources/postgres_endpoints/update_autoscaling/out.test.toml b/acceptance/bundle/resources/postgres_endpoints/update_autoscaling/out.test.toml index 110f841fa05..2adb592001c 100644 --- a/acceptance/bundle/resources/postgres_endpoints/update_autoscaling/out.test.toml +++ b/acceptance/bundle/resources/postgres_endpoints/update_autoscaling/out.test.toml @@ -3,4 +3,5 @@ Cloud = true RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/resources/postgres_endpoints/without_endpoint_id/out.test.toml b/acceptance/bundle/resources/postgres_endpoints/without_endpoint_id/out.test.toml index 110f841fa05..2adb592001c 100644 --- a/acceptance/bundle/resources/postgres_endpoints/without_endpoint_id/out.test.toml +++ b/acceptance/bundle/resources/postgres_endpoints/without_endpoint_id/out.test.toml @@ -3,4 +3,5 @@ Cloud = true RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/resources/postgres_projects/basic/out.test.toml b/acceptance/bundle/resources/postgres_projects/basic/out.test.toml index 110f841fa05..2adb592001c 100644 --- a/acceptance/bundle/resources/postgres_projects/basic/out.test.toml +++ b/acceptance/bundle/resources/postgres_projects/basic/out.test.toml @@ -3,4 +3,5 @@ Cloud = true RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/resources/postgres_projects/purge_on_delete/out.test.toml b/acceptance/bundle/resources/postgres_projects/purge_on_delete/out.test.toml index 110f841fa05..2adb592001c 100644 --- a/acceptance/bundle/resources/postgres_projects/purge_on_delete/out.test.toml +++ b/acceptance/bundle/resources/postgres_projects/purge_on_delete/out.test.toml @@ -3,4 +3,5 @@ Cloud = true RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/resources/postgres_projects/purge_on_delete_transitions/out.test.toml b/acceptance/bundle/resources/postgres_projects/purge_on_delete_transitions/out.test.toml index 4fe23e297fe..e9bd4504608 100644 --- a/acceptance/bundle/resources/postgres_projects/purge_on_delete_transitions/out.test.toml +++ b/acceptance/bundle/resources/postgres_projects/purge_on_delete_transitions/out.test.toml @@ -3,4 +3,5 @@ Cloud = true RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/postgres_projects/recreate/out.test.toml b/acceptance/bundle/resources/postgres_projects/recreate/out.test.toml index 110f841fa05..2adb592001c 100644 --- a/acceptance/bundle/resources/postgres_projects/recreate/out.test.toml +++ b/acceptance/bundle/resources/postgres_projects/recreate/out.test.toml @@ -3,4 +3,5 @@ Cloud = true RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/resources/postgres_projects/update_display_name/out.test.toml b/acceptance/bundle/resources/postgres_projects/update_display_name/out.test.toml index 110f841fa05..2adb592001c 100644 --- a/acceptance/bundle/resources/postgres_projects/update_display_name/out.test.toml +++ b/acceptance/bundle/resources/postgres_projects/update_display_name/out.test.toml @@ -3,4 +3,5 @@ Cloud = true RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/resources/postgres_projects/without_project_id/out.test.toml b/acceptance/bundle/resources/postgres_projects/without_project_id/out.test.toml index 110f841fa05..2adb592001c 100644 --- a/acceptance/bundle/resources/postgres_projects/without_project_id/out.test.toml +++ b/acceptance/bundle/resources/postgres_projects/without_project_id/out.test.toml @@ -3,4 +3,5 @@ Cloud = true RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/resources/postgres_roles/basic/out.test.toml b/acceptance/bundle/resources/postgres_roles/basic/out.test.toml index 110f841fa05..2adb592001c 100644 --- a/acceptance/bundle/resources/postgres_roles/basic/out.test.toml +++ b/acceptance/bundle/resources/postgres_roles/basic/out.test.toml @@ -3,4 +3,5 @@ Cloud = true RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/resources/postgres_roles/inherited-role-bind/out.test.toml b/acceptance/bundle/resources/postgres_roles/inherited-role-bind/out.test.toml index c5b8e7c8a71..67797faa7b2 100644 --- a/acceptance/bundle/resources/postgres_roles/inherited-role-bind/out.test.toml +++ b/acceptance/bundle/resources/postgres_roles/inherited-role-bind/out.test.toml @@ -3,4 +3,5 @@ Cloud = false RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/postgres_roles/inherited-role-conflict/out.test.toml b/acceptance/bundle/resources/postgres_roles/inherited-role-conflict/out.test.toml index c5b8e7c8a71..67797faa7b2 100644 --- a/acceptance/bundle/resources/postgres_roles/inherited-role-conflict/out.test.toml +++ b/acceptance/bundle/resources/postgres_roles/inherited-role-conflict/out.test.toml @@ -3,4 +3,5 @@ Cloud = false RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/postgres_roles/recreate-postgres-role/out.test.toml b/acceptance/bundle/resources/postgres_roles/recreate-postgres-role/out.test.toml index 4fe23e297fe..e9bd4504608 100644 --- a/acceptance/bundle/resources/postgres_roles/recreate-postgres-role/out.test.toml +++ b/acceptance/bundle/resources/postgres_roles/recreate-postgres-role/out.test.toml @@ -3,4 +3,5 @@ Cloud = true RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/postgres_roles/recreate/out.test.toml b/acceptance/bundle/resources/postgres_roles/recreate/out.test.toml index 110f841fa05..2adb592001c 100644 --- a/acceptance/bundle/resources/postgres_roles/recreate/out.test.toml +++ b/acceptance/bundle/resources/postgres_roles/recreate/out.test.toml @@ -3,4 +3,5 @@ Cloud = true RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/resources/postgres_roles/replace_existing/out.test.toml b/acceptance/bundle/resources/postgres_roles/replace_existing/out.test.toml index 110f841fa05..2adb592001c 100644 --- a/acceptance/bundle/resources/postgres_roles/replace_existing/out.test.toml +++ b/acceptance/bundle/resources/postgres_roles/replace_existing/out.test.toml @@ -3,4 +3,5 @@ Cloud = true RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/resources/postgres_roles/update/out.test.toml b/acceptance/bundle/resources/postgres_roles/update/out.test.toml index 110f841fa05..2adb592001c 100644 --- a/acceptance/bundle/resources/postgres_roles/update/out.test.toml +++ b/acceptance/bundle/resources/postgres_roles/update/out.test.toml @@ -3,4 +3,5 @@ Cloud = true RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/resources/postgres_synced_tables/basic/out.test.toml b/acceptance/bundle/resources/postgres_synced_tables/basic/out.test.toml index 110f841fa05..2adb592001c 100644 --- a/acceptance/bundle/resources/postgres_synced_tables/basic/out.test.toml +++ b/acceptance/bundle/resources/postgres_synced_tables/basic/out.test.toml @@ -3,4 +3,5 @@ Cloud = true RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/resources/postgres_synced_tables/recreate/out.test.toml b/acceptance/bundle/resources/postgres_synced_tables/recreate/out.test.toml index 110f841fa05..2adb592001c 100644 --- a/acceptance/bundle/resources/postgres_synced_tables/recreate/out.test.toml +++ b/acceptance/bundle/resources/postgres_synced_tables/recreate/out.test.toml @@ -3,4 +3,5 @@ Cloud = true RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/resources/quality_monitors/change_assets_dir/out.test.toml b/acceptance/bundle/resources/quality_monitors/change_assets_dir/out.test.toml index e849ec85ace..fdd9f954e6c 100644 --- a/acceptance/bundle/resources/quality_monitors/change_assets_dir/out.test.toml +++ b/acceptance/bundle/resources/quality_monitors/change_assets_dir/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/quality_monitors/change_output_schema_name/out.test.toml b/acceptance/bundle/resources/quality_monitors/change_output_schema_name/out.test.toml index e849ec85ace..fdd9f954e6c 100644 --- a/acceptance/bundle/resources/quality_monitors/change_output_schema_name/out.test.toml +++ b/acceptance/bundle/resources/quality_monitors/change_output_schema_name/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/quality_monitors/change_table_name/out.test.toml b/acceptance/bundle/resources/quality_monitors/change_table_name/out.test.toml index fe4076cdf9b..ce8dec17c30 100644 --- a/acceptance/bundle/resources/quality_monitors/change_table_name/out.test.toml +++ b/acceptance/bundle/resources/quality_monitors/change_table_name/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/quality_monitors/create/out.test.toml b/acceptance/bundle/resources/quality_monitors/create/out.test.toml index e849ec85ace..fdd9f954e6c 100644 --- a/acceptance/bundle/resources/quality_monitors/create/out.test.toml +++ b/acceptance/bundle/resources/quality_monitors/create/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/registered_models/aliases_converge/out.test.toml b/acceptance/bundle/resources/registered_models/aliases_converge/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/resources/registered_models/aliases_converge/out.test.toml +++ b/acceptance/bundle/resources/registered_models/aliases_converge/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/registered_models/basic/out.test.toml b/acceptance/bundle/resources/registered_models/basic/out.test.toml index 6e3397efe53..36ec3116026 100644 --- a/acceptance/bundle/resources/registered_models/basic/out.test.toml +++ b/acceptance/bundle/resources/registered_models/basic/out.test.toml @@ -2,4 +2,5 @@ Local = true Cloud = true RequiresUnityCatalog = true RunsOnDbr = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/registered_models/drift/browse_only/out.test.toml b/acceptance/bundle/resources/registered_models/drift/browse_only/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/resources/registered_models/drift/browse_only/out.test.toml +++ b/acceptance/bundle/resources/registered_models/drift/browse_only/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/schemas/auto-approve/out.test.toml b/acceptance/bundle/resources/schemas/auto-approve/out.test.toml index 6e3397efe53..36ec3116026 100644 --- a/acceptance/bundle/resources/schemas/auto-approve/out.test.toml +++ b/acceptance/bundle/resources/schemas/auto-approve/out.test.toml @@ -2,4 +2,5 @@ Local = true Cloud = true RequiresUnityCatalog = true RunsOnDbr = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/schemas/drift/managed_properties/out.test.toml b/acceptance/bundle/resources/schemas/drift/managed_properties/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/resources/schemas/drift/managed_properties/out.test.toml +++ b/acceptance/bundle/resources/schemas/drift/managed_properties/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/schemas/recreate/out.test.toml b/acceptance/bundle/resources/schemas/recreate/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/schemas/recreate/out.test.toml +++ b/acceptance/bundle/resources/schemas/recreate/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/schemas/update/out.test.toml b/acceptance/bundle/resources/schemas/update/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/schemas/update/out.test.toml +++ b/acceptance/bundle/resources/schemas/update/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/secret_scopes/backend-type/out.test.toml b/acceptance/bundle/resources/secret_scopes/backend-type/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/secret_scopes/backend-type/out.test.toml +++ b/acceptance/bundle/resources/secret_scopes/backend-type/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/secret_scopes/basic/out.test.toml b/acceptance/bundle/resources/secret_scopes/basic/out.test.toml index bbc7fcfd1bd..78c615e06e5 100644 --- a/acceptance/bundle/resources/secret_scopes/basic/out.test.toml +++ b/acceptance/bundle/resources/secret_scopes/basic/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/secret_scopes/delete_scope/out.test.toml b/acceptance/bundle/resources/secret_scopes/delete_scope/out.test.toml index bbc7fcfd1bd..78c615e06e5 100644 --- a/acceptance/bundle/resources/secret_scopes/delete_scope/out.test.toml +++ b/acceptance/bundle/resources/secret_scopes/delete_scope/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/secret_scopes/permissions-collapse/out.test.toml b/acceptance/bundle/resources/secret_scopes/permissions-collapse/out.test.toml index 6fc644d5164..aa7060abb02 100644 --- a/acceptance/bundle/resources/secret_scopes/permissions-collapse/out.test.toml +++ b/acceptance/bundle/resources/secret_scopes/permissions-collapse/out.test.toml @@ -2,4 +2,5 @@ Local = true Cloud = true RunsOnDbr = true CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/secret_scopes/permissions/out.test.toml b/acceptance/bundle/resources/secret_scopes/permissions/out.test.toml index 6b858c4df47..5ab6742ca97 100644 --- a/acceptance/bundle/resources/secret_scopes/permissions/out.test.toml +++ b/acceptance/bundle/resources/secret_scopes/permissions/out.test.toml @@ -2,4 +2,5 @@ Local = true Cloud = true RunsOnDbr = true CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/secrets/basic/out.test.toml b/acceptance/bundle/resources/secrets/basic/out.test.toml index fe4076cdf9b..ce8dec17c30 100644 --- a/acceptance/bundle/resources/secrets/basic/out.test.toml +++ b/acceptance/bundle/resources/secrets/basic/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/secrets/direct-only/out.test.toml b/acceptance/bundle/resources/secrets/direct-only/out.test.toml index 65156e0457c..76ce926fd59 100644 --- a/acceptance/bundle/resources/secrets/direct-only/out.test.toml +++ b/acceptance/bundle/resources/secrets/direct-only/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform"] diff --git a/acceptance/bundle/resources/secrets/update-value/out.test.toml b/acceptance/bundle/resources/secrets/update-value/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/resources/secrets/update-value/out.test.toml +++ b/acceptance/bundle/resources/secrets/update-value/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/secrets/validate-no-plain-text-default/out.test.toml b/acceptance/bundle/resources/secrets/validate-no-plain-text-default/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/resources/secrets/validate-no-plain-text-default/out.test.toml +++ b/acceptance/bundle/resources/secrets/validate-no-plain-text-default/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/secrets/validate-no-plain-text/out.test.toml b/acceptance/bundle/resources/secrets/validate-no-plain-text/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/resources/secrets/validate-no-plain-text/out.test.toml +++ b/acceptance/bundle/resources/secrets/validate-no-plain-text/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/sql_warehouses/lifecycle-started-edit/out.test.toml b/acceptance/bundle/resources/sql_warehouses/lifecycle-started-edit/out.test.toml index 5bbfaf5e65a..fe1401b93c2 100644 --- a/acceptance/bundle/resources/sql_warehouses/lifecycle-started-edit/out.test.toml +++ b/acceptance/bundle/resources/sql_warehouses/lifecycle-started-edit/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true CloudSlow = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/sql_warehouses/lifecycle-started-terraform-error/out.test.toml b/acceptance/bundle/resources/sql_warehouses/lifecycle-started-terraform-error/out.test.toml index 91e30e807cf..82f3670f861 100644 --- a/acceptance/bundle/resources/sql_warehouses/lifecycle-started-terraform-error/out.test.toml +++ b/acceptance/bundle/resources/sql_warehouses/lifecycle-started-terraform-error/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false CloudSlow = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform"] diff --git a/acceptance/bundle/resources/sql_warehouses/lifecycle-started-toggle/out.test.toml b/acceptance/bundle/resources/sql_warehouses/lifecycle-started-toggle/out.test.toml index d0abd00ab97..755a233e350 100644 --- a/acceptance/bundle/resources/sql_warehouses/lifecycle-started-toggle/out.test.toml +++ b/acceptance/bundle/resources/sql_warehouses/lifecycle-started-toggle/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false CloudSlow = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/sql_warehouses/lifecycle-started/out.test.toml b/acceptance/bundle/resources/sql_warehouses/lifecycle-started/out.test.toml index d0abd00ab97..755a233e350 100644 --- a/acceptance/bundle/resources/sql_warehouses/lifecycle-started/out.test.toml +++ b/acceptance/bundle/resources/sql_warehouses/lifecycle-started/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false CloudSlow = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/sql_warehouses/out.test.toml b/acceptance/bundle/resources/sql_warehouses/out.test.toml index 355ae0775bc..b1754ac936e 100644 --- a/acceptance/bundle/resources/sql_warehouses/out.test.toml +++ b/acceptance/bundle/resources/sql_warehouses/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false CloudSlow = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/synced_database_tables/basic/out.test.toml b/acceptance/bundle/resources/synced_database_tables/basic/out.test.toml index e991fce9180..dbcf84075ed 100644 --- a/acceptance/bundle/resources/synced_database_tables/basic/out.test.toml +++ b/acceptance/bundle/resources/synced_database_tables/basic/out.test.toml @@ -3,4 +3,5 @@ Cloud = true RequiresUnityCatalog = true RunsOnDbr = false CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/synced_database_tables/recreate/out.test.toml b/acceptance/bundle/resources/synced_database_tables/recreate/out.test.toml index c777e3ce206..9e11601cb89 100644 --- a/acceptance/bundle/resources/synced_database_tables/recreate/out.test.toml +++ b/acceptance/bundle/resources/synced_database_tables/recreate/out.test.toml @@ -2,4 +2,5 @@ Local = true Cloud = false RequiresUnityCatalog = true CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/test.toml b/acceptance/bundle/resources/test.toml index a8d852b9821..df0bcf4b84a 100644 --- a/acceptance/bundle/resources/test.toml +++ b/acceptance/bundle/resources/test.toml @@ -1,34 +1,27 @@ RecordRequests = true -# Recording adds two things to a deploy's output, and both are normalized away so the -# DMS run asserts the same goldens as the engine runs. That is the point of the run: -# every test then checks that recording does not change what a deploy does, rather than -# needing a second copy of 600-odd output files. They live here rather than in the parent so -# bundle/dms, which asserts the recording itself, does not inherit them. +# These normalize what deployment history recording adds to a deploy's output, so a test +# asserts the same golden files whether or not recording is on - that is the point of the +# DMS run, rather than keeping a second copy of 600-odd output files. They live here +# rather than in the parent so bundle/dms, which asserts the recording itself, does not +# inherit them. # -# The link printed after a deploy: +# The stamp on jobs and pipelines, which the plan reports as a change of its own. It shows +# up at whatever depth the enclosing object sits at, so the indent is matched loosely; the +# body lines are matched as `"key": value` pairs rather than `.*` so the match stops at the +# entry's own closing brace instead of running into its siblings. (Go's regexp is RE2, so +# the indent cannot be captured and back-referenced.) [[Repls]] -Old = '(?m)^Deployment history: .*\n' +Old = '(?m)^ *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\},\n' New = '' -# And the stamp on jobs and pipelines, which the plan reports as a change of its own. -# Matched with the trailing comma and without, since it can be the only entry - in which -# case the whole "changes" object exists only because of recording, and goes too. +# Same entry when it is the last one in the object, so the comma is on the line before. [[Repls]] -Old = '(?m)^ *"changes": \{\n *"deployment\.(deployment_id|version_id)": \{\n(?: *.*\n)*? *\}\n *\},?\n' -New = '' - -[[Repls]] -Old = '(?m)^ *"deployment\.(deployment_id|version_id)": \{\n(?: *.*\n)*? *\},\n' -New = '' - -# The stamp itself, where it appears inside a serialized deployment block (plan JSON, -# state dumps). Both orderings are covered: the pair can sit before or after the fields -# that stay, so the comma may be on this line or the one before. -[[Repls]] -Old = '(?m)^( *)"(deployment_id|version_id)": "[^"]*",\n' -New = '' +Old = '(?m),\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n' +New = ''' +''' +# And when it is the only entry, the whole "changes" object exists because of recording. [[Repls]] -Old = ',(\n *"(deployment_id|version_id)": "[^"]*")+' +Old = '(?m)^ *"changes": \{\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n *\},?\n' New = '' diff --git a/acceptance/bundle/resources/vector_search_endpoints/basic/out.test.toml b/acceptance/bundle/resources/vector_search_endpoints/basic/out.test.toml index fe4076cdf9b..ce8dec17c30 100644 --- a/acceptance/bundle/resources/vector_search_endpoints/basic/out.test.toml +++ b/acceptance/bundle/resources/vector_search_endpoints/basic/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/vector_search_endpoints/drift/budget_policy/out.test.toml b/acceptance/bundle/resources/vector_search_endpoints/drift/budget_policy/out.test.toml index 88423408186..e3153c50fff 100644 --- a/acceptance/bundle/resources/vector_search_endpoints/drift/budget_policy/out.test.toml +++ b/acceptance/bundle/resources/vector_search_endpoints/drift/budget_policy/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/vector_search_endpoints/drift/recreated_same_name/out.test.toml b/acceptance/bundle/resources/vector_search_endpoints/drift/recreated_same_name/out.test.toml index fe4076cdf9b..ce8dec17c30 100644 --- a/acceptance/bundle/resources/vector_search_endpoints/drift/recreated_same_name/out.test.toml +++ b/acceptance/bundle/resources/vector_search_endpoints/drift/recreated_same_name/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/vector_search_endpoints/drift/target_qps/out.test.toml b/acceptance/bundle/resources/vector_search_endpoints/drift/target_qps/out.test.toml index 88423408186..e3153c50fff 100644 --- a/acceptance/bundle/resources/vector_search_endpoints/drift/target_qps/out.test.toml +++ b/acceptance/bundle/resources/vector_search_endpoints/drift/target_qps/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/vector_search_endpoints/recreate/create-fails/out.test.toml b/acceptance/bundle/resources/vector_search_endpoints/recreate/create-fails/out.test.toml index 88423408186..e3153c50fff 100644 --- a/acceptance/bundle/resources/vector_search_endpoints/recreate/create-fails/out.test.toml +++ b/acceptance/bundle/resources/vector_search_endpoints/recreate/create-fails/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/vector_search_endpoints/recreate/endpoint_type/out.test.toml b/acceptance/bundle/resources/vector_search_endpoints/recreate/endpoint_type/out.test.toml index 88423408186..e3153c50fff 100644 --- a/acceptance/bundle/resources/vector_search_endpoints/recreate/endpoint_type/out.test.toml +++ b/acceptance/bundle/resources/vector_search_endpoints/recreate/endpoint_type/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/vector_search_endpoints/update/budget_policy/out.test.toml b/acceptance/bundle/resources/vector_search_endpoints/update/budget_policy/out.test.toml index 88423408186..e3153c50fff 100644 --- a/acceptance/bundle/resources/vector_search_endpoints/update/budget_policy/out.test.toml +++ b/acceptance/bundle/resources/vector_search_endpoints/update/budget_policy/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/vector_search_endpoints/update/target_qps/out.test.toml b/acceptance/bundle/resources/vector_search_endpoints/update/target_qps/out.test.toml index 88423408186..e3153c50fff 100644 --- a/acceptance/bundle/resources/vector_search_endpoints/update/target_qps/out.test.toml +++ b/acceptance/bundle/resources/vector_search_endpoints/update/target_qps/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/vector_search_indexes/basic/out.test.toml b/acceptance/bundle/resources/vector_search_indexes/basic/out.test.toml index 48203e833cd..8c71b922b55 100644 --- a/acceptance/bundle/resources/vector_search_indexes/basic/out.test.toml +++ b/acceptance/bundle/resources/vector_search_indexes/basic/out.test.toml @@ -2,4 +2,5 @@ Local = true Cloud = true CloudSlow = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/vector_search_indexes/drift/deleted_remotely/out.test.toml b/acceptance/bundle/resources/vector_search_indexes/drift/deleted_remotely/out.test.toml index af665307d39..01ee791bdc3 100644 --- a/acceptance/bundle/resources/vector_search_indexes/drift/deleted_remotely/out.test.toml +++ b/acceptance/bundle/resources/vector_search_indexes/drift/deleted_remotely/out.test.toml @@ -2,4 +2,5 @@ Local = true Cloud = false CloudSlow = false RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/vector_search_indexes/drift/orphaned_endpoint/out.test.toml b/acceptance/bundle/resources/vector_search_indexes/drift/orphaned_endpoint/out.test.toml index af665307d39..01ee791bdc3 100644 --- a/acceptance/bundle/resources/vector_search_indexes/drift/orphaned_endpoint/out.test.toml +++ b/acceptance/bundle/resources/vector_search_indexes/drift/orphaned_endpoint/out.test.toml @@ -2,4 +2,5 @@ Local = true Cloud = false CloudSlow = false RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/vector_search_indexes/grants/select/out.test.toml b/acceptance/bundle/resources/vector_search_indexes/grants/select/out.test.toml index 48203e833cd..8c71b922b55 100644 --- a/acceptance/bundle/resources/vector_search_indexes/grants/select/out.test.toml +++ b/acceptance/bundle/resources/vector_search_indexes/grants/select/out.test.toml @@ -2,4 +2,5 @@ Local = true Cloud = true CloudSlow = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/vector_search_indexes/recreate/embedding_dimension/out.test.toml b/acceptance/bundle/resources/vector_search_indexes/recreate/embedding_dimension/out.test.toml index 48203e833cd..8c71b922b55 100644 --- a/acceptance/bundle/resources/vector_search_indexes/recreate/embedding_dimension/out.test.toml +++ b/acceptance/bundle/resources/vector_search_indexes/recreate/embedding_dimension/out.test.toml @@ -2,4 +2,5 @@ Local = true Cloud = true CloudSlow = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/vector_search_indexes/recreate/pending_deletion/out.test.toml b/acceptance/bundle/resources/vector_search_indexes/recreate/pending_deletion/out.test.toml index af665307d39..01ee791bdc3 100644 --- a/acceptance/bundle/resources/vector_search_indexes/recreate/pending_deletion/out.test.toml +++ b/acceptance/bundle/resources/vector_search_indexes/recreate/pending_deletion/out.test.toml @@ -2,4 +2,5 @@ Local = true Cloud = false CloudSlow = false RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/vector_search_indexes/recreate/with_endpoint/out.test.toml b/acceptance/bundle/resources/vector_search_indexes/recreate/with_endpoint/out.test.toml index af665307d39..01ee791bdc3 100644 --- a/acceptance/bundle/resources/vector_search_indexes/recreate/with_endpoint/out.test.toml +++ b/acceptance/bundle/resources/vector_search_indexes/recreate/with_endpoint/out.test.toml @@ -2,4 +2,5 @@ Local = true Cloud = false CloudSlow = false RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/vector_search_indexes/schema_normalization/out.test.toml b/acceptance/bundle/resources/vector_search_indexes/schema_normalization/out.test.toml index 48203e833cd..8c71b922b55 100644 --- a/acceptance/bundle/resources/vector_search_indexes/schema_normalization/out.test.toml +++ b/acceptance/bundle/resources/vector_search_indexes/schema_normalization/out.test.toml @@ -2,4 +2,5 @@ Local = true Cloud = true CloudSlow = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/volumes/catalog-var-ref/out.test.toml b/acceptance/bundle/resources/volumes/catalog-var-ref/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/resources/volumes/catalog-var-ref/out.test.toml +++ b/acceptance/bundle/resources/volumes/catalog-var-ref/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/volumes/change-comment/out.test.toml b/acceptance/bundle/resources/volumes/change-comment/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/volumes/change-comment/out.test.toml +++ b/acceptance/bundle/resources/volumes/change-comment/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/volumes/change-name/out.test.toml b/acceptance/bundle/resources/volumes/change-name/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/volumes/change-name/out.test.toml +++ b/acceptance/bundle/resources/volumes/change-name/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/volumes/change-schema-name/out.test.toml b/acceptance/bundle/resources/volumes/change-schema-name/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/volumes/change-schema-name/out.test.toml +++ b/acceptance/bundle/resources/volumes/change-schema-name/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/volumes/recreate/out.test.toml b/acceptance/bundle/resources/volumes/recreate/out.test.toml index 6e3397efe53..36ec3116026 100644 --- a/acceptance/bundle/resources/volumes/recreate/out.test.toml +++ b/acceptance/bundle/resources/volumes/recreate/out.test.toml @@ -2,4 +2,5 @@ Local = true Cloud = true RequiresUnityCatalog = true RunsOnDbr = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/volumes/remote-change-name/out.test.toml b/acceptance/bundle/resources/volumes/remote-change-name/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/volumes/remote-change-name/out.test.toml +++ b/acceptance/bundle/resources/volumes/remote-change-name/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/volumes/remote-delete/out.test.toml b/acceptance/bundle/resources/volumes/remote-delete/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/volumes/remote-delete/out.test.toml +++ b/acceptance/bundle/resources/volumes/remote-delete/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/volumes/set-storage-location/out.test.toml b/acceptance/bundle/resources/volumes/set-storage-location/out.test.toml index 8c738f635ac..4ba1c38c46b 100644 --- a/acceptance/bundle/resources/volumes/set-storage-location/out.test.toml +++ b/acceptance/bundle/resources/volumes/set-storage-location/out.test.toml @@ -3,4 +3,5 @@ Cloud = true RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/volumes/set-volume-path/out.test.toml b/acceptance/bundle/resources/volumes/set-volume-path/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/resources/volumes/set-volume-path/out.test.toml +++ b/acceptance/bundle/resources/volumes/set-volume-path/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/volumes/uppercase-name/out.test.toml b/acceptance/bundle/resources/volumes/uppercase-name/out.test.toml index e849ec85ace..fdd9f954e6c 100644 --- a/acceptance/bundle/resources/volumes/uppercase-name/out.test.toml +++ b/acceptance/bundle/resources/volumes/uppercase-name/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/root/env-not-a-directory/out.test.toml b/acceptance/bundle/root/env-not-a-directory/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/root/env-not-a-directory/out.test.toml +++ b/acceptance/bundle/root/env-not-a-directory/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/root/env-not-found/out.test.toml b/acceptance/bundle/root/env-not-found/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/root/env-not-found/out.test.toml +++ b/acceptance/bundle/root/env-not-found/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/root/not-found/out.test.toml b/acceptance/bundle/root/not-found/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/root/not-found/out.test.toml +++ b/acceptance/bundle/root/not-found/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/root/real-empty-dir/out.test.toml b/acceptance/bundle/root/real-empty-dir/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/root/real-empty-dir/out.test.toml +++ b/acceptance/bundle/root/real-empty-dir/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/run/app-with-job/out.test.toml b/acceptance/bundle/run/app-with-job/out.test.toml index 880431d5a9f..95d684f1aba 100644 --- a/acceptance/bundle/run/app-with-job/out.test.toml +++ b/acceptance/bundle/run/app-with-job/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true CloudSlow = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run/basic/out.test.toml b/acceptance/bundle/run/basic/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/run/basic/out.test.toml +++ b/acceptance/bundle/run/basic/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run/diagnostics/out.test.toml b/acceptance/bundle/run/diagnostics/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/run/diagnostics/out.test.toml +++ b/acceptance/bundle/run/diagnostics/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run/inline-script/basic/out.test.toml b/acceptance/bundle/run/inline-script/basic/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/run/inline-script/basic/out.test.toml +++ b/acceptance/bundle/run/inline-script/basic/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run/inline-script/cwd/out.test.toml b/acceptance/bundle/run/inline-script/cwd/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/run/inline-script/cwd/out.test.toml +++ b/acceptance/bundle/run/inline-script/cwd/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run/inline-script/databricks-cli/profile-is-passed/from_flag/out.test.toml b/acceptance/bundle/run/inline-script/databricks-cli/profile-is-passed/from_flag/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/run/inline-script/databricks-cli/profile-is-passed/from_flag/out.test.toml +++ b/acceptance/bundle/run/inline-script/databricks-cli/profile-is-passed/from_flag/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run/inline-script/databricks-cli/target-is-passed/default/out.test.toml b/acceptance/bundle/run/inline-script/databricks-cli/target-is-passed/default/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/run/inline-script/databricks-cli/target-is-passed/default/out.test.toml +++ b/acceptance/bundle/run/inline-script/databricks-cli/target-is-passed/default/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run/inline-script/databricks-cli/target-is-passed/from_flag/out.test.toml b/acceptance/bundle/run/inline-script/databricks-cli/target-is-passed/from_flag/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/run/inline-script/databricks-cli/target-is-passed/from_flag/out.test.toml +++ b/acceptance/bundle/run/inline-script/databricks-cli/target-is-passed/from_flag/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run/inline-script/no-auth/out.test.toml b/acceptance/bundle/run/inline-script/no-auth/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/run/inline-script/no-auth/out.test.toml +++ b/acceptance/bundle/run/inline-script/no-auth/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run/inline-script/no-bundle/out.test.toml b/acceptance/bundle/run/inline-script/no-bundle/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/run/inline-script/no-bundle/out.test.toml +++ b/acceptance/bundle/run/inline-script/no-bundle/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run/inline-script/no-separator/out.test.toml b/acceptance/bundle/run/inline-script/no-separator/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/run/inline-script/no-separator/out.test.toml +++ b/acceptance/bundle/run/inline-script/no-separator/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run/jobs/partial_run/out.test.toml b/acceptance/bundle/run/jobs/partial_run/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/run/jobs/partial_run/out.test.toml +++ b/acceptance/bundle/run/jobs/partial_run/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run/no-state/out.test.toml b/acceptance/bundle/run/no-state/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/run/no-state/out.test.toml +++ b/acceptance/bundle/run/no-state/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run/refresh-flags/out.test.toml b/acceptance/bundle/run/refresh-flags/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/run/refresh-flags/out.test.toml +++ b/acceptance/bundle/run/refresh-flags/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run/scripts/basic/out.test.toml b/acceptance/bundle/run/scripts/basic/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/run/scripts/basic/out.test.toml +++ b/acceptance/bundle/run/scripts/basic/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run/scripts/cwd/out.test.toml b/acceptance/bundle/run/scripts/cwd/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/run/scripts/cwd/out.test.toml +++ b/acceptance/bundle/run/scripts/cwd/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run/scripts/databricks-cli/profile-is-passed/from_flag/out.test.toml b/acceptance/bundle/run/scripts/databricks-cli/profile-is-passed/from_flag/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/run/scripts/databricks-cli/profile-is-passed/from_flag/out.test.toml +++ b/acceptance/bundle/run/scripts/databricks-cli/profile-is-passed/from_flag/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run/scripts/databricks-cli/target-is-passed/default/out.test.toml b/acceptance/bundle/run/scripts/databricks-cli/target-is-passed/default/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/run/scripts/databricks-cli/target-is-passed/default/out.test.toml +++ b/acceptance/bundle/run/scripts/databricks-cli/target-is-passed/default/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run/scripts/databricks-cli/target-is-passed/from_flag/out.test.toml b/acceptance/bundle/run/scripts/databricks-cli/target-is-passed/from_flag/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/run/scripts/databricks-cli/target-is-passed/from_flag/out.test.toml +++ b/acceptance/bundle/run/scripts/databricks-cli/target-is-passed/from_flag/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run/scripts/env-bad-prefix/out.test.toml b/acceptance/bundle/run/scripts/env-bad-prefix/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/run/scripts/env-bad-prefix/out.test.toml +++ b/acceptance/bundle/run/scripts/env-bad-prefix/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run/scripts/env-precedence/out.test.toml b/acceptance/bundle/run/scripts/env-precedence/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/run/scripts/env-precedence/out.test.toml +++ b/acceptance/bundle/run/scripts/env-precedence/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run/scripts/env-section/out.test.toml b/acceptance/bundle/run/scripts/env-section/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/run/scripts/env-section/out.test.toml +++ b/acceptance/bundle/run/scripts/env-section/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run/scripts/exit_code/out.test.toml b/acceptance/bundle/run/scripts/exit_code/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/run/scripts/exit_code/out.test.toml +++ b/acceptance/bundle/run/scripts/exit_code/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run/scripts/io/out.test.toml b/acceptance/bundle/run/scripts/io/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/run/scripts/io/out.test.toml +++ b/acceptance/bundle/run/scripts/io/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run/scripts/no-auth/out.test.toml b/acceptance/bundle/run/scripts/no-auth/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/run/scripts/no-auth/out.test.toml +++ b/acceptance/bundle/run/scripts/no-auth/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run/scripts/no-interpolation/out.test.toml b/acceptance/bundle/run/scripts/no-interpolation/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/run/scripts/no-interpolation/out.test.toml +++ b/acceptance/bundle/run/scripts/no-interpolation/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run/scripts/no_content/out.test.toml b/acceptance/bundle/run/scripts/no_content/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/run/scripts/no_content/out.test.toml +++ b/acceptance/bundle/run/scripts/no_content/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run/scripts/shell/envvar/out.test.toml b/acceptance/bundle/run/scripts/shell/envvar/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/run/scripts/shell/envvar/out.test.toml +++ b/acceptance/bundle/run/scripts/shell/envvar/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run/scripts/shell/math/out.test.toml b/acceptance/bundle/run/scripts/shell/math/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/run/scripts/shell/math/out.test.toml +++ b/acceptance/bundle/run/scripts/shell/math/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run/scripts/unique_keys/duplicate_resource_and_script_name_root/out.test.toml b/acceptance/bundle/run/scripts/unique_keys/duplicate_resource_and_script_name_root/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/run/scripts/unique_keys/duplicate_resource_and_script_name_root/out.test.toml +++ b/acceptance/bundle/run/scripts/unique_keys/duplicate_resource_and_script_name_root/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run/scripts/unique_keys/duplicate_resource_and_script_subconfigurations/out.test.toml b/acceptance/bundle/run/scripts/unique_keys/duplicate_resource_and_script_subconfigurations/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/run/scripts/unique_keys/duplicate_resource_and_script_subconfigurations/out.test.toml +++ b/acceptance/bundle/run/scripts/unique_keys/duplicate_resource_and_script_subconfigurations/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run/scripts/unique_keys/duplicate_script_names_in_subconfiguration/out.test.toml b/acceptance/bundle/run/scripts/unique_keys/duplicate_script_names_in_subconfiguration/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/run/scripts/unique_keys/duplicate_script_names_in_subconfiguration/out.test.toml +++ b/acceptance/bundle/run/scripts/unique_keys/duplicate_script_names_in_subconfiguration/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run/state-wiped/out.test.toml b/acceptance/bundle/run/state-wiped/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/run/state-wiped/out.test.toml +++ b/acceptance/bundle/run/state-wiped/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run_as/allowed/regular_user/out.test.toml b/acceptance/bundle/run_as/allowed/regular_user/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/run_as/allowed/regular_user/out.test.toml +++ b/acceptance/bundle/run_as/allowed/regular_user/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run_as/allowed/service_principal/out.test.toml b/acceptance/bundle/run_as/allowed/service_principal/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/run_as/allowed/service_principal/out.test.toml +++ b/acceptance/bundle/run_as/allowed/service_principal/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run_as/dashboard_embed/out.test.toml b/acceptance/bundle/run_as/dashboard_embed/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/run_as/dashboard_embed/out.test.toml +++ b/acceptance/bundle/run_as/dashboard_embed/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run_as/empty_override/out.test.toml b/acceptance/bundle/run_as/empty_override/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/run_as/empty_override/out.test.toml +++ b/acceptance/bundle/run_as/empty_override/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run_as/empty_run_as/out.test.toml b/acceptance/bundle/run_as/empty_run_as/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/run_as/empty_run_as/out.test.toml +++ b/acceptance/bundle/run_as/empty_run_as/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run_as/empty_run_as_dict/out.test.toml b/acceptance/bundle/run_as/empty_run_as_dict/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/run_as/empty_run_as_dict/out.test.toml +++ b/acceptance/bundle/run_as/empty_run_as_dict/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run_as/empty_sp/out.test.toml b/acceptance/bundle/run_as/empty_sp/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/run_as/empty_sp/out.test.toml +++ b/acceptance/bundle/run_as/empty_sp/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run_as/empty_user/out.test.toml b/acceptance/bundle/run_as/empty_user/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/run_as/empty_user/out.test.toml +++ b/acceptance/bundle/run_as/empty_user/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run_as/empty_user_and_sp/out.test.toml b/acceptance/bundle/run_as/empty_user_and_sp/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/run_as/empty_user_and_sp/out.test.toml +++ b/acceptance/bundle/run_as/empty_user_and_sp/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run_as/invalid_both_sp_and_user/out.test.toml b/acceptance/bundle/run_as/invalid_both_sp_and_user/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/run_as/invalid_both_sp_and_user/out.test.toml +++ b/acceptance/bundle/run_as/invalid_both_sp_and_user/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run_as/job_default/out.test.toml b/acceptance/bundle/run_as/job_default/out.test.toml index 9cfad3fb0d5..db8b4387cfe 100644 --- a/acceptance/bundle/run_as/job_default/out.test.toml +++ b/acceptance/bundle/run_as/job_default/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/run_as/model_serving_different/out.test.toml b/acceptance/bundle/run_as/model_serving_different/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/run_as/model_serving_different/out.test.toml +++ b/acceptance/bundle/run_as/model_serving_different/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run_as/model_serving_matching/out.test.toml b/acceptance/bundle/run_as/model_serving_matching/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/run_as/model_serving_matching/out.test.toml +++ b/acceptance/bundle/run_as/model_serving_matching/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run_as/out.test.toml b/acceptance/bundle/run_as/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/run_as/out.test.toml +++ b/acceptance/bundle/run_as/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run_as/pipelines/regular_user/out.test.toml b/acceptance/bundle/run_as/pipelines/regular_user/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/run_as/pipelines/regular_user/out.test.toml +++ b/acceptance/bundle/run_as/pipelines/regular_user/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run_as/pipelines/service_principal/out.test.toml b/acceptance/bundle/run_as/pipelines/service_principal/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/run_as/pipelines/service_principal/out.test.toml +++ b/acceptance/bundle/run_as/pipelines/service_principal/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/run_as/pipelines_legacy/out.test.toml b/acceptance/bundle/run_as/pipelines_legacy/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/run_as/pipelines_legacy/out.test.toml +++ b/acceptance/bundle/run_as/pipelines_legacy/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/scripts/no-trailing-newline/out.test.toml b/acceptance/bundle/scripts/no-trailing-newline/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/scripts/no-trailing-newline/out.test.toml +++ b/acceptance/bundle/scripts/no-trailing-newline/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/scripts/out.test.toml b/acceptance/bundle/scripts/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/scripts/out.test.toml +++ b/acceptance/bundle/scripts/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/scripts/restricted-execution/out.test.toml b/acceptance/bundle/scripts/restricted-execution/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/scripts/restricted-execution/out.test.toml +++ b/acceptance/bundle/scripts/restricted-execution/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/select/ambiguous/out.test.toml b/acceptance/bundle/select/ambiguous/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/select/ambiguous/out.test.toml +++ b/acceptance/bundle/select/ambiguous/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/select/basic/out.test.toml b/acceptance/bundle/select/basic/out.test.toml index 8b995e4d177..9c22c36f16e 100644 --- a/acceptance/bundle/select/basic/out.test.toml +++ b/acceptance/bundle/select/basic/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/select/grants_permissions/out.test.toml b/acceptance/bundle/select/grants_permissions/out.test.toml index 55ed5ee6619..c96f9a9f6c9 100644 --- a/acceptance/bundle/select/grants_permissions/out.test.toml +++ b/acceptance/bundle/select/grants_permissions/out.test.toml @@ -1,5 +1,6 @@ Local = true Cloud = false RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/select/missing/out.test.toml b/acceptance/bundle/select/missing/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/select/missing/out.test.toml +++ b/acceptance/bundle/select/missing/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/select/rejected/out.test.toml b/acceptance/bundle/select/rejected/out.test.toml index 65156e0457c..76ce926fd59 100644 --- a/acceptance/bundle/select/rejected/out.test.toml +++ b/acceptance/bundle/select/rejected/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform"] diff --git a/acceptance/bundle/state/bad_env/out.test.toml b/acceptance/bundle/state/bad_env/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/state/bad_env/out.test.toml +++ b/acceptance/bundle/state/bad_env/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/state/bad_json_local/out.test.toml b/acceptance/bundle/state/bad_json_local/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/state/bad_json_local/out.test.toml +++ b/acceptance/bundle/state/bad_json_local/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/state/basic/out.test.toml b/acceptance/bundle/state/basic/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/state/basic/out.test.toml +++ b/acceptance/bundle/state/basic/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/state/engine_default/out.test.toml b/acceptance/bundle/state/engine_default/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/state/engine_default/out.test.toml +++ b/acceptance/bundle/state/engine_default/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/state/engine_mismatch/out.test.toml b/acceptance/bundle/state/engine_mismatch/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/state/engine_mismatch/out.test.toml +++ b/acceptance/bundle/state/engine_mismatch/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/state/feature_flags/out.test.toml b/acceptance/bundle/state/feature_flags/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/state/feature_flags/out.test.toml +++ b/acceptance/bundle/state/feature_flags/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/state/force_pull_commands/out.test.toml b/acceptance/bundle/state/force_pull_commands/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/state/force_pull_commands/out.test.toml +++ b/acceptance/bundle/state/force_pull_commands/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/state/future_version/out.test.toml b/acceptance/bundle/state/future_version/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/state/future_version/out.test.toml +++ b/acceptance/bundle/state/future_version/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/state/lineage_different/out.test.toml b/acceptance/bundle/state/lineage_different/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/state/lineage_different/out.test.toml +++ b/acceptance/bundle/state/lineage_different/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/state/permission_level_migration/out.test.toml b/acceptance/bundle/state/permission_level_migration/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/state/permission_level_migration/out.test.toml +++ b/acceptance/bundle/state/permission_level_migration/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/state/same_serial/out.test.toml b/acceptance/bundle/state/same_serial/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/state/same_serial/out.test.toml +++ b/acceptance/bundle/state/same_serial/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/state/state_present/out.test.toml b/acceptance/bundle/state/state_present/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/state/state_present/out.test.toml +++ b/acceptance/bundle/state/state_present/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/summary/missing-libraries-file-path/out.test.toml b/acceptance/bundle/summary/missing-libraries-file-path/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/summary/missing-libraries-file-path/out.test.toml +++ b/acceptance/bundle/summary/missing-libraries-file-path/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/summary/modified_status/out.test.toml b/acceptance/bundle/summary/modified_status/out.test.toml index 7f4e2c0ca80..262e580a832 100644 --- a/acceptance/bundle/summary/modified_status/out.test.toml +++ b/acceptance/bundle/summary/modified_status/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] EnvMatrix.VARIANT = ["empty_resources.yml", "no_resources.yml"] diff --git a/acceptance/bundle/sync/dryrun/out.test.toml b/acceptance/bundle/sync/dryrun/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/sync/dryrun/out.test.toml +++ b/acceptance/bundle/sync/dryrun/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/sync/out.test.toml b/acceptance/bundle/sync/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/sync/out.test.toml +++ b/acceptance/bundle/sync/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/syncroot/dotdot-git/out.test.toml b/acceptance/bundle/syncroot/dotdot-git/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/syncroot/dotdot-git/out.test.toml +++ b/acceptance/bundle/syncroot/dotdot-git/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/syncroot/dotdot-nogit/out.test.toml b/acceptance/bundle/syncroot/dotdot-nogit/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/syncroot/dotdot-nogit/out.test.toml +++ b/acceptance/bundle/syncroot/dotdot-nogit/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/telemetry/config-remote-sync-error/out.test.toml b/acceptance/bundle/telemetry/config-remote-sync-error/out.test.toml index 4e136c6838f..4e97b0db661 100644 --- a/acceptance/bundle/telemetry/config-remote-sync-error/out.test.toml +++ b/acceptance/bundle/telemetry/config-remote-sync-error/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform"] diff --git a/acceptance/bundle/telemetry/config-remote-sync-recreate/out.test.toml b/acceptance/bundle/telemetry/config-remote-sync-recreate/out.test.toml index f7c4cf648a9..a3d9e265a64 100644 --- a/acceptance/bundle/telemetry/config-remote-sync-recreate/out.test.toml +++ b/acceptance/bundle/telemetry/config-remote-sync-recreate/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/telemetry/config-remote-sync-save/out.test.toml b/acceptance/bundle/telemetry/config-remote-sync-save/out.test.toml index f7c4cf648a9..a3d9e265a64 100644 --- a/acceptance/bundle/telemetry/config-remote-sync-save/out.test.toml +++ b/acceptance/bundle/telemetry/config-remote-sync-save/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/telemetry/config-remote-sync/out.test.toml b/acceptance/bundle/telemetry/config-remote-sync/out.test.toml index f7c4cf648a9..a3d9e265a64 100644 --- a/acceptance/bundle/telemetry/config-remote-sync/out.test.toml +++ b/acceptance/bundle/telemetry/config-remote-sync/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false GOOS.windows = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/telemetry/deploy-app-lifecycle-started/out.test.toml b/acceptance/bundle/telemetry/deploy-app-lifecycle-started/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/telemetry/deploy-app-lifecycle-started/out.test.toml +++ b/acceptance/bundle/telemetry/deploy-app-lifecycle-started/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/telemetry/deploy-artifact-path-type/out.test.toml b/acceptance/bundle/telemetry/deploy-artifact-path-type/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/telemetry/deploy-artifact-path-type/out.test.toml +++ b/acceptance/bundle/telemetry/deploy-artifact-path-type/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/telemetry/deploy-artifacts-variables/out.test.toml b/acceptance/bundle/telemetry/deploy-artifacts-variables/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/telemetry/deploy-artifacts-variables/out.test.toml +++ b/acceptance/bundle/telemetry/deploy-artifacts-variables/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/telemetry/deploy-compute-type/out.test.toml b/acceptance/bundle/telemetry/deploy-compute-type/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/telemetry/deploy-compute-type/out.test.toml +++ b/acceptance/bundle/telemetry/deploy-compute-type/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/telemetry/deploy-config-file-count/out.test.toml b/acceptance/bundle/telemetry/deploy-config-file-count/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/telemetry/deploy-config-file-count/out.test.toml +++ b/acceptance/bundle/telemetry/deploy-config-file-count/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/telemetry/deploy-error-message/out.test.toml b/acceptance/bundle/telemetry/deploy-error-message/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/telemetry/deploy-error-message/out.test.toml +++ b/acceptance/bundle/telemetry/deploy-error-message/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/telemetry/deploy-error/out.test.toml b/acceptance/bundle/telemetry/deploy-error/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/telemetry/deploy-error/out.test.toml +++ b/acceptance/bundle/telemetry/deploy-error/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/telemetry/deploy-experimental/out.test.toml b/acceptance/bundle/telemetry/deploy-experimental/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/telemetry/deploy-experimental/out.test.toml +++ b/acceptance/bundle/telemetry/deploy-experimental/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/telemetry/deploy-mode/out.test.toml b/acceptance/bundle/telemetry/deploy-mode/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/telemetry/deploy-mode/out.test.toml +++ b/acceptance/bundle/telemetry/deploy-mode/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/telemetry/deploy-name-prefix/custom/out.test.toml b/acceptance/bundle/telemetry/deploy-name-prefix/custom/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/telemetry/deploy-name-prefix/custom/out.test.toml +++ b/acceptance/bundle/telemetry/deploy-name-prefix/custom/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/telemetry/deploy-name-prefix/mode-development/out.test.toml b/acceptance/bundle/telemetry/deploy-name-prefix/mode-development/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/telemetry/deploy-name-prefix/mode-development/out.test.toml +++ b/acceptance/bundle/telemetry/deploy-name-prefix/mode-development/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/telemetry/deploy-no-uuid/out.test.toml b/acceptance/bundle/telemetry/deploy-no-uuid/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/telemetry/deploy-no-uuid/out.test.toml +++ b/acceptance/bundle/telemetry/deploy-no-uuid/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/telemetry/deploy-run-as/out.test.toml b/acceptance/bundle/telemetry/deploy-run-as/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/telemetry/deploy-run-as/out.test.toml +++ b/acceptance/bundle/telemetry/deploy-run-as/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/telemetry/deploy-target-count/out.test.toml b/acceptance/bundle/telemetry/deploy-target-count/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/telemetry/deploy-target-count/out.test.toml +++ b/acceptance/bundle/telemetry/deploy-target-count/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/telemetry/deploy-variable-count/out.test.toml b/acceptance/bundle/telemetry/deploy-variable-count/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/telemetry/deploy-variable-count/out.test.toml +++ b/acceptance/bundle/telemetry/deploy-variable-count/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/telemetry/deploy-whl-artifacts/out.test.toml b/acceptance/bundle/telemetry/deploy-whl-artifacts/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/telemetry/deploy-whl-artifacts/out.test.toml +++ b/acceptance/bundle/telemetry/deploy-whl-artifacts/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/telemetry/deploy-workspace-folder-permissions/out.test.toml b/acceptance/bundle/telemetry/deploy-workspace-folder-permissions/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/telemetry/deploy-workspace-folder-permissions/out.test.toml +++ b/acceptance/bundle/telemetry/deploy-workspace-folder-permissions/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/telemetry/deploy/out.test.toml b/acceptance/bundle/telemetry/deploy/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/telemetry/deploy/out.test.toml +++ b/acceptance/bundle/telemetry/deploy/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/templates-machinery/helper_upper_lower/out.test.toml b/acceptance/bundle/templates-machinery/helper_upper_lower/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/templates-machinery/helper_upper_lower/out.test.toml +++ b/acceptance/bundle/templates-machinery/helper_upper_lower/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/templates-machinery/helper_username/out.test.toml b/acceptance/bundle/templates-machinery/helper_username/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/templates-machinery/helper_username/out.test.toml +++ b/acceptance/bundle/templates-machinery/helper_username/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/templates-machinery/helpers-error/out.test.toml b/acceptance/bundle/templates-machinery/helpers-error/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/templates-machinery/helpers-error/out.test.toml +++ b/acceptance/bundle/templates-machinery/helpers-error/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/templates-machinery/number-precision/out.test.toml b/acceptance/bundle/templates-machinery/number-precision/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/templates-machinery/number-precision/out.test.toml +++ b/acceptance/bundle/templates-machinery/number-precision/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/templates-machinery/supported-url/out.test.toml b/acceptance/bundle/templates-machinery/supported-url/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/templates-machinery/supported-url/out.test.toml +++ b/acceptance/bundle/templates-machinery/supported-url/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/templates-machinery/unsupported-url/out.test.toml b/acceptance/bundle/templates-machinery/unsupported-url/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/templates-machinery/unsupported-url/out.test.toml +++ b/acceptance/bundle/templates-machinery/unsupported-url/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/templates-machinery/wrong-path/out.test.toml b/acceptance/bundle/templates-machinery/wrong-path/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/templates-machinery/wrong-path/out.test.toml +++ b/acceptance/bundle/templates-machinery/wrong-path/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/templates-machinery/wrong-url/out.test.toml b/acceptance/bundle/templates-machinery/wrong-url/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/templates-machinery/wrong-url/out.test.toml +++ b/acceptance/bundle/templates-machinery/wrong-url/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/templates/dbt-sql/out.test.toml b/acceptance/bundle/templates/dbt-sql/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/templates/dbt-sql/out.test.toml +++ b/acceptance/bundle/templates/dbt-sql/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/templates/default-minimal/python/out.test.toml b/acceptance/bundle/templates/default-minimal/python/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/templates/default-minimal/python/out.test.toml +++ b/acceptance/bundle/templates/default-minimal/python/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/templates/default-minimal/skip/out.test.toml b/acceptance/bundle/templates/default-minimal/skip/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/templates/default-minimal/skip/out.test.toml +++ b/acceptance/bundle/templates/default-minimal/skip/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/templates/default-minimal/sql/out.test.toml b/acceptance/bundle/templates/default-minimal/sql/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/templates/default-minimal/sql/out.test.toml +++ b/acceptance/bundle/templates/default-minimal/sql/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/templates/default-python/azure-government/out.test.toml b/acceptance/bundle/templates/default-python/azure-government/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/templates/default-python/azure-government/out.test.toml +++ b/acceptance/bundle/templates/default-python/azure-government/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/templates/default-python/classic/out.test.toml b/acceptance/bundle/templates/default-python/classic/out.test.toml index 2f44fc0b7cc..99483caeee6 100644 --- a/acceptance/bundle/templates/default-python/classic/out.test.toml +++ b/acceptance/bundle/templates/default-python/classic/out.test.toml @@ -1,5 +1,6 @@ Local = true Cloud = false Phase = 1 +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/templates/default-python/combinations/classic/out.test.toml b/acceptance/bundle/templates/default-python/combinations/classic/out.test.toml index 9f9b4934ffe..79230bd2367 100644 --- a/acceptance/bundle/templates/default-python/combinations/classic/out.test.toml +++ b/acceptance/bundle/templates/default-python/combinations/classic/out.test.toml @@ -1,5 +1,6 @@ Local = true Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] EnvMatrix.DLT = ["yes", "no"] EnvMatrix.NBOOK = ["yes", "no"] diff --git a/acceptance/bundle/templates/default-python/combinations/serverless/out.test.toml b/acceptance/bundle/templates/default-python/combinations/serverless/out.test.toml index 9f9b4934ffe..79230bd2367 100644 --- a/acceptance/bundle/templates/default-python/combinations/serverless/out.test.toml +++ b/acceptance/bundle/templates/default-python/combinations/serverless/out.test.toml @@ -1,5 +1,6 @@ Local = true Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] EnvMatrix.DLT = ["yes", "no"] EnvMatrix.NBOOK = ["yes", "no"] diff --git a/acceptance/bundle/templates/default-python/fail-missing-uv/out.test.toml b/acceptance/bundle/templates/default-python/fail-missing-uv/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/templates/default-python/fail-missing-uv/out.test.toml +++ b/acceptance/bundle/templates/default-python/fail-missing-uv/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/templates/default-python/integration_classic/out.test.toml b/acceptance/bundle/templates/default-python/integration_classic/out.test.toml index 50677b5f636..ed19028b891 100644 --- a/acceptance/bundle/templates/default-python/integration_classic/out.test.toml +++ b/acceptance/bundle/templates/default-python/integration_classic/out.test.toml @@ -1,5 +1,6 @@ Local = true Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] EnvMatrix.UV_PYTHON = [ "3.9", diff --git a/acceptance/bundle/templates/default-python/no-uc/out.test.toml b/acceptance/bundle/templates/default-python/no-uc/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/templates/default-python/no-uc/out.test.toml +++ b/acceptance/bundle/templates/default-python/no-uc/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/templates/default-python/serverless-customcatalog/out.test.toml b/acceptance/bundle/templates/default-python/serverless-customcatalog/out.test.toml index be193812ec2..b827ff3f062 100644 --- a/acceptance/bundle/templates/default-python/serverless-customcatalog/out.test.toml +++ b/acceptance/bundle/templates/default-python/serverless-customcatalog/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false Phase = 1 +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/templates/default-python/serverless/out.test.toml b/acceptance/bundle/templates/default-python/serverless/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/templates/default-python/serverless/out.test.toml +++ b/acceptance/bundle/templates/default-python/serverless/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/templates/default-scala/out.test.toml b/acceptance/bundle/templates/default-scala/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/templates/default-scala/out.test.toml +++ b/acceptance/bundle/templates/default-scala/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/templates/default-sql/out.test.toml b/acceptance/bundle/templates/default-sql/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/templates/default-sql/out.test.toml +++ b/acceptance/bundle/templates/default-sql/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/templates/lakeflow-integrations/out.test.toml b/acceptance/bundle/templates/lakeflow-integrations/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/templates/lakeflow-integrations/out.test.toml +++ b/acceptance/bundle/templates/lakeflow-integrations/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/templates/lakeflow-pipelines/python/out.test.toml b/acceptance/bundle/templates/lakeflow-pipelines/python/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/templates/lakeflow-pipelines/python/out.test.toml +++ b/acceptance/bundle/templates/lakeflow-pipelines/python/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/templates/lakeflow-pipelines/sql/out.test.toml b/acceptance/bundle/templates/lakeflow-pipelines/sql/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/templates/lakeflow-pipelines/sql/out.test.toml +++ b/acceptance/bundle/templates/lakeflow-pipelines/sql/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/templates/nested-output/out.test.toml b/acceptance/bundle/templates/nested-output/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/templates/nested-output/out.test.toml +++ b/acceptance/bundle/templates/nested-output/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/templates/pydabs/check-consistency/out.test.toml b/acceptance/bundle/templates/pydabs/check-consistency/out.test.toml index 88bd948e0a9..464dbdb3ab7 100644 --- a/acceptance/bundle/templates/pydabs/check-consistency/out.test.toml +++ b/acceptance/bundle/templates/pydabs/check-consistency/out.test.toml @@ -1,5 +1,6 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] EnvMatrix.INCLUDE_JOB = ["yes", "no"] EnvMatrix.INCLUDE_PIPELINE = ["yes", "no"] diff --git a/acceptance/bundle/templates/pydabs/check-formatting/out.test.toml b/acceptance/bundle/templates/pydabs/check-formatting/out.test.toml index 88bd948e0a9..464dbdb3ab7 100644 --- a/acceptance/bundle/templates/pydabs/check-formatting/out.test.toml +++ b/acceptance/bundle/templates/pydabs/check-formatting/out.test.toml @@ -1,5 +1,6 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] EnvMatrix.INCLUDE_JOB = ["yes", "no"] EnvMatrix.INCLUDE_PIPELINE = ["yes", "no"] diff --git a/acceptance/bundle/templates/pydabs/deploy-classic/out.test.toml b/acceptance/bundle/templates/pydabs/deploy-classic/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/templates/pydabs/deploy-classic/out.test.toml +++ b/acceptance/bundle/templates/pydabs/deploy-classic/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/templates/pydabs/init-classic/out.test.toml b/acceptance/bundle/templates/pydabs/init-classic/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/templates/pydabs/init-classic/out.test.toml +++ b/acceptance/bundle/templates/pydabs/init-classic/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/templates/telemetry/custom-template/out.test.toml b/acceptance/bundle/templates/telemetry/custom-template/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/templates/telemetry/custom-template/out.test.toml +++ b/acceptance/bundle/templates/telemetry/custom-template/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/templates/telemetry/dbt-sql/out.test.toml b/acceptance/bundle/templates/telemetry/dbt-sql/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/templates/telemetry/dbt-sql/out.test.toml +++ b/acceptance/bundle/templates/telemetry/dbt-sql/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/templates/telemetry/default-python/out.test.toml b/acceptance/bundle/templates/telemetry/default-python/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/templates/telemetry/default-python/out.test.toml +++ b/acceptance/bundle/templates/telemetry/default-python/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/templates/telemetry/default-sql/out.test.toml b/acceptance/bundle/templates/telemetry/default-sql/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/templates/telemetry/default-sql/out.test.toml +++ b/acceptance/bundle/templates/telemetry/default-sql/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/test.toml b/acceptance/bundle/test.toml index c9acc1e0635..2ff57477252 100644 --- a/acceptance/bundle/test.toml +++ b/acceptance/bundle/test.toml @@ -1,11 +1,38 @@ # This allows recording per-deployment output files, e.g. $CLI bundle deploy > out.$DATABRICKS_BUNDLE_ENGINE.txt EnvVaryOutput = "DATABRICKS_BUNDLE_ENGINE" +# Runs the whole bundle suite a second time with deployment history recording on, so the +# deployment metadata service (DMS) is exercised by every test rather than only the +# handful under bundle/dms. Empty is the default pair of engine runs. +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] + +# DMS is only supported by the direct engine, and only against the local testserver: +# the service runs in dev and staging, so a cloud run has nothing to record to. +EnvMatrixExclude.dms_needs_direct = ["DATABRICKS_BUNDLE_DMS=true", "DATABRICKS_BUNDLE_ENGINE=terraform"] +EnvMatrixExclude.dms_local_only = ["DATABRICKS_BUNDLE_DMS=true", "CONFIG_Cloud=true"] + +# Recording is gated off for users (see validate.ValidateRecordDeploymentHistory) and +# refuses a bundle whose state already tracks resources - which most tests here seed. +# Both are forced on: these tests assert what a deploy does, so the resource duplication +# the refusal guards against cannot bite them. +Env.DATABRICKS_BUNDLE_FORCE_ALLOW_RECORD_DEPLOYMENT_HISTORY = "1" +Env.DATABRICKS_BUNDLE_DMS_ALLOW_EXISTING_RESOURCES = "1" + +# The DMS run asserts the same golden files as the engine runs. +EnvRepl.DATABRICKS_BUNDLE_DMS = false + Ignore = ["databricks.yml"] # The lowest Python version we support. Alternative to "uv run --python 3.10" Env.UV_PYTHON = "3.10" +# The link a recorded deploy prints, dropped so a test asserts the same output whether or +# not recording is on. The URL itself is covered by workspaceurls.TestDeploymentURL, and +# the calls behind it by bundle/dms/record. +[[Repls]] +Old = '(?m)^Deployment history: .*\n' +New = '' + # User-agent: [[Repls]] Old = 'os/darwin' diff --git a/acceptance/bundle/trampoline/warning_message/out.test.toml b/acceptance/bundle/trampoline/warning_message/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/trampoline/warning_message/out.test.toml +++ b/acceptance/bundle/trampoline/warning_message/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/trampoline/warning_message_with_new_spark/out.test.toml b/acceptance/bundle/trampoline/warning_message_with_new_spark/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/trampoline/warning_message_with_new_spark/out.test.toml +++ b/acceptance/bundle/trampoline/warning_message_with_new_spark/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/trampoline/warning_message_with_old_spark/out.test.toml b/acceptance/bundle/trampoline/warning_message_with_old_spark/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/trampoline/warning_message_with_old_spark/out.test.toml +++ b/acceptance/bundle/trampoline/warning_message_with_old_spark/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/undefined_resources/out.test.toml b/acceptance/bundle/undefined_resources/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/undefined_resources/out.test.toml +++ b/acceptance/bundle/undefined_resources/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/upload/internal_server_error/out.test.toml b/acceptance/bundle/upload/internal_server_error/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/upload/internal_server_error/out.test.toml +++ b/acceptance/bundle/upload/internal_server_error/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/upload/timeout/out.test.toml b/acceptance/bundle/upload/timeout/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/upload/timeout/out.test.toml +++ b/acceptance/bundle/upload/timeout/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/user_agent/out.test.toml b/acceptance/bundle/user_agent/out.test.toml index be193812ec2..b827ff3f062 100644 --- a/acceptance/bundle/user_agent/out.test.toml +++ b/acceptance/bundle/user_agent/out.test.toml @@ -1,4 +1,5 @@ Local = true Cloud = false Phase = 1 +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/user_agent/simple/out.test.toml b/acceptance/bundle/user_agent/simple/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/user_agent/simple/out.test.toml +++ b/acceptance/bundle/user_agent/simple/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/anchor_containers/out.test.toml b/acceptance/bundle/validate/anchor_containers/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/validate/anchor_containers/out.test.toml +++ b/acceptance/bundle/validate/anchor_containers/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/catalog_requires_direct_mode/out.test.toml b/acceptance/bundle/validate/catalog_requires_direct_mode/out.test.toml index 65156e0457c..76ce926fd59 100644 --- a/acceptance/bundle/validate/catalog_requires_direct_mode/out.test.toml +++ b/acceptance/bundle/validate/catalog_requires_direct_mode/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform"] diff --git a/acceptance/bundle/validate/dashboard_defaults/out.test.toml b/acceptance/bundle/validate/dashboard_defaults/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/validate/dashboard_defaults/out.test.toml +++ b/acceptance/bundle/validate/dashboard_defaults/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/dashboard_required_name/out.test.toml b/acceptance/bundle/validate/dashboard_required_name/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/validate/dashboard_required_name/out.test.toml +++ b/acceptance/bundle/validate/dashboard_required_name/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/dashboard_required_warehouse_id/out.test.toml b/acceptance/bundle/validate/dashboard_required_warehouse_id/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/validate/dashboard_required_warehouse_id/out.test.toml +++ b/acceptance/bundle/validate/dashboard_required_warehouse_id/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/definitions_yaml_anchors/out.test.toml b/acceptance/bundle/validate/definitions_yaml_anchors/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/validate/definitions_yaml_anchors/out.test.toml +++ b/acceptance/bundle/validate/definitions_yaml_anchors/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/duplicate_yaml_merge_key/out.test.toml b/acceptance/bundle/validate/duplicate_yaml_merge_key/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/validate/duplicate_yaml_merge_key/out.test.toml +++ b/acceptance/bundle/validate/duplicate_yaml_merge_key/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/empty_resources/empty_def/out.test.toml b/acceptance/bundle/validate/empty_resources/empty_def/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/validate/empty_resources/empty_def/out.test.toml +++ b/acceptance/bundle/validate/empty_resources/empty_def/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/empty_resources/empty_dict/out.test.toml b/acceptance/bundle/validate/empty_resources/empty_dict/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/validate/empty_resources/empty_dict/out.test.toml +++ b/acceptance/bundle/validate/empty_resources/empty_dict/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/empty_resources/null/out.test.toml b/acceptance/bundle/validate/empty_resources/null/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/validate/empty_resources/null/out.test.toml +++ b/acceptance/bundle/validate/empty_resources/null/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/empty_resources/with_grants/out.test.toml b/acceptance/bundle/validate/empty_resources/with_grants/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/validate/empty_resources/with_grants/out.test.toml +++ b/acceptance/bundle/validate/empty_resources/with_grants/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/empty_resources/with_permissions/out.test.toml b/acceptance/bundle/validate/empty_resources/with_permissions/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/validate/empty_resources/with_permissions/out.test.toml +++ b/acceptance/bundle/validate/empty_resources/with_permissions/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/empty_tasks/out.test.toml b/acceptance/bundle/validate/empty_tasks/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/validate/empty_tasks/out.test.toml +++ b/acceptance/bundle/validate/empty_tasks/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/engine-config-valid/out.test.toml b/acceptance/bundle/validate/engine-config-valid/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/validate/engine-config-valid/out.test.toml +++ b/acceptance/bundle/validate/engine-config-valid/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/enum/out.test.toml b/acceptance/bundle/validate/enum/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/validate/enum/out.test.toml +++ b/acceptance/bundle/validate/enum/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/enum_resource_refs/out.test.toml b/acceptance/bundle/validate/enum_resource_refs/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/validate/enum_resource_refs/out.test.toml +++ b/acceptance/bundle/validate/enum_resource_refs/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/genie_space_complex/out.test.toml b/acceptance/bundle/validate/genie_space_complex/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/validate/genie_space_complex/out.test.toml +++ b/acceptance/bundle/validate/genie_space_complex/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/genie_space_defaults/out.test.toml b/acceptance/bundle/validate/genie_space_defaults/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/validate/genie_space_defaults/out.test.toml +++ b/acceptance/bundle/validate/genie_space_defaults/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/genie_space_file_path_and_inline/out.test.toml b/acceptance/bundle/validate/genie_space_file_path_and_inline/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/validate/genie_space_file_path_and_inline/out.test.toml +++ b/acceptance/bundle/validate/genie_space_file_path_and_inline/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/grants_required_principal/out.test.toml b/acceptance/bundle/validate/grants_required_principal/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/validate/grants_required_principal/out.test.toml +++ b/acceptance/bundle/validate/grants_required_principal/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/validate/immutable_workspace_paths/out.test.toml b/acceptance/bundle/validate/immutable_workspace_paths/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/validate/immutable_workspace_paths/out.test.toml +++ b/acceptance/bundle/validate/immutable_workspace_paths/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/include_locations/out.test.toml b/acceptance/bundle/validate/include_locations/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/validate/include_locations/out.test.toml +++ b/acceptance/bundle/validate/include_locations/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/invalid-engine-bundle/out.test.toml b/acceptance/bundle/validate/invalid-engine-bundle/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/validate/invalid-engine-bundle/out.test.toml +++ b/acceptance/bundle/validate/invalid-engine-bundle/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/invalid-engine-target/out.test.toml b/acceptance/bundle/validate/invalid-engine-target/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/validate/invalid-engine-target/out.test.toml +++ b/acceptance/bundle/validate/invalid-engine-target/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/job-references/out.test.toml b/acceptance/bundle/validate/job-references/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/validate/job-references/out.test.toml +++ b/acceptance/bundle/validate/job-references/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/model_serving_both_fields_error/out.test.toml b/acceptance/bundle/validate/model_serving_both_fields_error/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/validate/model_serving_both_fields_error/out.test.toml +++ b/acceptance/bundle/validate/model_serving_both_fields_error/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/model_serving_conversion/out.test.toml b/acceptance/bundle/validate/model_serving_conversion/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/validate/model_serving_conversion/out.test.toml +++ b/acceptance/bundle/validate/model_serving_conversion/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/models/missing_name/out.test.toml b/acceptance/bundle/validate/models/missing_name/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/validate/models/missing_name/out.test.toml +++ b/acceptance/bundle/validate/models/missing_name/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/models/user_id/out.test.toml b/acceptance/bundle/validate/models/user_id/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/validate/models/user_id/out.test.toml +++ b/acceptance/bundle/validate/models/user_id/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/no_dashboard_etag/out.test.toml b/acceptance/bundle/validate/no_dashboard_etag/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/validate/no_dashboard_etag/out.test.toml +++ b/acceptance/bundle/validate/no_dashboard_etag/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/no_genie_space_etag/out.test.toml b/acceptance/bundle/validate/no_genie_space_etag/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/validate/no_genie_space_etag/out.test.toml +++ b/acceptance/bundle/validate/no_genie_space_etag/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/validate/permissions/out.test.toml b/acceptance/bundle/validate/permissions/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/validate/permissions/out.test.toml +++ b/acceptance/bundle/validate/permissions/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/permissions_overlap/out.test.toml b/acceptance/bundle/validate/permissions_overlap/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/validate/permissions_overlap/out.test.toml +++ b/acceptance/bundle/validate/permissions_overlap/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/presets_max_concurrent_runs/out.test.toml b/acceptance/bundle/validate/presets_max_concurrent_runs/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/validate/presets_max_concurrent_runs/out.test.toml +++ b/acceptance/bundle/validate/presets_max_concurrent_runs/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/presets_name_prefix/out.test.toml b/acceptance/bundle/validate/presets_name_prefix/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/validate/presets_name_prefix/out.test.toml +++ b/acceptance/bundle/validate/presets_name_prefix/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/presets_name_prefix_dev/out.test.toml b/acceptance/bundle/validate/presets_name_prefix_dev/out.test.toml index e90b6d5d1ba..96a25cb4752 100644 --- a/acceptance/bundle/validate/presets_name_prefix_dev/out.test.toml +++ b/acceptance/bundle/validate/presets_name_prefix_dev/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/validate/presets_tags/out.test.toml b/acceptance/bundle/validate/presets_tags/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/validate/presets_tags/out.test.toml +++ b/acceptance/bundle/validate/presets_tags/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/required/out.test.toml b/acceptance/bundle/validate/required/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/validate/required/out.test.toml +++ b/acceptance/bundle/validate/required/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/reserved_deployment_fields/out.test.toml b/acceptance/bundle/validate/reserved_deployment_fields/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/validate/reserved_deployment_fields/out.test.toml +++ b/acceptance/bundle/validate/reserved_deployment_fields/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/sql_warehouse_required_name/out.test.toml b/acceptance/bundle/validate/sql_warehouse_required_name/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/validate/sql_warehouse_required_name/out.test.toml +++ b/acceptance/bundle/validate/sql_warehouse_required_name/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/strict/out.test.toml b/acceptance/bundle/validate/strict/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/validate/strict/out.test.toml +++ b/acceptance/bundle/validate/strict/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/sync_patterns/out.test.toml b/acceptance/bundle/validate/sync_patterns/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/validate/sync_patterns/out.test.toml +++ b/acceptance/bundle/validate/sync_patterns/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/var_in_bundle_name/out.test.toml b/acceptance/bundle/validate/var_in_bundle_name/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/validate/var_in_bundle_name/out.test.toml +++ b/acceptance/bundle/validate/var_in_bundle_name/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/volume_defaults/out.test.toml b/acceptance/bundle/validate/volume_defaults/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/validate/volume_defaults/out.test.toml +++ b/acceptance/bundle/validate/volume_defaults/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/variables/arg-repeat/out.test.toml b/acceptance/bundle/variables/arg-repeat/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/variables/arg-repeat/out.test.toml +++ b/acceptance/bundle/variables/arg-repeat/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/variables/complex-cross-ref/out.test.toml b/acceptance/bundle/variables/complex-cross-ref/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/variables/complex-cross-ref/out.test.toml +++ b/acceptance/bundle/variables/complex-cross-ref/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/variables/complex-cycle-self/out.test.toml b/acceptance/bundle/variables/complex-cycle-self/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/variables/complex-cycle-self/out.test.toml +++ b/acceptance/bundle/variables/complex-cycle-self/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/variables/complex-cycle/out.test.toml b/acceptance/bundle/variables/complex-cycle/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/variables/complex-cycle/out.test.toml +++ b/acceptance/bundle/variables/complex-cycle/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/variables/complex-simple/out.test.toml b/acceptance/bundle/variables/complex-simple/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/variables/complex-simple/out.test.toml +++ b/acceptance/bundle/variables/complex-simple/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/variables/complex-transitive-deep/out.test.toml b/acceptance/bundle/variables/complex-transitive-deep/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/variables/complex-transitive-deep/out.test.toml +++ b/acceptance/bundle/variables/complex-transitive-deep/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/variables/complex-transitive-deeper/out.test.toml b/acceptance/bundle/variables/complex-transitive-deeper/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/variables/complex-transitive-deeper/out.test.toml +++ b/acceptance/bundle/variables/complex-transitive-deeper/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/variables/complex-transitive/out.test.toml b/acceptance/bundle/variables/complex-transitive/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/variables/complex-transitive/out.test.toml +++ b/acceptance/bundle/variables/complex-transitive/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/variables/complex-with-var-reference/out.test.toml b/acceptance/bundle/variables/complex-with-var-reference/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/variables/complex-with-var-reference/out.test.toml +++ b/acceptance/bundle/variables/complex-with-var-reference/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/variables/complex-within-complex/out.test.toml b/acceptance/bundle/variables/complex-within-complex/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/variables/complex-within-complex/out.test.toml +++ b/acceptance/bundle/variables/complex-within-complex/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/variables/complex/out.test.toml b/acceptance/bundle/variables/complex/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/variables/complex/out.test.toml +++ b/acceptance/bundle/variables/complex/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/variables/complex_multiple_files/out.test.toml b/acceptance/bundle/variables/complex_multiple_files/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/variables/complex_multiple_files/out.test.toml +++ b/acceptance/bundle/variables/complex_multiple_files/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/variables/cycle/out.test.toml b/acceptance/bundle/variables/cycle/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/variables/cycle/out.test.toml +++ b/acceptance/bundle/variables/cycle/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/variables/double_underscore/out.test.toml b/acceptance/bundle/variables/double_underscore/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/variables/double_underscore/out.test.toml +++ b/acceptance/bundle/variables/double_underscore/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/variables/empty/out.test.toml b/acceptance/bundle/variables/empty/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/variables/empty/out.test.toml +++ b/acceptance/bundle/variables/empty/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/variables/env_overrides/out.test.toml b/acceptance/bundle/variables/env_overrides/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/variables/env_overrides/out.test.toml +++ b/acceptance/bundle/variables/env_overrides/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/variables/file-defaults/out.test.toml b/acceptance/bundle/variables/file-defaults/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/variables/file-defaults/out.test.toml +++ b/acceptance/bundle/variables/file-defaults/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/variables/git-branch/out.test.toml b/acceptance/bundle/variables/git-branch/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/variables/git-branch/out.test.toml +++ b/acceptance/bundle/variables/git-branch/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/variables/host/out.test.toml b/acceptance/bundle/variables/host/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/variables/host/out.test.toml +++ b/acceptance/bundle/variables/host/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/variables/int/out.test.toml b/acceptance/bundle/variables/int/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/variables/int/out.test.toml +++ b/acceptance/bundle/variables/int/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/variables/issue_2436/out.test.toml b/acceptance/bundle/variables/issue_2436/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/variables/issue_2436/out.test.toml +++ b/acceptance/bundle/variables/issue_2436/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/variables/issue_3039_lookup_with_ref/out.test.toml b/acceptance/bundle/variables/issue_3039_lookup_with_ref/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/variables/issue_3039_lookup_with_ref/out.test.toml +++ b/acceptance/bundle/variables/issue_3039_lookup_with_ref/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/variables/lookup/out.test.toml b/acceptance/bundle/variables/lookup/out.test.toml index bbc7fcfd1bd..78c615e06e5 100644 --- a/acceptance/bundle/variables/lookup/out.test.toml +++ b/acceptance/bundle/variables/lookup/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/variables/prepend-workspace-var/out.test.toml b/acceptance/bundle/variables/prepend-workspace-var/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/variables/prepend-workspace-var/out.test.toml +++ b/acceptance/bundle/variables/prepend-workspace-var/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/variables/resolve-builtin/out.test.toml b/acceptance/bundle/variables/resolve-builtin/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/variables/resolve-builtin/out.test.toml +++ b/acceptance/bundle/variables/resolve-builtin/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/variables/resolve-empty/out.test.toml b/acceptance/bundle/variables/resolve-empty/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/variables/resolve-empty/out.test.toml +++ b/acceptance/bundle/variables/resolve-empty/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/variables/resolve-field-within-complex/out.test.toml b/acceptance/bundle/variables/resolve-field-within-complex/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/variables/resolve-field-within-complex/out.test.toml +++ b/acceptance/bundle/variables/resolve-field-within-complex/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/variables/resolve-nonstrings/out.test.toml b/acceptance/bundle/variables/resolve-nonstrings/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/variables/resolve-nonstrings/out.test.toml +++ b/acceptance/bundle/variables/resolve-nonstrings/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/variables/resolve-resources-fields/out.test.toml b/acceptance/bundle/variables/resolve-resources-fields/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/variables/resolve-resources-fields/out.test.toml +++ b/acceptance/bundle/variables/resolve-resources-fields/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/variables/resolve-vars-in-root-path/out.test.toml b/acceptance/bundle/variables/resolve-vars-in-root-path/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/variables/resolve-vars-in-root-path/out.test.toml +++ b/acceptance/bundle/variables/resolve-vars-in-root-path/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/variables/unicode_reference/out.test.toml b/acceptance/bundle/variables/unicode_reference/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/variables/unicode_reference/out.test.toml +++ b/acceptance/bundle/variables/unicode_reference/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/variables/vanilla/out.test.toml b/acceptance/bundle/variables/vanilla/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/variables/vanilla/out.test.toml +++ b/acceptance/bundle/variables/vanilla/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/variables/var_in_var/out.test.toml b/acceptance/bundle/variables/var_in_var/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/variables/var_in_var/out.test.toml +++ b/acceptance/bundle/variables/var_in_var/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/variables/variable_in_resource_key/out.test.toml b/acceptance/bundle/variables/variable_in_resource_key/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/variables/variable_in_resource_key/out.test.toml +++ b/acceptance/bundle/variables/variable_in_resource_key/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/variables/variable_overrides_in_target/out.test.toml b/acceptance/bundle/variables/variable_overrides_in_target/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/variables/variable_overrides_in_target/out.test.toml +++ b/acceptance/bundle/variables/variable_overrides_in_target/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/variables/without_definition/out.test.toml b/acceptance/bundle/variables/without_definition/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/variables/without_definition/out.test.toml +++ b/acceptance/bundle/variables/without_definition/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/volume_path/invalid_file/out.test.toml b/acceptance/bundle/volume_path/invalid_file/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/volume_path/invalid_file/out.test.toml +++ b/acceptance/bundle/volume_path/invalid_file/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/volume_path/invalid_resource/out.test.toml b/acceptance/bundle/volume_path/invalid_resource/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/volume_path/invalid_resource/out.test.toml +++ b/acceptance/bundle/volume_path/invalid_resource/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/volume_path/invalid_root/out.test.toml b/acceptance/bundle/volume_path/invalid_root/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/volume_path/invalid_root/out.test.toml +++ b/acceptance/bundle/volume_path/invalid_root/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/volume_path/invalid_state/out.test.toml b/acceptance/bundle/volume_path/invalid_state/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/volume_path/invalid_state/out.test.toml +++ b/acceptance/bundle/volume_path/invalid_state/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/volume_path/valid/out.test.toml b/acceptance/bundle/volume_path/valid/out.test.toml index f784a183258..92ced1275e4 100644 --- a/acceptance/bundle/volume_path/valid/out.test.toml +++ b/acceptance/bundle/volume_path/valid/out.test.toml @@ -1,3 +1,4 @@ Local = true Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] From f49c0ce95f3b9b5a221a5deb44267832ea1b6ca1 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Fri, 7 Aug 2026 18:16:19 +0000 Subject: [PATCH 053/125] bundle: extend the DMS acceptance run across bundle/resources The jobs tests were converted first; this does the rest of bundle/resources the same way - job and pipeline request assertions drop deployment_id and version_id with print_requests.py --del-body, and the tests that capture every request exclude the deployment metadata service's own calls, which bundle/dms asserts instead. bundle/resources now passes with recording both on and off, apart from 23 tests whose plan or state dumps still carry the stamp. Co-authored-by: Isaac --- .../bundle/resources/apps/lifecycle-started-omitted/script | 2 +- .../clusters/deploy/update-and-resize-autoscale/script | 2 +- .../bundle/resources/clusters/deploy/update-and-resize/script | 2 +- .../bundle/resources/dashboards/unpublish-out-of-band/script | 2 +- .../permissions/genie_spaces/current_can_manage/script | 2 +- .../resources/permissions/jobs/current_can_manage_run/script | 4 ++-- .../resources/permissions/jobs/other_can_manage_run/script | 4 ++-- .../resources/permissions/models/current_can_manage/script | 2 +- .../bundle/resources/permissions/target_permissions/script | 2 +- .../resources/pipelines/allow-duplicate-names/output.txt | 2 +- .../bundle/resources/pipelines/allow-duplicate-names/script | 2 +- .../bundle/resources/pipelines/remote_matches_config/script | 2 +- acceptance/bundle/resources/pipelines/update/script | 2 +- .../resources/quality_monitors/change_assets_dir/output.txt | 2 +- .../resources/quality_monitors/change_assets_dir/script | 2 +- .../quality_monitors/change_output_schema_name/output.txt | 2 +- .../quality_monitors/change_output_schema_name/script | 2 +- .../resources/quality_monitors/change_table_name/output.txt | 2 +- .../resources/quality_monitors/change_table_name/script | 2 +- .../bundle/resources/quality_monitors/create/output.txt | 2 +- acceptance/bundle/resources/quality_monitors/create/script | 2 +- .../secret_scopes/delete_scope/out.deploy.requests.txt | 2 +- acceptance/bundle/resources/secret_scopes/delete_scope/script | 2 +- acceptance/bundle/resources/volumes/change-name/script | 2 +- acceptance/bundle/resources/volumes/remote-change-name/script | 2 +- 25 files changed, 27 insertions(+), 27 deletions(-) diff --git a/acceptance/bundle/resources/apps/lifecycle-started-omitted/script b/acceptance/bundle/resources/apps/lifecycle-started-omitted/script index c836adff587..79a1ff2e93b 100644 --- a/acceptance/bundle/resources/apps/lifecycle-started-omitted/script +++ b/acceptance/bundle/resources/apps/lifecycle-started-omitted/script @@ -79,6 +79,6 @@ trace $CLI bundle deploy trace print_app_requests title "(started omitted, app running) -> bundle plan shows no drift" -$CLI bundle plan -o json > LOG.planjson +$CLI bundle plan -o json | jq 'del(.. | objects | .deployment_id?, .version_id?)' > LOG.planjson verify_no_drift.py LOG.planjson echo "Plan: no drift detected" diff --git a/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/script b/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/script index 1370846fe4b..16928c71889 100755 --- a/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/script +++ b/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/script @@ -7,7 +7,7 @@ cleanup() { trap cleanup EXIT $CLI bundle plan > out.plan_.$DATABRICKS_BUNDLE_ENGINE.txt -$CLI bundle plan -o json > out.plan_.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | jq 'del(.. | objects | .deployment_id?, .version_id?)' > out.plan_.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy title "Cluster should exist with num_workers after bundle deployment:\n" diff --git a/acceptance/bundle/resources/clusters/deploy/update-and-resize/script b/acceptance/bundle/resources/clusters/deploy/update-and-resize/script index f2d80d05de0..06cdfc7caaf 100644 --- a/acceptance/bundle/resources/clusters/deploy/update-and-resize/script +++ b/acceptance/bundle/resources/clusters/deploy/update-and-resize/script @@ -7,7 +7,7 @@ cleanup() { trap cleanup EXIT $CLI bundle plan > out.plan_.$DATABRICKS_BUNDLE_ENGINE.txt -$CLI bundle plan -o json > out.plan_.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | jq 'del(.. | objects | .deployment_id?, .version_id?)' > out.plan_.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy title "Cluster should exist after bundle deployment:\n" diff --git a/acceptance/bundle/resources/dashboards/unpublish-out-of-band/script b/acceptance/bundle/resources/dashboards/unpublish-out-of-band/script index 8631b921342..e6267ead0a8 100644 --- a/acceptance/bundle/resources/dashboards/unpublish-out-of-band/script +++ b/acceptance/bundle/resources/dashboards/unpublish-out-of-band/script @@ -25,7 +25,7 @@ trace $CLI lakeview unpublish $DASHBOARD_ID # Direct: shows "update" because Published field changes from false to true trace $CLI bundle plan > out.plan.$DATABRICKS_BUNDLE_ENGINE.txt -$CLI bundle plan -o json > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | jq 'del(.. | objects | .deployment_id?, .version_id?)' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json json_in_json_normalize.py out.plan.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy diff --git a/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/script b/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/script index ae805caebbc..22a34cb117e 100644 --- a/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/script +++ b/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/script @@ -1,7 +1,7 @@ trace $CLI bundle validate -o json | jq .resources.genie_spaces.foo.permissions rm out.requests.txt -$CLI bundle plan -o json > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | jq 'del(.. | objects | .deployment_id?, .version_id?)' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json rm out.requests.txt trace $CLI bundle deploy diff --git a/acceptance/bundle/resources/permissions/jobs/current_can_manage_run/script b/acceptance/bundle/resources/permissions/jobs/current_can_manage_run/script index 96eaf8a4c75..3cfdd63e806 100644 --- a/acceptance/bundle/resources/permissions/jobs/current_can_manage_run/script +++ b/acceptance/bundle/resources/permissions/jobs/current_can_manage_run/script @@ -2,10 +2,10 @@ envsubst < databricks.yml.tmpl > databricks.yml trace $CLI bundle validate -t green -o json | jq .resources trace errcode $CLI bundle deploy -t green -print_requests.py //jobs &> out.deploy.requests.json +print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id,new_settings.deployment.deployment_id,new_settings.deployment.version_id &> out.deploy.requests.json # check plan to ensure there is not drift trace $CLI bundle plan -o json -t green > out.plan.$DATABRICKS_BUNDLE_ENGINE.txt trace errcode $CLI bundle destroy -t green --auto-approve -print_requests.py //jobs &> out.destroy.requests.$DATABRICKS_BUNDLE_ENGINE.json +print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id,new_settings.deployment.deployment_id,new_settings.deployment.version_id &> out.destroy.requests.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/permissions/jobs/other_can_manage_run/script b/acceptance/bundle/resources/permissions/jobs/other_can_manage_run/script index 4c406d91cfa..6d9f9c0453f 100644 --- a/acceptance/bundle/resources/permissions/jobs/other_can_manage_run/script +++ b/acceptance/bundle/resources/permissions/jobs/other_can_manage_run/script @@ -2,7 +2,7 @@ envsubst < databricks.yml.tmpl > databricks.yml trace $CLI bundle validate -t green -o json | jq .resources trace errcode $CLI bundle deploy -t green -print_requests.py //jobs &> out.deploy.requests.json +print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id,new_settings.deployment.deployment_id,new_settings.deployment.version_id &> out.deploy.requests.json trace errcode $CLI bundle destroy -t green --auto-approve -print_requests.py //jobs &> out.destroy.requests.$DATABRICKS_BUNDLE_ENGINE.json +print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id,new_settings.deployment.deployment_id,new_settings.deployment.version_id &> out.destroy.requests.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/permissions/models/current_can_manage/script b/acceptance/bundle/resources/permissions/models/current_can_manage/script index 9ac6f2cd41a..eb4446a728d 100644 --- a/acceptance/bundle/resources/permissions/models/current_can_manage/script +++ b/acceptance/bundle/resources/permissions/models/current_can_manage/script @@ -1,7 +1,7 @@ trace $CLI bundle validate -o json | jq .resources.$RESOURCE.foo.permissions rm out.requests.txt -$CLI bundle plan -o json > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | jq 'del(.. | objects | .deployment_id?, .version_id?)' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json rm out.requests.txt trace errcode $CLI bundle deploy &> out.deploy.txt diff --git a/acceptance/bundle/resources/permissions/target_permissions/script b/acceptance/bundle/resources/permissions/target_permissions/script index 1a07dad637e..67a775cfb49 100644 --- a/acceptance/bundle/resources/permissions/target_permissions/script +++ b/acceptance/bundle/resources/permissions/target_permissions/script @@ -1,5 +1,5 @@ trace $CLI bundle plan -$CLI bundle plan -o json > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | jq 'del(.. | objects | .deployment_id?, .version_id?)' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy print_requests.py //jobs/ > out.requests_create.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/pipelines/allow-duplicate-names/output.txt b/acceptance/bundle/resources/pipelines/allow-duplicate-names/output.txt index 8cea9634565..7f517420608 100644 --- a/acceptance/bundle/resources/pipelines/allow-duplicate-names/output.txt +++ b/acceptance/bundle/resources/pipelines/allow-duplicate-names/output.txt @@ -5,7 +5,7 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> print_requests.py //pipelines +>>> print_requests.py //pipelines --del-body deployment.deployment_id,deployment.version_id,new_settings.deployment.deployment_id,new_settings.deployment.version_id { "method": "POST", "path": "/api/2.0/pipelines", diff --git a/acceptance/bundle/resources/pipelines/allow-duplicate-names/script b/acceptance/bundle/resources/pipelines/allow-duplicate-names/script index d121e073a3d..9055e4ba02b 100644 --- a/acceptance/bundle/resources/pipelines/allow-duplicate-names/script +++ b/acceptance/bundle/resources/pipelines/allow-duplicate-names/script @@ -16,4 +16,4 @@ export PIPELINE_ID # Deploy the bundle that has a pipeline with the same name: trace $CLI bundle deploy -trace print_requests.py //pipelines +trace print_requests.py //pipelines --del-body deployment.deployment_id,deployment.version_id,new_settings.deployment.deployment_id,new_settings.deployment.version_id diff --git a/acceptance/bundle/resources/pipelines/remote_matches_config/script b/acceptance/bundle/resources/pipelines/remote_matches_config/script index 5b9c37402c7..ccbc1936b57 100644 --- a/acceptance/bundle/resources/pipelines/remote_matches_config/script +++ b/acceptance/bundle/resources/pipelines/remote_matches_config/script @@ -15,6 +15,6 @@ r["run_as"] = {"user_name": "changed@example.test"} EOF trace $CLI bundle plan -$CLI bundle plan -o json > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | jq 'del(.. | objects | .deployment_id?, .version_id?)' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json rm out.requests.txt diff --git a/acceptance/bundle/resources/pipelines/update/script b/acceptance/bundle/resources/pipelines/update/script index 255dea2e00d..f48e025aba2 100644 --- a/acceptance/bundle/resources/pipelines/update/script +++ b/acceptance/bundle/resources/pipelines/update/script @@ -4,7 +4,7 @@ touch bar.py trace $CLI bundle deploy print_requests() { - print_requests.py //pipelines + print_requests.py //pipelines --del-body deployment.deployment_id,deployment.version_id,new_settings.deployment.deployment_id,new_settings.deployment.version_id read_state.py pipelines my id name } diff --git a/acceptance/bundle/resources/quality_monitors/change_assets_dir/output.txt b/acceptance/bundle/resources/quality_monitors/change_assets_dir/output.txt index 08f6c53ae17..fba01abcffb 100644 --- a/acceptance/bundle/resources/quality_monitors/change_assets_dir/output.txt +++ b/acceptance/bundle/resources/quality_monitors/change_assets_dir/output.txt @@ -23,7 +23,7 @@ Deployment complete! >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged ->>> print_requests.py ^//import-file/ ^//workspace/ ^//telemetry-ext +>>> print_requests.py ^//api/2.0/bundle ^//import-file/ ^//workspace/ ^//telemetry-ext >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: diff --git a/acceptance/bundle/resources/quality_monitors/change_assets_dir/script b/acceptance/bundle/resources/quality_monitors/change_assets_dir/script index 6caf49a7f4e..99e80f5f70a 100644 --- a/acceptance/bundle/resources/quality_monitors/change_assets_dir/script +++ b/acceptance/bundle/resources/quality_monitors/change_assets_dir/script @@ -26,5 +26,5 @@ trace errcode $CLI bundle plan -o json &> out.plan.$DATABRICKS_BUNDLE_ENGINE.jso rm out.requests.txt trace errcode $CLI bundle deploy &> out.deploy.$DATABRICKS_BUNDLE_ENGINE.txt -trace print_requests.py '^//import-file/' '^//workspace/' '^//telemetry-ext' > out.deploy.requests.$DATABRICKS_BUNDLE_ENGINE.json +trace print_requests.py '^//api/2.0/bundle' '^//import-file/' '^//workspace/' '^//telemetry-ext' > out.deploy.requests.$DATABRICKS_BUNDLE_ENGINE.json trace errcode $CLI bundle plan &> out.plan_after_deploy.$DATABRICKS_BUNDLE_ENGINE.txt diff --git a/acceptance/bundle/resources/quality_monitors/change_output_schema_name/output.txt b/acceptance/bundle/resources/quality_monitors/change_output_schema_name/output.txt index d67ee41975f..c3e638e9907 100644 --- a/acceptance/bundle/resources/quality_monitors/change_output_schema_name/output.txt +++ b/acceptance/bundle/resources/quality_monitors/change_output_schema_name/output.txt @@ -36,7 +36,7 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> print_requests.py ^//import-file/ ^//workspace/ ^//telemetry-ext +>>> print_requests.py ^//api/2.0/bundle ^//import-file/ ^//workspace/ ^//telemetry-ext >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged diff --git a/acceptance/bundle/resources/quality_monitors/change_output_schema_name/script b/acceptance/bundle/resources/quality_monitors/change_output_schema_name/script index 72c57e08401..a14879d79f1 100644 --- a/acceptance/bundle/resources/quality_monitors/change_output_schema_name/script +++ b/acceptance/bundle/resources/quality_monitors/change_output_schema_name/script @@ -27,6 +27,6 @@ trace $CLI bundle plan -o json > out.plan.$DATABRICKS_BUNDLE_ENGINE.json rm out.requests.txt trace $CLI bundle deploy # dashboard_id is output only field that terraform adds -trace print_requests.py '^//import-file/' '^//workspace/' '^//telemetry-ext' | grep -v '"dashboard_id":' > out.deploy.requests.json +trace print_requests.py '^//api/2.0/bundle' '^//import-file/' '^//workspace/' '^//telemetry-ext' | grep -v '"dashboard_id":' > out.deploy.requests.json trace $CLI bundle plan | contains.py "1 unchanged" diff --git a/acceptance/bundle/resources/quality_monitors/change_table_name/output.txt b/acceptance/bundle/resources/quality_monitors/change_table_name/output.txt index 879222aa8a4..9ee37b2f03d 100644 --- a/acceptance/bundle/resources/quality_monitors/change_table_name/output.txt +++ b/acceptance/bundle/resources/quality_monitors/change_table_name/output.txt @@ -23,7 +23,7 @@ Deployment complete! >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged ->>> print_requests.py ^//import-file/ ^//workspace/ ^//telemetry-ext +>>> print_requests.py ^//api/2.0/bundle ^//import-file/ ^//workspace/ ^//telemetry-ext >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: diff --git a/acceptance/bundle/resources/quality_monitors/change_table_name/script b/acceptance/bundle/resources/quality_monitors/change_table_name/script index 891aece1c11..6cf47e95c59 100644 --- a/acceptance/bundle/resources/quality_monitors/change_table_name/script +++ b/acceptance/bundle/resources/quality_monitors/change_table_name/script @@ -26,7 +26,7 @@ trace errcode $CLI bundle plan -o json &> out.plan.$DATABRICKS_BUNDLE_ENGINE.jso rm out.requests.txt trace errcode $CLI bundle deploy &> out.deploy.$DATABRICKS_BUNDLE_ENGINE.txt -trace print_requests.py '^//import-file/' '^//workspace/' '^//telemetry-ext' > out.deploy.requests.$DATABRICKS_BUNDLE_ENGINE.json +trace print_requests.py '^//api/2.0/bundle' '^//import-file/' '^//workspace/' '^//telemetry-ext' > out.deploy.requests.$DATABRICKS_BUNDLE_ENGINE.json trace errcode $CLI bundle plan &> out.plan_after_deploy.$DATABRICKS_BUNDLE_ENGINE.txt trace errcode $CLI quality-monitors get ${TABLE_NAME}_2 2> /dev/null > out.get.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/quality_monitors/create/output.txt b/acceptance/bundle/resources/quality_monitors/create/output.txt index 8037d5ec9cc..5390fd078e6 100644 --- a/acceptance/bundle/resources/quality_monitors/create/output.txt +++ b/acceptance/bundle/resources/quality_monitors/create/output.txt @@ -16,7 +16,7 @@ Table main.qm_test_[UNIQUE_NAME].test_table is now visible (catalog_name=main) >>> [CLI] bundle plan -o json ->>> print_requests.py ^//import-file/ ^//workspace/ ^//telemetry-ext +>>> print_requests.py ^//api/2.0/bundle ^//import-file/ ^//workspace/ ^//telemetry-ext >>> [CLI] bundle plan -o json diff --git a/acceptance/bundle/resources/quality_monitors/create/script b/acceptance/bundle/resources/quality_monitors/create/script index 78c7853b264..22aaeec97aa 100644 --- a/acceptance/bundle/resources/quality_monitors/create/script +++ b/acceptance/bundle/resources/quality_monitors/create/script @@ -21,7 +21,7 @@ trace $CLI bundle plan -o json > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json rm out.requests.txt trace $CLI bundle deploy &> out.deploy.$DATABRICKS_BUNDLE_ENGINE.txt -trace print_requests.py '^//import-file/' '^//workspace/' '^//telemetry-ext' > out.deploy.requests.json +trace print_requests.py '^//api/2.0/bundle' '^//import-file/' '^//workspace/' '^//telemetry-ext' > out.deploy.requests.json # store state to ensure we have table_name there print_state.py | grep name > out.state.$DATABRICKS_BUNDLE_ENGINE.txt diff --git a/acceptance/bundle/resources/secret_scopes/delete_scope/out.deploy.requests.txt b/acceptance/bundle/resources/secret_scopes/delete_scope/out.deploy.requests.txt index 2d469e4abce..e21f54fcf1f 100644 --- a/acceptance/bundle/resources/secret_scopes/delete_scope/out.deploy.requests.txt +++ b/acceptance/bundle/resources/secret_scopes/delete_scope/out.deploy.requests.txt @@ -1,5 +1,5 @@ ->>> print_requests.py ^//import-file/ ^//workspace/ ^//telemetry-ext +>>> print_requests.py ^//api/2.0/bundle ^//import-file/ ^//workspace/ ^//telemetry-ext { "method": "POST", "path": "/api/2.0/secrets/scopes/delete", diff --git a/acceptance/bundle/resources/secret_scopes/delete_scope/script b/acceptance/bundle/resources/secret_scopes/delete_scope/script index b12e98775a3..c7bcc9bf0a5 100755 --- a/acceptance/bundle/resources/secret_scopes/delete_scope/script +++ b/acceptance/bundle/resources/secret_scopes/delete_scope/script @@ -15,4 +15,4 @@ trace $CLI bundle plan &> out.plan.$DATABRICKS_BUNDLE_ENGINE.txt rm out.requests.txt trace $CLI bundle deploy -trace print_requests.py '^//import-file/' '^//workspace/' '^//telemetry-ext' &> out.deploy.requests.txt +trace print_requests.py '^//api/2.0/bundle' '^//import-file/' '^//workspace/' '^//telemetry-ext' &> out.deploy.requests.txt diff --git a/acceptance/bundle/resources/volumes/change-name/script b/acceptance/bundle/resources/volumes/change-name/script index ba4d63c4032..a897616ae12 100644 --- a/acceptance/bundle/resources/volumes/change-name/script +++ b/acceptance/bundle/resources/volumes/change-name/script @@ -10,7 +10,7 @@ trace update_file.py databricks.yml myvolume mynewvolume trace $CLI bundle plan # terraform marks this as "update", direct marks this as "update_with_id" -$CLI bundle plan -o json > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | jq 'del(.. | objects | .deployment_id?, .version_id?)' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py //unity diff --git a/acceptance/bundle/resources/volumes/remote-change-name/script b/acceptance/bundle/resources/volumes/remote-change-name/script index 2485737fdf4..b4ca369abe4 100644 --- a/acceptance/bundle/resources/volumes/remote-change-name/script +++ b/acceptance/bundle/resources/volumes/remote-change-name/script @@ -1,4 +1,4 @@ -$CLI bundle plan -o json > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | jq 'del(.. | objects | .deployment_id?, .version_id?)' > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace $CLI volumes update mycatalog.myschema.myname --json '{"new_name": "my_new_name"}' From 98c50c5907787c6735f37aa6bf12172e07d37da1 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Fri, 7 Aug 2026 18:22:00 +0000 Subject: [PATCH 054/125] bundle: drop the deployment stamp from get responses in the DMS run The last of bundle/resources: `jobs get` and `pipelines get` print the deployment block of the live resource, so with recording on it carries deployment_id and version_id. Each pattern anchors on the "kind" or "metadata_file_path" line that always sits beside them, so it cannot match an unrelated field of the same name. bundle/resources now passes with recording both on and off. Co-authored-by: Isaac --- acceptance/bundle/resources/test.toml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/acceptance/bundle/resources/test.toml b/acceptance/bundle/resources/test.toml index df0bcf4b84a..7d509deb132 100644 --- a/acceptance/bundle/resources/test.toml +++ b/acceptance/bundle/resources/test.toml @@ -25,3 +25,16 @@ New = ''' [[Repls]] Old = '(?m)^ *"changes": \{\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n *\},?\n' New = '' + +# The stamp inside a serialized deployment block, as printed by `jobs get` / `pipelines +# get` and in state dumps. Both keys always sit alongside "kind" and +# "metadata_file_path", so each pattern anchors on one of those - that keeps it from +# matching an unrelated field named deployment_id elsewhere in the output. +[[Repls]] +Old = '(?m)^ *"deployment_id": "\[NUMID\]",\n(?= *"kind": )' +New = '' + +[[Repls]] +Old = '(?m)(^ *"metadata_file_path": [^\n]*),\n *"version_id": "\d+"\n' +New = '''$1 +''' From 518434f94f9f600de6796c72d5ef89f0cefa40a3 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Fri, 7 Aug 2026 19:21:11 +0000 Subject: [PATCH 055/125] bundle: run the resources acceptance tests with deployment history on Record an emptied-out resource as a delete rather than an update. DMS drops a resource from the deployment only for a delete, so an update left it listed with an id but no state, and the next plan failed to unmarshal it ("unexpected end of JSON input" on an emptied grants node). The rest is test-only. The stamp normalization needs Order = 20 so it runs after the root's numeric rules have turned the id into [NUMID], and a variant for gron.py's flattened output. bind/unbind is not supported yet, so the three postgres tests that stage state through unbind opt out of the DMS run. Co-authored-by: Isaac --- .../resources/apps/lifecycle-started/output.txt | 2 +- .../resources/apps/lifecycle-started/script | 2 +- .../replace_existing/out.test.toml | 2 +- .../replace_existing/test.toml | 3 +++ .../inherited-role-conflict/out.test.toml | 2 +- .../inherited-role-conflict/test.toml | 4 ++++ .../replace_existing/out.test.toml | 2 +- .../postgres_roles/replace_existing/test.toml | 3 +++ acceptance/bundle/resources/test.toml | 17 ++++++++++++++--- bundle/direct/apply.go | 6 +++++- 10 files changed, 34 insertions(+), 9 deletions(-) create mode 100644 acceptance/bundle/resources/postgres_databases/replace_existing/test.toml create mode 100644 acceptance/bundle/resources/postgres_roles/replace_existing/test.toml diff --git a/acceptance/bundle/resources/apps/lifecycle-started/output.txt b/acceptance/bundle/resources/apps/lifecycle-started/output.txt index 4562a6b9b22..910f5722e38 100644 --- a/acceptance/bundle/resources/apps/lifecycle-started/output.txt +++ b/acceptance/bundle/resources/apps/lifecycle-started/output.txt @@ -36,7 +36,7 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> print_requests.py //deployments +>>> print_requests.py //deployments ^//api/2.0/bundle { "method": "POST", "path": "/api/2.0/apps/[UNIQUE_NAME]/deployments", diff --git a/acceptance/bundle/resources/apps/lifecycle-started/script b/acceptance/bundle/resources/apps/lifecycle-started/script index 710dec5a10a..15a9197c3c5 100644 --- a/acceptance/bundle/resources/apps/lifecycle-started/script +++ b/acceptance/bundle/resources/apps/lifecycle-started/script @@ -15,7 +15,7 @@ rm -f out.requests.txt title "Re-deploy with description change: code deployed again" trace update_file.py databricks.yml my_app_description MY_APP_DESCRIPTION trace errcode $CLI bundle deploy -trace print_requests.py //deployments +trace print_requests.py //deployments ^//api/2.0/bundle rm -f out.requests.txt title "Stop app externally while config says started=true: plan detects drift" diff --git a/acceptance/bundle/resources/postgres_databases/replace_existing/out.test.toml b/acceptance/bundle/resources/postgres_databases/replace_existing/out.test.toml index 2adb592001c..587c52b624e 100644 --- a/acceptance/bundle/resources/postgres_databases/replace_existing/out.test.toml +++ b/acceptance/bundle/resources/postgres_databases/replace_existing/out.test.toml @@ -3,5 +3,5 @@ Cloud = true RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/resources/postgres_databases/replace_existing/test.toml b/acceptance/bundle/resources/postgres_databases/replace_existing/test.toml new file mode 100644 index 00000000000..31e3a00a75e --- /dev/null +++ b/acceptance/bundle/resources/postgres_databases/replace_existing/test.toml @@ -0,0 +1,3 @@ +# `bundle unbind` does not yet drop the resource from what the deployment metadata service +# reports, so the plan after the unbind still sees the database as tracked. +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] diff --git a/acceptance/bundle/resources/postgres_roles/inherited-role-conflict/out.test.toml b/acceptance/bundle/resources/postgres_roles/inherited-role-conflict/out.test.toml index 67797faa7b2..50ada34cac8 100644 --- a/acceptance/bundle/resources/postgres_roles/inherited-role-conflict/out.test.toml +++ b/acceptance/bundle/resources/postgres_roles/inherited-role-conflict/out.test.toml @@ -3,5 +3,5 @@ Cloud = false RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/postgres_roles/inherited-role-conflict/test.toml b/acceptance/bundle/resources/postgres_roles/inherited-role-conflict/test.toml index 3e475e74819..18183d0f8f0 100644 --- a/acceptance/bundle/resources/postgres_roles/inherited-role-conflict/test.toml +++ b/acceptance/bundle/resources/postgres_roles/inherited-role-conflict/test.toml @@ -5,3 +5,7 @@ Cloud = false # Deploy error wording differs between engines; the conflict itself is engine-agnostic. EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] + +# `bundle unbind` does not yet drop the resource from what the deployment metadata service +# reports, so the role staged above still looks tracked. +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] diff --git a/acceptance/bundle/resources/postgres_roles/replace_existing/out.test.toml b/acceptance/bundle/resources/postgres_roles/replace_existing/out.test.toml index 2adb592001c..587c52b624e 100644 --- a/acceptance/bundle/resources/postgres_roles/replace_existing/out.test.toml +++ b/acceptance/bundle/resources/postgres_roles/replace_existing/out.test.toml @@ -3,5 +3,5 @@ Cloud = true RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/resources/postgres_roles/replace_existing/test.toml b/acceptance/bundle/resources/postgres_roles/replace_existing/test.toml new file mode 100644 index 00000000000..2123f2f1544 --- /dev/null +++ b/acceptance/bundle/resources/postgres_roles/replace_existing/test.toml @@ -0,0 +1,3 @@ +# `bundle unbind` does not yet drop the resource from what the deployment metadata service +# reports, so the plan after the unbind still sees the role as tracked. +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] diff --git a/acceptance/bundle/resources/test.toml b/acceptance/bundle/resources/test.toml index 7d509deb132..d9d6f59b631 100644 --- a/acceptance/bundle/resources/test.toml +++ b/acceptance/bundle/resources/test.toml @@ -30,11 +30,22 @@ New = '' # get` and in state dumps. Both keys always sit alongside "kind" and # "metadata_file_path", so each pattern anchors on one of those - that keeps it from # matching an unrelated field named deployment_id elsewhere in the output. +# +# Order puts these after the root's numeric rules (Order = 10), which have by then turned +# the id into [NUMID]. [[Repls]] -Old = '(?m)^ *"deployment_id": "\[NUMID\]",\n(?= *"kind": )' -New = '' +Old = '(?m)^ *"deployment_id": "\[NUMID\]",\n( *"kind": "BUNDLE")' +New = '${1}' +Order = 20 [[Repls]] -Old = '(?m)(^ *"metadata_file_path": [^\n]*),\n *"version_id": "\d+"\n' +Old = '(?m)(^ *"metadata_file_path": [^\n]*),\n *"version_id": "[^\n]*"\n' New = '''$1 ''' +Order = 20 + +# Same two keys in gron.py's flattened form, where each is its own line. +[[Repls]] +Old = '(?m)^json.*\.deployment\.(deployment_id|version_id) = [^\n]*\n' +New = '' +Order = 20 diff --git a/bundle/direct/apply.go b/bundle/direct/apply.go index e8f95daa48a..d615b5c3566 100644 --- a/bundle/direct/apply.go +++ b/bundle/direct/apply.go @@ -162,7 +162,11 @@ func (d *DeploymentUnit) Update(ctx context.Context, db *dstate.DeploymentState, // The update emptied the resource out (e.g. all grants revoked). Keeping an entry // would report the node as tracked-and-unchanged forever, while a fresh deploy of // the same config plans no node at all; drop it so the two agree. - err = db.DeleteState(ctx, d.ResourceKey, deployplan.Update) + // + // Recorded as a delete, not the update that caused it: the resource is no longer + // tracked, and DMS drops it from the deployment only for a delete. Recording an + // update would leave it listed with no state, which the next plan cannot read. + err = db.DeleteState(ctx, d.ResourceKey, deployplan.Delete) if err != nil { return fmt.Errorf("deleting state id=%s: %w", id, err) } From 9a0a8009f7675d719aceaaad4a2eb5c027fadbab Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Fri, 7 Aug 2026 20:41:34 +0000 Subject: [PATCH 056/125] bundle: run the whole bundle acceptance suite with deployment history on The DATABRICKS_BUNDLE_DMS matrix variable now produces zero failures across acceptance/bundle. Recording is exercised by every test rather than only the handful under bundle/dms. Recording adds a deployment stamp to every job and pipeline, so each subtree that asserts a plan, a state dump, or a request body normalizes those two keys. The rules are repeated per subtree rather than living in the shared parent: bundle/dms asserts the stamp itself and would inherit them. Every pattern is anchored on an adjacent key so it cannot match an unrelated deployment_id, and version_id is required to be non-empty - a terraform state dump carries "version_id": "" for a job it never stamped. Opted out, each with the reason in its test.toml: the saved-plan path (a plan written before the deployment exists carries no stamp), terraform-to-direct migration and continue_293 (resources a pre-DMS deploy created are recorded nowhere), a 1000-task job (state exceeds the 64 KB per-operation limit), bind/unbind, templates (minutes of runtime for no new coverage), and two tests whose assertion is a state byte size or the User-Agent of every request. Co-authored-by: Isaac --- acceptance/bundle/ai_runtime_task/test.toml | 55 ++++++++++++++++++ acceptance/bundle/artifacts/test.toml | 56 +++++++++++++++++++ acceptance/bundle/bundle_tag/test.toml | 56 +++++++++++++++++++ .../deploy/readplan/basic/out.test.toml | 2 +- .../cli-version-mismatch/out.test.toml | 2 +- .../grants-remove-principal/out.test.toml | 2 +- .../readplan/invalid-plan/out.test.toml | 2 +- .../readplan/lineage-mismatch/out.test.toml | 2 +- .../readplan/plan-not-found/out.test.toml | 2 +- .../plan-version-mismatch/out.test.toml | 2 +- .../readplan/postgres_role/out.test.toml | 2 +- .../readplan/serial-mismatch/out.test.toml | 2 +- .../readplan/terraform-error/out.test.toml | 2 +- acceptance/bundle/deploy/readplan/test.toml | 6 ++ .../readplan/unknown-field/out.test.toml | 2 +- acceptance/bundle/deploy/test.toml | 56 +++++++++++++++++++ .../deployment/bind/alert/out.test.toml | 2 +- .../deployment/bind/catalog/out.test.toml | 2 +- .../deployment/bind/cluster/out.test.toml | 2 +- .../deployment/bind/dashboard/out.test.toml | 2 +- .../bind/dashboard/recreation/out.test.toml | 2 +- .../bind/database_instance/out.test.toml | 2 +- .../deployment/bind/experiment/out.test.toml | 2 +- .../bind/external_location/out.test.toml | 2 +- .../deployment/bind/genie_space/out.test.toml | 2 +- .../already-managed-different/out.test.toml | 2 +- .../job/already-managed-same/out.test.toml | 2 +- .../bind/job/engine-from-config/out.test.toml | 2 +- .../bind/job/generate-and-bind/out.test.toml | 2 +- .../bind/job/job-abort-bind/out.test.toml | 2 +- .../job/job-spark-python-task/out.test.toml | 2 +- .../bind/job/noop-job/out.test.toml | 2 +- .../bind/job/python-job/out.test.toml | 2 +- .../bind/job/stale-state/out.test.toml | 2 +- .../bind/model-serving-endpoint/out.test.toml | 2 +- .../bind/pipelines/recreate/out.test.toml | 2 +- .../bind/pipelines/update/out.test.toml | 2 +- .../bind/postgres_database/out.test.toml | 2 +- .../bind/postgres_role/out.test.toml | 2 +- .../bind/quality-monitor/out.test.toml | 2 +- .../bind/registered-model/out.test.toml | 2 +- .../deployment/bind/schema/out.test.toml | 2 +- .../bind/secret-scope/out.test.toml | 2 +- .../bind/sql_warehouse/out.test.toml | 2 +- acceptance/bundle/deployment/bind/test.toml | 2 + .../bind/vector_search_endpoint/out.test.toml | 2 +- .../bind/vector_search_index/out.test.toml | 2 +- .../deployment/bind/volume/out.test.toml | 2 +- acceptance/bundle/deployment/test.toml | 56 +++++++++++++++++++ .../unbind/engine-from-config/out.test.toml | 2 +- .../deployment/unbind/grants/out.test.toml | 2 +- .../deployment/unbind/job/out.test.toml | 2 +- .../unbind/permissions/out.test.toml | 2 +- .../unbind/python-job/out.test.toml | 2 +- acceptance/bundle/deployment/unbind/test.toml | 2 + acceptance/bundle/destroy/test.toml | 55 ++++++++++++++++++ acceptance/bundle/empty_string_dropped/script | 7 ++- .../bundle/empty_string_dropped/test.toml | 56 +++++++++++++++++++ acceptance/bundle/environments/test.toml | 55 ++++++++++++++++++ .../invariant/continue_293/out.test.toml | 2 +- .../bundle/invariant/continue_293/test.toml | 6 ++ .../bundle/invariant/no_drift/test.toml | 6 ++ acceptance/bundle/invariant/test.toml | 56 +++++++++++++++++++ acceptance/bundle/migrate/added/out.test.toml | 2 +- .../migrate/auto-migrate-clean/out.test.toml | 2 +- .../auto-migrate-empty-tfstate/out.test.toml | 2 +- .../migrate/auto-migrate-envvar/out.test.toml | 2 +- .../auto-migrate-push-failure/out.test.toml | 2 +- .../out.test.toml | 2 +- acceptance/bundle/migrate/basic/out.test.toml | 2 +- .../bundle/migrate/dashboards/out.test.toml | 2 +- .../migrate/default-python/out.test.toml | 2 +- .../engine-config-direct/out.test.toml | 2 +- .../engine-config-terraform/out.test.toml | 2 +- .../bundle/migrate/grants/out.test.toml | 2 +- .../bundle/migrate/permissions/out.test.toml | 2 +- .../bundle/migrate/profile_arg/out.test.toml | 2 +- .../bundle/migrate/removed/out.test.toml | 2 +- acceptance/bundle/migrate/runas/out.test.toml | 2 +- acceptance/bundle/migrate/test.toml | 6 ++ .../bundle/migrate/var_arg/out.test.toml | 2 +- .../resource_deps/remote_app_url/output.txt | 6 +- .../resource_deps/remote_app_url/script | 6 +- acceptance/bundle/resource_deps/test.toml | 56 +++++++++++++++++++ acceptance/bundle/resources/test.toml | 44 ++++++++------- acceptance/bundle/run_as/test.toml | 55 ++++++++++++++++++ acceptance/bundle/select/test.toml | 56 +++++++++++++++++++ acceptance/bundle/state/test.toml | 55 ++++++++++++++++++ acceptance/bundle/summary/test.toml | 55 ++++++++++++++++++ .../config-remote-sync-error/out.test.toml | 2 +- .../config-remote-sync-recreate/out.test.toml | 2 +- .../config-remote-sync-save/out.test.toml | 2 +- .../config-remote-sync/out.test.toml | 2 +- .../out.test.toml | 2 +- .../deploy-artifact-path-type/out.test.toml | 2 +- .../deploy-artifacts-variables/out.test.toml | 2 +- .../deploy-compute-type/out.test.toml | 2 +- .../deploy-config-file-count/out.test.toml | 2 +- .../deploy-error-message/out.test.toml | 2 +- .../telemetry/deploy-error/out.test.toml | 2 +- .../deploy-experimental/out.test.toml | 2 +- .../telemetry/deploy-mode/out.test.toml | 2 +- .../deploy-name-prefix/custom/out.test.toml | 2 +- .../mode-development/out.test.toml | 2 +- .../telemetry/deploy-no-uuid/out.test.toml | 2 +- .../telemetry/deploy-run-as/out.test.toml | 2 +- .../deploy-target-count/out.test.toml | 2 +- .../deploy-variable-count/out.test.toml | 2 +- .../deploy-whl-artifacts/out.test.toml | 2 +- .../out.test.toml | 2 +- .../bundle/telemetry/deploy/out.test.toml | 2 +- acceptance/bundle/telemetry/test.toml | 4 ++ .../bundle/templates/dbt-sql/out.test.toml | 2 +- .../default-minimal/python/out.test.toml | 2 +- .../default-minimal/skip/out.test.toml | 2 +- .../default-minimal/sql/out.test.toml | 2 +- .../azure-government/out.test.toml | 2 +- .../default-python/classic/out.test.toml | 2 +- .../combinations/classic/out.test.toml | 2 +- .../combinations/serverless/out.test.toml | 2 +- .../fail-missing-uv/out.test.toml | 2 +- .../integration_classic/out.test.toml | 2 +- .../default-python/no-uc/out.test.toml | 2 +- .../serverless-customcatalog/out.test.toml | 2 +- .../default-python/serverless/out.test.toml | 2 +- .../templates/default-scala/out.test.toml | 2 +- .../templates/default-sql/out.test.toml | 2 +- .../lakeflow-integrations/out.test.toml | 2 +- .../lakeflow-pipelines/python/out.test.toml | 2 +- .../lakeflow-pipelines/sql/out.test.toml | 2 +- .../templates/nested-output/out.test.toml | 2 +- .../pydabs/check-consistency/out.test.toml | 2 +- .../pydabs/check-formatting/out.test.toml | 2 +- .../pydabs/deploy-classic/out.test.toml | 2 +- .../pydabs/init-classic/out.test.toml | 2 +- .../telemetry/custom-template/out.test.toml | 2 +- .../templates/telemetry/dbt-sql/out.test.toml | 2 +- .../telemetry/default-python/out.test.toml | 2 +- .../telemetry/default-sql/out.test.toml | 2 +- acceptance/bundle/templates/test.toml | 6 ++ acceptance/bundle/test.toml | 7 +++ acceptance/bundle/user_agent/out.test.toml | 2 +- .../bundle/user_agent/simple/out.test.toml | 2 +- acceptance/bundle/user_agent/test.toml | 5 ++ 144 files changed, 979 insertions(+), 144 deletions(-) create mode 100644 acceptance/bundle/ai_runtime_task/test.toml create mode 100644 acceptance/bundle/deploy/readplan/test.toml create mode 100644 acceptance/bundle/deployment/bind/test.toml create mode 100644 acceptance/bundle/deployment/unbind/test.toml create mode 100644 acceptance/bundle/destroy/test.toml create mode 100644 acceptance/bundle/environments/test.toml create mode 100644 acceptance/bundle/run_as/test.toml create mode 100644 acceptance/bundle/state/test.toml create mode 100644 acceptance/bundle/summary/test.toml diff --git a/acceptance/bundle/ai_runtime_task/test.toml b/acceptance/bundle/ai_runtime_task/test.toml new file mode 100644 index 00000000000..abf7cc15484 --- /dev/null +++ b/acceptance/bundle/ai_runtime_task/test.toml @@ -0,0 +1,55 @@ +# These normalize the deployment stamp recording adds to every job and pipeline, so a test +# asserts the same golden files whether or not recording is on - that is the point of the +# DMS run, rather than keeping a second copy of 600-odd output files. They are repeated per +# subtree rather than living in the parent because bundle/dms asserts the stamp itself and +# would inherit them. +# +# The stamp as a change the plan reports on its own. It shows up at whatever depth the +# enclosing object sits at, so the indent is matched loosely; the body lines are matched as +# `"key": value` pairs rather than `.*` so the match stops at the entry's own closing brace +# instead of running into its siblings. (Go's regexp is RE2, so the indent cannot be +# captured and back-referenced.) +[[Repls]] +Old = '(?m)^ *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\},\n' +New = '' + +# Same entry when it is the last one in the object, so the comma is on the line before. +[[Repls]] +Old = '(?m),\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n' +New = "\n" + +# When the stamp is the only entry, the whole "changes" object exists because of recording. +# The trailing-comma form comes first: the rule after it would match the same text and +# leave the comma on the previous line dangling. +[[Repls]] +Old = '(?m),\n *"changes": \{\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n *\}\n' +New = "\n" + +[[Repls]] +Old = '(?m)^ *"changes": \{\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n *\},?\n' +New = '' + +# The stamp inside a serialized deployment block, as printed by `jobs get` / `pipelines get` +# and in state dumps. Both keys always sit alongside "kind" and "metadata_file_path", so +# each pattern anchors on one of those - that keeps it from matching an unrelated field +# named deployment_id elsewhere in the output. +# +# Order puts these after the root's numeric rules (Order = 10), which have by then turned +# the id into [NUMID]. +[[Repls]] +Old = '(?m)^ *"deployment_id": "\[NUMID\]",\n( *"kind": "BUNDLE")' +New = '${1}' +Order = 20 + +[[Repls]] +# The value is required to be non-empty: a terraform state dump carries +# "version_id": "" for a job it never stamped, and that line is not ours to drop. +Old = '(?m)(^ *"metadata_file_path": [^\n]*),\n *"version_id": "[^"\n]+"\n' +New = "$1\n" +Order = 20 + +# Same two keys in gron.py's flattened form, where each is its own line. +[[Repls]] +Old = '(?m)^json.*\.deployment\.(deployment_id|version_id) = [^\n]*\n' +New = '' +Order = 20 diff --git a/acceptance/bundle/artifacts/test.toml b/acceptance/bundle/artifacts/test.toml index 61bf8345e7b..a8051bc8ca6 100644 --- a/acceptance/bundle/artifacts/test.toml +++ b/acceptance/bundle/artifacts/test.toml @@ -29,3 +29,59 @@ Response.Body = ''' "spark_version": "13.3.x-scala2.12" } ''' + +# These normalize the deployment stamp recording adds to every job and pipeline, so a test +# asserts the same golden files whether or not recording is on - that is the point of the +# DMS run, rather than keeping a second copy of 600-odd output files. They are repeated per +# subtree rather than living in the parent because bundle/dms asserts the stamp itself and +# would inherit them. +# +# The stamp as a change the plan reports on its own. It shows up at whatever depth the +# enclosing object sits at, so the indent is matched loosely; the body lines are matched as +# `"key": value` pairs rather than `.*` so the match stops at the entry's own closing brace +# instead of running into its siblings. (Go's regexp is RE2, so the indent cannot be +# captured and back-referenced.) +[[Repls]] +Old = '(?m)^ *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\},\n' +New = '' + +# Same entry when it is the last one in the object, so the comma is on the line before. +[[Repls]] +Old = '(?m),\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n' +New = "\n" + +# When the stamp is the only entry, the whole "changes" object exists because of recording. +# The trailing-comma form comes first: the rule after it would match the same text and +# leave the comma on the previous line dangling. +[[Repls]] +Old = '(?m),\n *"changes": \{\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n *\}\n' +New = "\n" + +[[Repls]] +Old = '(?m)^ *"changes": \{\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n *\},?\n' +New = '' + +# The stamp inside a serialized deployment block, as printed by `jobs get` / `pipelines get` +# and in state dumps. Both keys always sit alongside "kind" and "metadata_file_path", so +# each pattern anchors on one of those - that keeps it from matching an unrelated field +# named deployment_id elsewhere in the output. +# +# Order puts these after the root's numeric rules (Order = 10), which have by then turned +# the id into [NUMID]. +[[Repls]] +Old = '(?m)^ *"deployment_id": "\[NUMID\]",\n( *"kind": "BUNDLE")' +New = '${1}' +Order = 20 + +[[Repls]] +# The value is required to be non-empty: a terraform state dump carries +# "version_id": "" for a job it never stamped, and that line is not ours to drop. +Old = '(?m)(^ *"metadata_file_path": [^\n]*),\n *"version_id": "[^"\n]+"\n' +New = "$1\n" +Order = 20 + +# Same two keys in gron.py's flattened form, where each is its own line. +[[Repls]] +Old = '(?m)^json.*\.deployment\.(deployment_id|version_id) = [^\n]*\n' +New = '' +Order = 20 diff --git a/acceptance/bundle/bundle_tag/test.toml b/acceptance/bundle/bundle_tag/test.toml index 8540f9500e6..ea76209cc5a 100644 --- a/acceptance/bundle/bundle_tag/test.toml +++ b/acceptance/bundle/bundle_tag/test.toml @@ -1 +1,57 @@ Badness = "configs with id and url should be rejected" + +# These normalize the deployment stamp recording adds to every job and pipeline, so a test +# asserts the same golden files whether or not recording is on - that is the point of the +# DMS run, rather than keeping a second copy of 600-odd output files. They are repeated per +# subtree rather than living in the parent because bundle/dms asserts the stamp itself and +# would inherit them. +# +# The stamp as a change the plan reports on its own. It shows up at whatever depth the +# enclosing object sits at, so the indent is matched loosely; the body lines are matched as +# `"key": value` pairs rather than `.*` so the match stops at the entry's own closing brace +# instead of running into its siblings. (Go's regexp is RE2, so the indent cannot be +# captured and back-referenced.) +[[Repls]] +Old = '(?m)^ *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\},\n' +New = '' + +# Same entry when it is the last one in the object, so the comma is on the line before. +[[Repls]] +Old = '(?m),\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n' +New = "\n" + +# When the stamp is the only entry, the whole "changes" object exists because of recording. +# The trailing-comma form comes first: the rule after it would match the same text and +# leave the comma on the previous line dangling. +[[Repls]] +Old = '(?m),\n *"changes": \{\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n *\}\n' +New = "\n" + +[[Repls]] +Old = '(?m)^ *"changes": \{\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n *\},?\n' +New = '' + +# The stamp inside a serialized deployment block, as printed by `jobs get` / `pipelines get` +# and in state dumps. Both keys always sit alongside "kind" and "metadata_file_path", so +# each pattern anchors on one of those - that keeps it from matching an unrelated field +# named deployment_id elsewhere in the output. +# +# Order puts these after the root's numeric rules (Order = 10), which have by then turned +# the id into [NUMID]. +[[Repls]] +Old = '(?m)^ *"deployment_id": "\[NUMID\]",\n( *"kind": "BUNDLE")' +New = '${1}' +Order = 20 + +[[Repls]] +# The value is required to be non-empty: a terraform state dump carries +# "version_id": "" for a job it never stamped, and that line is not ours to drop. +Old = '(?m)(^ *"metadata_file_path": [^\n]*),\n *"version_id": "[^"\n]+"\n' +New = "$1\n" +Order = 20 + +# Same two keys in gron.py's flattened form, where each is its own line. +[[Repls]] +Old = '(?m)^json.*\.deployment\.(deployment_id|version_id) = [^\n]*\n' +New = '' +Order = 20 diff --git a/acceptance/bundle/deploy/readplan/basic/out.test.toml b/acceptance/bundle/deploy/readplan/basic/out.test.toml index 96a25cb4752..2a52887146a 100644 --- a/acceptance/bundle/deploy/readplan/basic/out.test.toml +++ b/acceptance/bundle/deploy/readplan/basic/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deploy/readplan/cli-version-mismatch/out.test.toml b/acceptance/bundle/deploy/readplan/cli-version-mismatch/out.test.toml index 96a25cb4752..2a52887146a 100644 --- a/acceptance/bundle/deploy/readplan/cli-version-mismatch/out.test.toml +++ b/acceptance/bundle/deploy/readplan/cli-version-mismatch/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deploy/readplan/grants-remove-principal/out.test.toml b/acceptance/bundle/deploy/readplan/grants-remove-principal/out.test.toml index 2962c9963cc..310be221793 100644 --- a/acceptance/bundle/deploy/readplan/grants-remove-principal/out.test.toml +++ b/acceptance/bundle/deploy/readplan/grants-remove-principal/out.test.toml @@ -1,5 +1,5 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/deploy/readplan/invalid-plan/out.test.toml b/acceptance/bundle/deploy/readplan/invalid-plan/out.test.toml index 96a25cb4752..2a52887146a 100644 --- a/acceptance/bundle/deploy/readplan/invalid-plan/out.test.toml +++ b/acceptance/bundle/deploy/readplan/invalid-plan/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deploy/readplan/lineage-mismatch/out.test.toml b/acceptance/bundle/deploy/readplan/lineage-mismatch/out.test.toml index 96a25cb4752..2a52887146a 100644 --- a/acceptance/bundle/deploy/readplan/lineage-mismatch/out.test.toml +++ b/acceptance/bundle/deploy/readplan/lineage-mismatch/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deploy/readplan/plan-not-found/out.test.toml b/acceptance/bundle/deploy/readplan/plan-not-found/out.test.toml index a3d9e265a64..6f6238f01b3 100644 --- a/acceptance/bundle/deploy/readplan/plan-not-found/out.test.toml +++ b/acceptance/bundle/deploy/readplan/plan-not-found/out.test.toml @@ -1,5 +1,5 @@ Local = true Cloud = false GOOS.windows = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deploy/readplan/plan-version-mismatch/out.test.toml b/acceptance/bundle/deploy/readplan/plan-version-mismatch/out.test.toml index 96a25cb4752..2a52887146a 100644 --- a/acceptance/bundle/deploy/readplan/plan-version-mismatch/out.test.toml +++ b/acceptance/bundle/deploy/readplan/plan-version-mismatch/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deploy/readplan/postgres_role/out.test.toml b/acceptance/bundle/deploy/readplan/postgres_role/out.test.toml index 96a25cb4752..2a52887146a 100644 --- a/acceptance/bundle/deploy/readplan/postgres_role/out.test.toml +++ b/acceptance/bundle/deploy/readplan/postgres_role/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deploy/readplan/serial-mismatch/out.test.toml b/acceptance/bundle/deploy/readplan/serial-mismatch/out.test.toml index 96a25cb4752..2a52887146a 100644 --- a/acceptance/bundle/deploy/readplan/serial-mismatch/out.test.toml +++ b/acceptance/bundle/deploy/readplan/serial-mismatch/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deploy/readplan/terraform-error/out.test.toml b/acceptance/bundle/deploy/readplan/terraform-error/out.test.toml index 76ce926fd59..25ad1a52fcf 100644 --- a/acceptance/bundle/deploy/readplan/terraform-error/out.test.toml +++ b/acceptance/bundle/deploy/readplan/terraform-error/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform"] diff --git a/acceptance/bundle/deploy/readplan/test.toml b/acceptance/bundle/deploy/readplan/test.toml new file mode 100644 index 00000000000..d5628ffe5fe --- /dev/null +++ b/acceptance/bundle/deploy/readplan/test.toml @@ -0,0 +1,6 @@ +# These tests apply a saved plan, which does not carry the deployment stamp: on a first +# deploy there is no deployment to resolve when `bundle plan` runs, so the plan it writes +# leaves the field unset and applying it plans an update the next time. Same reason as +# EnvMatrixExclude.dms_no_readplan in acceptance/bundle/test.toml, which only covers the +# tests that take the saved-plan path through the READPLAN matrix variable. +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] diff --git a/acceptance/bundle/deploy/readplan/unknown-field/out.test.toml b/acceptance/bundle/deploy/readplan/unknown-field/out.test.toml index 96a25cb4752..2a52887146a 100644 --- a/acceptance/bundle/deploy/readplan/unknown-field/out.test.toml +++ b/acceptance/bundle/deploy/readplan/unknown-field/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deploy/test.toml b/acceptance/bundle/deploy/test.toml index 84e8a4a1990..10c2541173a 100644 --- a/acceptance/bundle/deploy/test.toml +++ b/acceptance/bundle/deploy/test.toml @@ -3,3 +3,59 @@ Ignore = [ '.databricks', '__pycache__', ] + +# These normalize the deployment stamp recording adds to every job and pipeline, so a test +# asserts the same golden files whether or not recording is on - that is the point of the +# DMS run, rather than keeping a second copy of 600-odd output files. They are repeated per +# subtree rather than living in the parent because bundle/dms asserts the stamp itself and +# would inherit them. +# +# The stamp as a change the plan reports on its own. It shows up at whatever depth the +# enclosing object sits at, so the indent is matched loosely; the body lines are matched as +# `"key": value` pairs rather than `.*` so the match stops at the entry's own closing brace +# instead of running into its siblings. (Go's regexp is RE2, so the indent cannot be +# captured and back-referenced.) +[[Repls]] +Old = '(?m)^ *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\},\n' +New = '' + +# Same entry when it is the last one in the object, so the comma is on the line before. +[[Repls]] +Old = '(?m),\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n' +New = "\n" + +# When the stamp is the only entry, the whole "changes" object exists because of recording. +# The trailing-comma form comes first: the rule after it would match the same text and +# leave the comma on the previous line dangling. +[[Repls]] +Old = '(?m),\n *"changes": \{\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n *\}\n' +New = "\n" + +[[Repls]] +Old = '(?m)^ *"changes": \{\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n *\},?\n' +New = '' + +# The stamp inside a serialized deployment block, as printed by `jobs get` / `pipelines get` +# and in state dumps. Both keys always sit alongside "kind" and "metadata_file_path", so +# each pattern anchors on one of those - that keeps it from matching an unrelated field +# named deployment_id elsewhere in the output. +# +# Order puts these after the root's numeric rules (Order = 10), which have by then turned +# the id into [NUMID]. +[[Repls]] +Old = '(?m)^ *"deployment_id": "\[NUMID\]",\n( *"kind": "BUNDLE")' +New = '${1}' +Order = 20 + +[[Repls]] +# The value is required to be non-empty: a terraform state dump carries +# "version_id": "" for a job it never stamped, and that line is not ours to drop. +Old = '(?m)(^ *"metadata_file_path": [^\n]*),\n *"version_id": "[^"\n]+"\n' +New = "$1\n" +Order = 20 + +# Same two keys in gron.py's flattened form, where each is its own line. +[[Repls]] +Old = '(?m)^json.*\.deployment\.(deployment_id|version_id) = [^\n]*\n' +New = '' +Order = 20 diff --git a/acceptance/bundle/deployment/bind/alert/out.test.toml b/acceptance/bundle/deployment/bind/alert/out.test.toml index 78c615e06e5..c1c99ad4f9c 100644 --- a/acceptance/bundle/deployment/bind/alert/out.test.toml +++ b/acceptance/bundle/deployment/bind/alert/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/catalog/out.test.toml b/acceptance/bundle/deployment/bind/catalog/out.test.toml index ce8dec17c30..add7ee1060c 100644 --- a/acceptance/bundle/deployment/bind/catalog/out.test.toml +++ b/acceptance/bundle/deployment/bind/catalog/out.test.toml @@ -1,5 +1,5 @@ Local = true Cloud = true RequiresUnityCatalog = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deployment/bind/cluster/out.test.toml b/acceptance/bundle/deployment/bind/cluster/out.test.toml index 3f6826cd945..ea5b0803e06 100644 --- a/acceptance/bundle/deployment/bind/cluster/out.test.toml +++ b/acceptance/bundle/deployment/bind/cluster/out.test.toml @@ -1,5 +1,5 @@ Local = true Cloud = true RequiresCluster = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/dashboard/out.test.toml b/acceptance/bundle/deployment/bind/dashboard/out.test.toml index c35c189b0af..587894f57b0 100644 --- a/acceptance/bundle/deployment/bind/dashboard/out.test.toml +++ b/acceptance/bundle/deployment/bind/dashboard/out.test.toml @@ -1,6 +1,6 @@ Local = true Cloud = true RequiresWarehouse = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/deployment/bind/dashboard/recreation/out.test.toml b/acceptance/bundle/deployment/bind/dashboard/recreation/out.test.toml index c35c189b0af..587894f57b0 100644 --- a/acceptance/bundle/deployment/bind/dashboard/recreation/out.test.toml +++ b/acceptance/bundle/deployment/bind/dashboard/recreation/out.test.toml @@ -1,6 +1,6 @@ Local = true Cloud = true RequiresWarehouse = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/deployment/bind/database_instance/out.test.toml b/acceptance/bundle/deployment/bind/database_instance/out.test.toml index 92ced1275e4..48cc8d7c1c4 100644 --- a/acceptance/bundle/deployment/bind/database_instance/out.test.toml +++ b/acceptance/bundle/deployment/bind/database_instance/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/experiment/out.test.toml b/acceptance/bundle/deployment/bind/experiment/out.test.toml index 78c615e06e5..c1c99ad4f9c 100644 --- a/acceptance/bundle/deployment/bind/experiment/out.test.toml +++ b/acceptance/bundle/deployment/bind/experiment/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/external_location/out.test.toml b/acceptance/bundle/deployment/bind/external_location/out.test.toml index 96a25cb4752..2a52887146a 100644 --- a/acceptance/bundle/deployment/bind/external_location/out.test.toml +++ b/acceptance/bundle/deployment/bind/external_location/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deployment/bind/genie_space/out.test.toml b/acceptance/bundle/deployment/bind/genie_space/out.test.toml index 96a25cb4752..2a52887146a 100644 --- a/acceptance/bundle/deployment/bind/genie_space/out.test.toml +++ b/acceptance/bundle/deployment/bind/genie_space/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deployment/bind/job/already-managed-different/out.test.toml b/acceptance/bundle/deployment/bind/job/already-managed-different/out.test.toml index 92ced1275e4..48cc8d7c1c4 100644 --- a/acceptance/bundle/deployment/bind/job/already-managed-different/out.test.toml +++ b/acceptance/bundle/deployment/bind/job/already-managed-different/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/job/already-managed-same/out.test.toml b/acceptance/bundle/deployment/bind/job/already-managed-same/out.test.toml index 92ced1275e4..48cc8d7c1c4 100644 --- a/acceptance/bundle/deployment/bind/job/already-managed-same/out.test.toml +++ b/acceptance/bundle/deployment/bind/job/already-managed-same/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/job/engine-from-config/out.test.toml b/acceptance/bundle/deployment/bind/job/engine-from-config/out.test.toml index 96a25cb4752..2a52887146a 100644 --- a/acceptance/bundle/deployment/bind/job/engine-from-config/out.test.toml +++ b/acceptance/bundle/deployment/bind/job/engine-from-config/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deployment/bind/job/generate-and-bind/out.test.toml b/acceptance/bundle/deployment/bind/job/generate-and-bind/out.test.toml index 78c615e06e5..c1c99ad4f9c 100644 --- a/acceptance/bundle/deployment/bind/job/generate-and-bind/out.test.toml +++ b/acceptance/bundle/deployment/bind/job/generate-and-bind/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/job/job-abort-bind/out.test.toml b/acceptance/bundle/deployment/bind/job/job-abort-bind/out.test.toml index 78c615e06e5..c1c99ad4f9c 100644 --- a/acceptance/bundle/deployment/bind/job/job-abort-bind/out.test.toml +++ b/acceptance/bundle/deployment/bind/job/job-abort-bind/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/job/job-spark-python-task/out.test.toml b/acceptance/bundle/deployment/bind/job/job-spark-python-task/out.test.toml index 78c615e06e5..c1c99ad4f9c 100644 --- a/acceptance/bundle/deployment/bind/job/job-spark-python-task/out.test.toml +++ b/acceptance/bundle/deployment/bind/job/job-spark-python-task/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/job/noop-job/out.test.toml b/acceptance/bundle/deployment/bind/job/noop-job/out.test.toml index 92ced1275e4..48cc8d7c1c4 100644 --- a/acceptance/bundle/deployment/bind/job/noop-job/out.test.toml +++ b/acceptance/bundle/deployment/bind/job/noop-job/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/job/python-job/out.test.toml b/acceptance/bundle/deployment/bind/job/python-job/out.test.toml index 92ced1275e4..48cc8d7c1c4 100644 --- a/acceptance/bundle/deployment/bind/job/python-job/out.test.toml +++ b/acceptance/bundle/deployment/bind/job/python-job/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/job/stale-state/out.test.toml b/acceptance/bundle/deployment/bind/job/stale-state/out.test.toml index 92ced1275e4..48cc8d7c1c4 100644 --- a/acceptance/bundle/deployment/bind/job/stale-state/out.test.toml +++ b/acceptance/bundle/deployment/bind/job/stale-state/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/model-serving-endpoint/out.test.toml b/acceptance/bundle/deployment/bind/model-serving-endpoint/out.test.toml index 78c615e06e5..c1c99ad4f9c 100644 --- a/acceptance/bundle/deployment/bind/model-serving-endpoint/out.test.toml +++ b/acceptance/bundle/deployment/bind/model-serving-endpoint/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/pipelines/recreate/out.test.toml b/acceptance/bundle/deployment/bind/pipelines/recreate/out.test.toml index 78c615e06e5..c1c99ad4f9c 100644 --- a/acceptance/bundle/deployment/bind/pipelines/recreate/out.test.toml +++ b/acceptance/bundle/deployment/bind/pipelines/recreate/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/pipelines/update/out.test.toml b/acceptance/bundle/deployment/bind/pipelines/update/out.test.toml index 92ced1275e4..48cc8d7c1c4 100644 --- a/acceptance/bundle/deployment/bind/pipelines/update/out.test.toml +++ b/acceptance/bundle/deployment/bind/pipelines/update/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/postgres_database/out.test.toml b/acceptance/bundle/deployment/bind/postgres_database/out.test.toml index 92ced1275e4..48cc8d7c1c4 100644 --- a/acceptance/bundle/deployment/bind/postgres_database/out.test.toml +++ b/acceptance/bundle/deployment/bind/postgres_database/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/postgres_role/out.test.toml b/acceptance/bundle/deployment/bind/postgres_role/out.test.toml index 92ced1275e4..48cc8d7c1c4 100644 --- a/acceptance/bundle/deployment/bind/postgres_role/out.test.toml +++ b/acceptance/bundle/deployment/bind/postgres_role/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/quality-monitor/out.test.toml b/acceptance/bundle/deployment/bind/quality-monitor/out.test.toml index 92ced1275e4..48cc8d7c1c4 100644 --- a/acceptance/bundle/deployment/bind/quality-monitor/out.test.toml +++ b/acceptance/bundle/deployment/bind/quality-monitor/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/registered-model/out.test.toml b/acceptance/bundle/deployment/bind/registered-model/out.test.toml index fdd9f954e6c..4ddf199e1a1 100644 --- a/acceptance/bundle/deployment/bind/registered-model/out.test.toml +++ b/acceptance/bundle/deployment/bind/registered-model/out.test.toml @@ -1,5 +1,5 @@ Local = true Cloud = true RequiresUnityCatalog = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/schema/out.test.toml b/acceptance/bundle/deployment/bind/schema/out.test.toml index fdd9f954e6c..4ddf199e1a1 100644 --- a/acceptance/bundle/deployment/bind/schema/out.test.toml +++ b/acceptance/bundle/deployment/bind/schema/out.test.toml @@ -1,5 +1,5 @@ Local = true Cloud = true RequiresUnityCatalog = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/secret-scope/out.test.toml b/acceptance/bundle/deployment/bind/secret-scope/out.test.toml index fdd9f954e6c..4ddf199e1a1 100644 --- a/acceptance/bundle/deployment/bind/secret-scope/out.test.toml +++ b/acceptance/bundle/deployment/bind/secret-scope/out.test.toml @@ -1,5 +1,5 @@ Local = true Cloud = true RequiresUnityCatalog = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/sql_warehouse/out.test.toml b/acceptance/bundle/deployment/bind/sql_warehouse/out.test.toml index 92ced1275e4..48cc8d7c1c4 100644 --- a/acceptance/bundle/deployment/bind/sql_warehouse/out.test.toml +++ b/acceptance/bundle/deployment/bind/sql_warehouse/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/bind/test.toml b/acceptance/bundle/deployment/bind/test.toml new file mode 100644 index 00000000000..10bef2f1ccb --- /dev/null +++ b/acceptance/bundle/deployment/bind/test.toml @@ -0,0 +1,2 @@ +# Bind operations are not yet supported by the Deployment Metadata Service (DMS) +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] diff --git a/acceptance/bundle/deployment/bind/vector_search_endpoint/out.test.toml b/acceptance/bundle/deployment/bind/vector_search_endpoint/out.test.toml index ce8dec17c30..add7ee1060c 100644 --- a/acceptance/bundle/deployment/bind/vector_search_endpoint/out.test.toml +++ b/acceptance/bundle/deployment/bind/vector_search_endpoint/out.test.toml @@ -1,5 +1,5 @@ Local = true Cloud = true RequiresUnityCatalog = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deployment/bind/vector_search_index/out.test.toml b/acceptance/bundle/deployment/bind/vector_search_index/out.test.toml index 8c71b922b55..a250199143a 100644 --- a/acceptance/bundle/deployment/bind/vector_search_index/out.test.toml +++ b/acceptance/bundle/deployment/bind/vector_search_index/out.test.toml @@ -2,5 +2,5 @@ Local = true Cloud = true CloudSlow = true RequiresUnityCatalog = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deployment/bind/volume/out.test.toml b/acceptance/bundle/deployment/bind/volume/out.test.toml index fdd9f954e6c..4ddf199e1a1 100644 --- a/acceptance/bundle/deployment/bind/volume/out.test.toml +++ b/acceptance/bundle/deployment/bind/volume/out.test.toml @@ -1,5 +1,5 @@ Local = true Cloud = true RequiresUnityCatalog = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/test.toml b/acceptance/bundle/deployment/test.toml index c7c6f58ed6e..32ecf0fa454 100644 --- a/acceptance/bundle/deployment/test.toml +++ b/acceptance/bundle/deployment/test.toml @@ -1 +1,57 @@ Cloud = true + +# These normalize the deployment stamp recording adds to every job and pipeline, so a test +# asserts the same golden files whether or not recording is on - that is the point of the +# DMS run, rather than keeping a second copy of 600-odd output files. They are repeated per +# subtree rather than living in the parent because bundle/dms asserts the stamp itself and +# would inherit them. +# +# The stamp as a change the plan reports on its own. It shows up at whatever depth the +# enclosing object sits at, so the indent is matched loosely; the body lines are matched as +# `"key": value` pairs rather than `.*` so the match stops at the entry's own closing brace +# instead of running into its siblings. (Go's regexp is RE2, so the indent cannot be +# captured and back-referenced.) +[[Repls]] +Old = '(?m)^ *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\},\n' +New = '' + +# Same entry when it is the last one in the object, so the comma is on the line before. +[[Repls]] +Old = '(?m),\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n' +New = "\n" + +# When the stamp is the only entry, the whole "changes" object exists because of recording. +# The trailing-comma form comes first: the rule after it would match the same text and +# leave the comma on the previous line dangling. +[[Repls]] +Old = '(?m),\n *"changes": \{\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n *\}\n' +New = "\n" + +[[Repls]] +Old = '(?m)^ *"changes": \{\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n *\},?\n' +New = '' + +# The stamp inside a serialized deployment block, as printed by `jobs get` / `pipelines get` +# and in state dumps. Both keys always sit alongside "kind" and "metadata_file_path", so +# each pattern anchors on one of those - that keeps it from matching an unrelated field +# named deployment_id elsewhere in the output. +# +# Order puts these after the root's numeric rules (Order = 10), which have by then turned +# the id into [NUMID]. +[[Repls]] +Old = '(?m)^ *"deployment_id": "\[NUMID\]",\n( *"kind": "BUNDLE")' +New = '${1}' +Order = 20 + +[[Repls]] +# The value is required to be non-empty: a terraform state dump carries +# "version_id": "" for a job it never stamped, and that line is not ours to drop. +Old = '(?m)(^ *"metadata_file_path": [^\n]*),\n *"version_id": "[^"\n]+"\n' +New = "$1\n" +Order = 20 + +# Same two keys in gron.py's flattened form, where each is its own line. +[[Repls]] +Old = '(?m)^json.*\.deployment\.(deployment_id|version_id) = [^\n]*\n' +New = '' +Order = 20 diff --git a/acceptance/bundle/deployment/unbind/engine-from-config/out.test.toml b/acceptance/bundle/deployment/unbind/engine-from-config/out.test.toml index 96a25cb4752..2a52887146a 100644 --- a/acceptance/bundle/deployment/unbind/engine-from-config/out.test.toml +++ b/acceptance/bundle/deployment/unbind/engine-from-config/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deployment/unbind/grants/out.test.toml b/acceptance/bundle/deployment/unbind/grants/out.test.toml index fdd9f954e6c..4ddf199e1a1 100644 --- a/acceptance/bundle/deployment/unbind/grants/out.test.toml +++ b/acceptance/bundle/deployment/unbind/grants/out.test.toml @@ -1,5 +1,5 @@ Local = true Cloud = true RequiresUnityCatalog = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/unbind/job/out.test.toml b/acceptance/bundle/deployment/unbind/job/out.test.toml index 92ced1275e4..48cc8d7c1c4 100644 --- a/acceptance/bundle/deployment/unbind/job/out.test.toml +++ b/acceptance/bundle/deployment/unbind/job/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/unbind/permissions/out.test.toml b/acceptance/bundle/deployment/unbind/permissions/out.test.toml index 78c615e06e5..c1c99ad4f9c 100644 --- a/acceptance/bundle/deployment/unbind/permissions/out.test.toml +++ b/acceptance/bundle/deployment/unbind/permissions/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/unbind/python-job/out.test.toml b/acceptance/bundle/deployment/unbind/python-job/out.test.toml index 92ced1275e4..48cc8d7c1c4 100644 --- a/acceptance/bundle/deployment/unbind/python-job/out.test.toml +++ b/acceptance/bundle/deployment/unbind/python-job/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deployment/unbind/test.toml b/acceptance/bundle/deployment/unbind/test.toml new file mode 100644 index 00000000000..2be1dcf74aa --- /dev/null +++ b/acceptance/bundle/deployment/unbind/test.toml @@ -0,0 +1,2 @@ +# Unbind operations are not yet supported by the Deployment Metadata Service (DMS) +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] diff --git a/acceptance/bundle/destroy/test.toml b/acceptance/bundle/destroy/test.toml new file mode 100644 index 00000000000..abf7cc15484 --- /dev/null +++ b/acceptance/bundle/destroy/test.toml @@ -0,0 +1,55 @@ +# These normalize the deployment stamp recording adds to every job and pipeline, so a test +# asserts the same golden files whether or not recording is on - that is the point of the +# DMS run, rather than keeping a second copy of 600-odd output files. They are repeated per +# subtree rather than living in the parent because bundle/dms asserts the stamp itself and +# would inherit them. +# +# The stamp as a change the plan reports on its own. It shows up at whatever depth the +# enclosing object sits at, so the indent is matched loosely; the body lines are matched as +# `"key": value` pairs rather than `.*` so the match stops at the entry's own closing brace +# instead of running into its siblings. (Go's regexp is RE2, so the indent cannot be +# captured and back-referenced.) +[[Repls]] +Old = '(?m)^ *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\},\n' +New = '' + +# Same entry when it is the last one in the object, so the comma is on the line before. +[[Repls]] +Old = '(?m),\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n' +New = "\n" + +# When the stamp is the only entry, the whole "changes" object exists because of recording. +# The trailing-comma form comes first: the rule after it would match the same text and +# leave the comma on the previous line dangling. +[[Repls]] +Old = '(?m),\n *"changes": \{\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n *\}\n' +New = "\n" + +[[Repls]] +Old = '(?m)^ *"changes": \{\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n *\},?\n' +New = '' + +# The stamp inside a serialized deployment block, as printed by `jobs get` / `pipelines get` +# and in state dumps. Both keys always sit alongside "kind" and "metadata_file_path", so +# each pattern anchors on one of those - that keeps it from matching an unrelated field +# named deployment_id elsewhere in the output. +# +# Order puts these after the root's numeric rules (Order = 10), which have by then turned +# the id into [NUMID]. +[[Repls]] +Old = '(?m)^ *"deployment_id": "\[NUMID\]",\n( *"kind": "BUNDLE")' +New = '${1}' +Order = 20 + +[[Repls]] +# The value is required to be non-empty: a terraform state dump carries +# "version_id": "" for a job it never stamped, and that line is not ours to drop. +Old = '(?m)(^ *"metadata_file_path": [^\n]*),\n *"version_id": "[^"\n]+"\n' +New = "$1\n" +Order = 20 + +# Same two keys in gron.py's flattened form, where each is its own line. +[[Repls]] +Old = '(?m)^json.*\.deployment\.(deployment_id|version_id) = [^\n]*\n' +New = '' +Order = 20 diff --git a/acceptance/bundle/empty_string_dropped/script b/acceptance/bundle/empty_string_dropped/script index 973e436617c..f856c966e20 100644 --- a/acceptance/bundle/empty_string_dropped/script +++ b/acceptance/bundle/empty_string_dropped/script @@ -10,11 +10,12 @@ # fix in the initialize phase would drop them, and this golden would show that. $CLI bundle validate -o json -t direct | jq .resources > out.validate.json -# Exclude non-create traffic: workspace file ops and telemetry (nondeterministic). +# Exclude non-create traffic: workspace file ops, telemetry (nondeterministic), and the +# deployment history calls the DMS run adds. trace DATABRICKS_BUNDLE_ENGINE=terraform $CLI bundle deploy -t tf -print_requests.py ^//api/2.0/workspace ^//telemetry --sort > out.requests.terraform.json +print_requests.py ^//api/2.0/workspace ^//telemetry ^//api/2.0/bundle --sort > out.requests.terraform.json trace DATABRICKS_BUNDLE_ENGINE=direct $CLI bundle deploy -t direct -print_requests.py ^//api/2.0/workspace ^//telemetry --sort > out.requests.direct.json +print_requests.py ^//api/2.0/workspace ^//telemetry ^//api/2.0/bundle --sort > out.requests.direct.json $TESTDIR/empty_sent.py diff --git a/acceptance/bundle/empty_string_dropped/test.toml b/acceptance/bundle/empty_string_dropped/test.toml index 51e7bc13e23..ebc74f5b64f 100644 --- a/acceptance/bundle/empty_string_dropped/test.toml +++ b/acceptance/bundle/empty_string_dropped/test.toml @@ -10,3 +10,59 @@ EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] Ignore = [ ".databricks", ] + +# These normalize the deployment stamp recording adds to every job and pipeline, so a test +# asserts the same golden files whether or not recording is on - that is the point of the +# DMS run, rather than keeping a second copy of 600-odd output files. They are repeated per +# subtree rather than living in the parent because bundle/dms asserts the stamp itself and +# would inherit them. +# +# The stamp as a change the plan reports on its own. It shows up at whatever depth the +# enclosing object sits at, so the indent is matched loosely; the body lines are matched as +# `"key": value` pairs rather than `.*` so the match stops at the entry's own closing brace +# instead of running into its siblings. (Go's regexp is RE2, so the indent cannot be +# captured and back-referenced.) +[[Repls]] +Old = '(?m)^ *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\},\n' +New = '' + +# Same entry when it is the last one in the object, so the comma is on the line before. +[[Repls]] +Old = '(?m),\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n' +New = "\n" + +# When the stamp is the only entry, the whole "changes" object exists because of recording. +# The trailing-comma form comes first: the rule after it would match the same text and +# leave the comma on the previous line dangling. +[[Repls]] +Old = '(?m),\n *"changes": \{\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n *\}\n' +New = "\n" + +[[Repls]] +Old = '(?m)^ *"changes": \{\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n *\},?\n' +New = '' + +# The stamp inside a serialized deployment block, as printed by `jobs get` / `pipelines get` +# and in state dumps. Both keys always sit alongside "kind" and "metadata_file_path", so +# each pattern anchors on one of those - that keeps it from matching an unrelated field +# named deployment_id elsewhere in the output. +# +# Order puts these after the root's numeric rules (Order = 10), which have by then turned +# the id into [NUMID]. +[[Repls]] +Old = '(?m)^ *"deployment_id": "\[NUMID\]",\n( *"kind": "BUNDLE")' +New = '${1}' +Order = 20 + +[[Repls]] +# The value is required to be non-empty: a terraform state dump carries +# "version_id": "" for a job it never stamped, and that line is not ours to drop. +Old = '(?m)(^ *"metadata_file_path": [^\n]*),\n *"version_id": "[^"\n]+"\n' +New = "$1\n" +Order = 20 + +# Same two keys in gron.py's flattened form, where each is its own line. +[[Repls]] +Old = '(?m)^json.*\.deployment\.(deployment_id|version_id) = [^\n]*\n' +New = '' +Order = 20 diff --git a/acceptance/bundle/environments/test.toml b/acceptance/bundle/environments/test.toml new file mode 100644 index 00000000000..abf7cc15484 --- /dev/null +++ b/acceptance/bundle/environments/test.toml @@ -0,0 +1,55 @@ +# These normalize the deployment stamp recording adds to every job and pipeline, so a test +# asserts the same golden files whether or not recording is on - that is the point of the +# DMS run, rather than keeping a second copy of 600-odd output files. They are repeated per +# subtree rather than living in the parent because bundle/dms asserts the stamp itself and +# would inherit them. +# +# The stamp as a change the plan reports on its own. It shows up at whatever depth the +# enclosing object sits at, so the indent is matched loosely; the body lines are matched as +# `"key": value` pairs rather than `.*` so the match stops at the entry's own closing brace +# instead of running into its siblings. (Go's regexp is RE2, so the indent cannot be +# captured and back-referenced.) +[[Repls]] +Old = '(?m)^ *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\},\n' +New = '' + +# Same entry when it is the last one in the object, so the comma is on the line before. +[[Repls]] +Old = '(?m),\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n' +New = "\n" + +# When the stamp is the only entry, the whole "changes" object exists because of recording. +# The trailing-comma form comes first: the rule after it would match the same text and +# leave the comma on the previous line dangling. +[[Repls]] +Old = '(?m),\n *"changes": \{\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n *\}\n' +New = "\n" + +[[Repls]] +Old = '(?m)^ *"changes": \{\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n *\},?\n' +New = '' + +# The stamp inside a serialized deployment block, as printed by `jobs get` / `pipelines get` +# and in state dumps. Both keys always sit alongside "kind" and "metadata_file_path", so +# each pattern anchors on one of those - that keeps it from matching an unrelated field +# named deployment_id elsewhere in the output. +# +# Order puts these after the root's numeric rules (Order = 10), which have by then turned +# the id into [NUMID]. +[[Repls]] +Old = '(?m)^ *"deployment_id": "\[NUMID\]",\n( *"kind": "BUNDLE")' +New = '${1}' +Order = 20 + +[[Repls]] +# The value is required to be non-empty: a terraform state dump carries +# "version_id": "" for a job it never stamped, and that line is not ours to drop. +Old = '(?m)(^ *"metadata_file_path": [^\n]*),\n *"version_id": "[^"\n]+"\n' +New = "$1\n" +Order = 20 + +# Same two keys in gron.py's flattened form, where each is its own line. +[[Repls]] +Old = '(?m)^json.*\.deployment\.(deployment_id|version_id) = [^\n]*\n' +New = '' +Order = 20 diff --git a/acceptance/bundle/invariant/continue_293/out.test.toml b/acceptance/bundle/invariant/continue_293/out.test.toml index c9d202227e3..663a49fa779 100644 --- a/acceptance/bundle/invariant/continue_293/out.test.toml +++ b/acceptance/bundle/invariant/continue_293/out.test.toml @@ -1,7 +1,7 @@ Local = true Cloud = true RequiresUnityCatalog = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] EnvMatrix.INPUT_CONFIG = [ "alert.yml.tmpl", diff --git a/acceptance/bundle/invariant/continue_293/test.toml b/acceptance/bundle/invariant/continue_293/test.toml index c6fba9c43fb..5daa377779c 100644 --- a/acceptance/bundle/invariant/continue_293/test.toml +++ b/acceptance/bundle/invariant/continue_293/test.toml @@ -1,3 +1,9 @@ +# The seed deploy runs an old CLI that knows nothing about the deployment metadata +# service, so the resources it creates are recorded nowhere. Reading state from the +# service then finds none and plans a create on top of them. Adopting resources a +# pre-DMS CLI deployed is a migration story of its own, not something this test covers. +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] + # $resources references to permissions and grants are not supported on v0.293.0 EnvMatrixExclude.no_permission_ref = ["INPUT_CONFIG=job_permission_ref.yml.tmpl"] EnvMatrixExclude.no_cross_resource_ref = ["INPUT_CONFIG=job_cross_resource_ref.yml.tmpl"] diff --git a/acceptance/bundle/invariant/no_drift/test.toml b/acceptance/bundle/invariant/no_drift/test.toml index ff8a66c196e..ddcf203eb9a 100644 --- a/acceptance/bundle/invariant/no_drift/test.toml +++ b/acceptance/bundle/invariant/no_drift/test.toml @@ -1 +1,7 @@ EnvMatrix.READPLAN = ["", "1"] + +# A 1000-task job serializes to ~110 KB, over the 64 KB per-operation state limit the +# deployment metadata service accepts. Recording skips the resource with a warning, so it +# is absent from the state the service reports and the next plan wants to create it again. +# Raising the limit or splitting the state is a service-side decision. +EnvMatrixExclude.dms_state_too_large = ["DATABRICKS_BUNDLE_DMS=true", "INPUT_CONFIG=job_pydabs_1000_tasks.yml.tmpl"] diff --git a/acceptance/bundle/invariant/test.toml b/acceptance/bundle/invariant/test.toml index 1d0d883f6d5..d20dc220ff5 100644 --- a/acceptance/bundle/invariant/test.toml +++ b/acceptance/bundle/invariant/test.toml @@ -125,3 +125,59 @@ Response.Body = '{"status": {"state": "SUCCEEDED"}, "manifest": {"schema": {"col [[Server]] Pattern = "DELETE /api/2.1/unity-catalog/tables/{full_name}" Response.Body = '{"status": "OK"}' + +# These normalize the deployment stamp recording adds to every job and pipeline, so a test +# asserts the same golden files whether or not recording is on - that is the point of the +# DMS run, rather than keeping a second copy of 600-odd output files. They are repeated per +# subtree rather than living in the parent because bundle/dms asserts the stamp itself and +# would inherit them. +# +# The stamp as a change the plan reports on its own. It shows up at whatever depth the +# enclosing object sits at, so the indent is matched loosely; the body lines are matched as +# `"key": value` pairs rather than `.*` so the match stops at the entry's own closing brace +# instead of running into its siblings. (Go's regexp is RE2, so the indent cannot be +# captured and back-referenced.) +[[Repls]] +Old = '(?m)^ *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\},\n' +New = '' + +# Same entry when it is the last one in the object, so the comma is on the line before. +[[Repls]] +Old = '(?m),\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n' +New = "\n" + +# When the stamp is the only entry, the whole "changes" object exists because of recording. +# The trailing-comma form comes first: the rule after it would match the same text and +# leave the comma on the previous line dangling. +[[Repls]] +Old = '(?m),\n *"changes": \{\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n *\}\n' +New = "\n" + +[[Repls]] +Old = '(?m)^ *"changes": \{\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n *\},?\n' +New = '' + +# The stamp inside a serialized deployment block, as printed by `jobs get` / `pipelines get` +# and in state dumps. Both keys always sit alongside "kind" and "metadata_file_path", so +# each pattern anchors on one of those - that keeps it from matching an unrelated field +# named deployment_id elsewhere in the output. +# +# Order puts these after the root's numeric rules (Order = 10), which have by then turned +# the id into [NUMID]. +[[Repls]] +Old = '(?m)^ *"deployment_id": "\[NUMID\]",\n( *"kind": "BUNDLE")' +New = '${1}' +Order = 20 + +[[Repls]] +# The value is required to be non-empty: a terraform state dump carries +# "version_id": "" for a job it never stamped, and that line is not ours to drop. +Old = '(?m)(^ *"metadata_file_path": [^\n]*),\n *"version_id": "[^"\n]+"\n' +New = "$1\n" +Order = 20 + +# Same two keys in gron.py's flattened form, where each is its own line. +[[Repls]] +Old = '(?m)^json.*\.deployment\.(deployment_id|version_id) = [^\n]*\n' +New = '' +Order = 20 diff --git a/acceptance/bundle/migrate/added/out.test.toml b/acceptance/bundle/migrate/added/out.test.toml index 96a25cb4752..2a52887146a 100644 --- a/acceptance/bundle/migrate/added/out.test.toml +++ b/acceptance/bundle/migrate/added/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/migrate/auto-migrate-clean/out.test.toml b/acceptance/bundle/migrate/auto-migrate-clean/out.test.toml index 2962c9963cc..310be221793 100644 --- a/acceptance/bundle/migrate/auto-migrate-clean/out.test.toml +++ b/acceptance/bundle/migrate/auto-migrate-clean/out.test.toml @@ -1,5 +1,5 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/migrate/auto-migrate-empty-tfstate/out.test.toml b/acceptance/bundle/migrate/auto-migrate-empty-tfstate/out.test.toml index 96a25cb4752..2a52887146a 100644 --- a/acceptance/bundle/migrate/auto-migrate-empty-tfstate/out.test.toml +++ b/acceptance/bundle/migrate/auto-migrate-empty-tfstate/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/migrate/auto-migrate-envvar/out.test.toml b/acceptance/bundle/migrate/auto-migrate-envvar/out.test.toml index 96a25cb4752..2a52887146a 100644 --- a/acceptance/bundle/migrate/auto-migrate-envvar/out.test.toml +++ b/acceptance/bundle/migrate/auto-migrate-envvar/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/migrate/auto-migrate-push-failure/out.test.toml b/acceptance/bundle/migrate/auto-migrate-push-failure/out.test.toml index 96a25cb4752..2a52887146a 100644 --- a/acceptance/bundle/migrate/auto-migrate-push-failure/out.test.toml +++ b/acceptance/bundle/migrate/auto-migrate-push-failure/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/migrate/auto-migrate-tfbackup-failure/out.test.toml b/acceptance/bundle/migrate/auto-migrate-tfbackup-failure/out.test.toml index 96a25cb4752..2a52887146a 100644 --- a/acceptance/bundle/migrate/auto-migrate-tfbackup-failure/out.test.toml +++ b/acceptance/bundle/migrate/auto-migrate-tfbackup-failure/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/migrate/basic/out.test.toml b/acceptance/bundle/migrate/basic/out.test.toml index 96a25cb4752..2a52887146a 100644 --- a/acceptance/bundle/migrate/basic/out.test.toml +++ b/acceptance/bundle/migrate/basic/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/migrate/dashboards/out.test.toml b/acceptance/bundle/migrate/dashboards/out.test.toml index 96a25cb4752..2a52887146a 100644 --- a/acceptance/bundle/migrate/dashboards/out.test.toml +++ b/acceptance/bundle/migrate/dashboards/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/migrate/default-python/out.test.toml b/acceptance/bundle/migrate/default-python/out.test.toml index 96a25cb4752..2a52887146a 100644 --- a/acceptance/bundle/migrate/default-python/out.test.toml +++ b/acceptance/bundle/migrate/default-python/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/migrate/engine-config-direct/out.test.toml b/acceptance/bundle/migrate/engine-config-direct/out.test.toml index 96a25cb4752..2a52887146a 100644 --- a/acceptance/bundle/migrate/engine-config-direct/out.test.toml +++ b/acceptance/bundle/migrate/engine-config-direct/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/migrate/engine-config-terraform/out.test.toml b/acceptance/bundle/migrate/engine-config-terraform/out.test.toml index 96a25cb4752..2a52887146a 100644 --- a/acceptance/bundle/migrate/engine-config-terraform/out.test.toml +++ b/acceptance/bundle/migrate/engine-config-terraform/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/migrate/grants/out.test.toml b/acceptance/bundle/migrate/grants/out.test.toml index 96a25cb4752..2a52887146a 100644 --- a/acceptance/bundle/migrate/grants/out.test.toml +++ b/acceptance/bundle/migrate/grants/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/migrate/permissions/out.test.toml b/acceptance/bundle/migrate/permissions/out.test.toml index 96a25cb4752..2a52887146a 100644 --- a/acceptance/bundle/migrate/permissions/out.test.toml +++ b/acceptance/bundle/migrate/permissions/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/migrate/profile_arg/out.test.toml b/acceptance/bundle/migrate/profile_arg/out.test.toml index 96a25cb4752..2a52887146a 100644 --- a/acceptance/bundle/migrate/profile_arg/out.test.toml +++ b/acceptance/bundle/migrate/profile_arg/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/migrate/removed/out.test.toml b/acceptance/bundle/migrate/removed/out.test.toml index 96a25cb4752..2a52887146a 100644 --- a/acceptance/bundle/migrate/removed/out.test.toml +++ b/acceptance/bundle/migrate/removed/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/migrate/runas/out.test.toml b/acceptance/bundle/migrate/runas/out.test.toml index 96a25cb4752..2a52887146a 100644 --- a/acceptance/bundle/migrate/runas/out.test.toml +++ b/acceptance/bundle/migrate/runas/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/migrate/test.toml b/acceptance/bundle/migrate/test.toml index 8c4484f983a..964175b8b01 100644 --- a/acceptance/bundle/migrate/test.toml +++ b/acceptance/bundle/migrate/test.toml @@ -7,3 +7,9 @@ Ignore = [".databricks"] # matrix to ["direct"] so CI's engine filter includes these tests without # also running the same script twice per engine. EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] + +# These tests deploy on terraform first and then migrate to direct. Recording is a +# direct-engine feature, so the resources the terraform half creates are recorded nowhere +# and the migration reads state the service does not have. Migrating a deployment onto the +# service is a story of its own; these tests are not it. +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] diff --git a/acceptance/bundle/migrate/var_arg/out.test.toml b/acceptance/bundle/migrate/var_arg/out.test.toml index 96a25cb4752..2a52887146a 100644 --- a/acceptance/bundle/migrate/var_arg/out.test.toml +++ b/acceptance/bundle/migrate/var_arg/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resource_deps/remote_app_url/output.txt b/acceptance/bundle/resource_deps/remote_app_url/output.txt index 4c51088f686..2f48d384b3f 100644 --- a/acceptance/bundle/resource_deps/remote_app_url/output.txt +++ b/acceptance/bundle/resource_deps/remote_app_url/output.txt @@ -14,7 +14,7 @@ create pipelines.mypipeline Plan: 2 to add, 0 to change, 0 to delete, 0 unchanged ->>> print_requests.py ^//import-file/ +>>> print_requests.py ^//import-file/ ^//api/2.0/bundle { "method": "POST", "path": "/api/2.0/workspace/mkdirs", @@ -29,7 +29,7 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> print_requests.py ^//import-file/ +>>> print_requests.py ^//import-file/ ^//api/2.0/bundle { "method": "POST", "path": "/api/2.0/workspace/mkdirs", @@ -98,7 +98,7 @@ All files and directories at the following location will be deleted: /Workspace/ Deleting files... Destroy complete! ->>> print_requests.py --sort ^//import-file/ +>>> print_requests.py --sort ^//import-file/ ^//api/2.0/bundle { "method": "DELETE", "path": "/api/2.0/apps/myapp" diff --git a/acceptance/bundle/resource_deps/remote_app_url/script b/acceptance/bundle/resource_deps/remote_app_url/script index d38692366b3..2ce8c3e3c5e 100644 --- a/acceptance/bundle/resource_deps/remote_app_url/script +++ b/acceptance/bundle/resource_deps/remote_app_url/script @@ -1,9 +1,9 @@ trace $CLI bundle validate trace $CLI bundle plan -trace print_requests.py '^//import-file/' +trace print_requests.py '^//import-file/' '^//api/2.0/bundle' trace $CLI bundle deploy -trace print_requests.py '^//import-file/' +trace print_requests.py '^//import-file/' '^//api/2.0/bundle' trace $CLI bundle destroy --auto-approve -trace print_requests.py --sort '^//import-file/' +trace print_requests.py --sort '^//import-file/' '^//api/2.0/bundle' diff --git a/acceptance/bundle/resource_deps/test.toml b/acceptance/bundle/resource_deps/test.toml index dc29b70c320..405ffae696a 100644 --- a/acceptance/bundle/resource_deps/test.toml +++ b/acceptance/bundle/resource_deps/test.toml @@ -4,3 +4,59 @@ Ignore = [ ".databricks", ".gitignore", ] + +# These normalize the deployment stamp recording adds to every job and pipeline, so a test +# asserts the same golden files whether or not recording is on - that is the point of the +# DMS run, rather than keeping a second copy of 600-odd output files. They are repeated per +# subtree rather than living in the parent because bundle/dms asserts the stamp itself and +# would inherit them. +# +# The stamp as a change the plan reports on its own. It shows up at whatever depth the +# enclosing object sits at, so the indent is matched loosely; the body lines are matched as +# `"key": value` pairs rather than `.*` so the match stops at the entry's own closing brace +# instead of running into its siblings. (Go's regexp is RE2, so the indent cannot be +# captured and back-referenced.) +[[Repls]] +Old = '(?m)^ *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\},\n' +New = '' + +# Same entry when it is the last one in the object, so the comma is on the line before. +[[Repls]] +Old = '(?m),\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n' +New = "\n" + +# When the stamp is the only entry, the whole "changes" object exists because of recording. +# The trailing-comma form comes first: the rule after it would match the same text and +# leave the comma on the previous line dangling. +[[Repls]] +Old = '(?m),\n *"changes": \{\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n *\}\n' +New = "\n" + +[[Repls]] +Old = '(?m)^ *"changes": \{\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n *\},?\n' +New = '' + +# The stamp inside a serialized deployment block, as printed by `jobs get` / `pipelines get` +# and in state dumps. Both keys always sit alongside "kind" and "metadata_file_path", so +# each pattern anchors on one of those - that keeps it from matching an unrelated field +# named deployment_id elsewhere in the output. +# +# Order puts these after the root's numeric rules (Order = 10), which have by then turned +# the id into [NUMID]. +[[Repls]] +Old = '(?m)^ *"deployment_id": "\[NUMID\]",\n( *"kind": "BUNDLE")' +New = '${1}' +Order = 20 + +[[Repls]] +# The value is required to be non-empty: a terraform state dump carries +# "version_id": "" for a job it never stamped, and that line is not ours to drop. +Old = '(?m)(^ *"metadata_file_path": [^\n]*),\n *"version_id": "[^"\n]+"\n' +New = "$1\n" +Order = 20 + +# Same two keys in gron.py's flattened form, where each is its own line. +[[Repls]] +Old = '(?m)^json.*\.deployment\.(deployment_id|version_id) = [^\n]*\n' +New = '' +Order = 20 diff --git a/acceptance/bundle/resources/test.toml b/acceptance/bundle/resources/test.toml index d9d6f59b631..3d68723dbe7 100644 --- a/acceptance/bundle/resources/test.toml +++ b/acceptance/bundle/resources/test.toml @@ -1,16 +1,16 @@ RecordRequests = true -# These normalize what deployment history recording adds to a deploy's output, so a test +# These normalize the deployment stamp recording adds to every job and pipeline, so a test # asserts the same golden files whether or not recording is on - that is the point of the -# DMS run, rather than keeping a second copy of 600-odd output files. They live here -# rather than in the parent so bundle/dms, which asserts the recording itself, does not -# inherit them. +# DMS run, rather than keeping a second copy of 600-odd output files. They are repeated per +# subtree rather than living in the parent because bundle/dms asserts the stamp itself and +# would inherit them. # -# The stamp on jobs and pipelines, which the plan reports as a change of its own. It shows -# up at whatever depth the enclosing object sits at, so the indent is matched loosely; the -# body lines are matched as `"key": value` pairs rather than `.*` so the match stops at the -# entry's own closing brace instead of running into its siblings. (Go's regexp is RE2, so -# the indent cannot be captured and back-referenced.) +# The stamp as a change the plan reports on its own. It shows up at whatever depth the +# enclosing object sits at, so the indent is matched loosely; the body lines are matched as +# `"key": value` pairs rather than `.*` so the match stops at the entry's own closing brace +# instead of running into its siblings. (Go's regexp is RE2, so the indent cannot be +# captured and back-referenced.) [[Repls]] Old = '(?m)^ *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\},\n' New = '' @@ -18,18 +18,23 @@ New = '' # Same entry when it is the last one in the object, so the comma is on the line before. [[Repls]] Old = '(?m),\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n' -New = ''' -''' +New = "\n" + +# When the stamp is the only entry, the whole "changes" object exists because of recording. +# The trailing-comma form comes first: the rule after it would match the same text and +# leave the comma on the previous line dangling. +[[Repls]] +Old = '(?m),\n *"changes": \{\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n *\}\n' +New = "\n" -# And when it is the only entry, the whole "changes" object exists because of recording. [[Repls]] Old = '(?m)^ *"changes": \{\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n *\},?\n' New = '' -# The stamp inside a serialized deployment block, as printed by `jobs get` / `pipelines -# get` and in state dumps. Both keys always sit alongside "kind" and -# "metadata_file_path", so each pattern anchors on one of those - that keeps it from -# matching an unrelated field named deployment_id elsewhere in the output. +# The stamp inside a serialized deployment block, as printed by `jobs get` / `pipelines get` +# and in state dumps. Both keys always sit alongside "kind" and "metadata_file_path", so +# each pattern anchors on one of those - that keeps it from matching an unrelated field +# named deployment_id elsewhere in the output. # # Order puts these after the root's numeric rules (Order = 10), which have by then turned # the id into [NUMID]. @@ -39,9 +44,10 @@ New = '${1}' Order = 20 [[Repls]] -Old = '(?m)(^ *"metadata_file_path": [^\n]*),\n *"version_id": "[^\n]*"\n' -New = '''$1 -''' +# The value is required to be non-empty: a terraform state dump carries +# "version_id": "" for a job it never stamped, and that line is not ours to drop. +Old = '(?m)(^ *"metadata_file_path": [^\n]*),\n *"version_id": "[^"\n]+"\n' +New = "$1\n" Order = 20 # Same two keys in gron.py's flattened form, where each is its own line. diff --git a/acceptance/bundle/run_as/test.toml b/acceptance/bundle/run_as/test.toml new file mode 100644 index 00000000000..abf7cc15484 --- /dev/null +++ b/acceptance/bundle/run_as/test.toml @@ -0,0 +1,55 @@ +# These normalize the deployment stamp recording adds to every job and pipeline, so a test +# asserts the same golden files whether or not recording is on - that is the point of the +# DMS run, rather than keeping a second copy of 600-odd output files. They are repeated per +# subtree rather than living in the parent because bundle/dms asserts the stamp itself and +# would inherit them. +# +# The stamp as a change the plan reports on its own. It shows up at whatever depth the +# enclosing object sits at, so the indent is matched loosely; the body lines are matched as +# `"key": value` pairs rather than `.*` so the match stops at the entry's own closing brace +# instead of running into its siblings. (Go's regexp is RE2, so the indent cannot be +# captured and back-referenced.) +[[Repls]] +Old = '(?m)^ *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\},\n' +New = '' + +# Same entry when it is the last one in the object, so the comma is on the line before. +[[Repls]] +Old = '(?m),\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n' +New = "\n" + +# When the stamp is the only entry, the whole "changes" object exists because of recording. +# The trailing-comma form comes first: the rule after it would match the same text and +# leave the comma on the previous line dangling. +[[Repls]] +Old = '(?m),\n *"changes": \{\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n *\}\n' +New = "\n" + +[[Repls]] +Old = '(?m)^ *"changes": \{\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n *\},?\n' +New = '' + +# The stamp inside a serialized deployment block, as printed by `jobs get` / `pipelines get` +# and in state dumps. Both keys always sit alongside "kind" and "metadata_file_path", so +# each pattern anchors on one of those - that keeps it from matching an unrelated field +# named deployment_id elsewhere in the output. +# +# Order puts these after the root's numeric rules (Order = 10), which have by then turned +# the id into [NUMID]. +[[Repls]] +Old = '(?m)^ *"deployment_id": "\[NUMID\]",\n( *"kind": "BUNDLE")' +New = '${1}' +Order = 20 + +[[Repls]] +# The value is required to be non-empty: a terraform state dump carries +# "version_id": "" for a job it never stamped, and that line is not ours to drop. +Old = '(?m)(^ *"metadata_file_path": [^\n]*),\n *"version_id": "[^"\n]+"\n' +New = "$1\n" +Order = 20 + +# Same two keys in gron.py's flattened form, where each is its own line. +[[Repls]] +Old = '(?m)^json.*\.deployment\.(deployment_id|version_id) = [^\n]*\n' +New = '' +Order = 20 diff --git a/acceptance/bundle/select/test.toml b/acceptance/bundle/select/test.toml index 85ce448afd3..792257f226f 100644 --- a/acceptance/bundle/select/test.toml +++ b/acceptance/bundle/select/test.toml @@ -1,3 +1,59 @@ Local = true Cloud = false Ignore = [".databricks", ".gitignore"] + +# These normalize the deployment stamp recording adds to every job and pipeline, so a test +# asserts the same golden files whether or not recording is on - that is the point of the +# DMS run, rather than keeping a second copy of 600-odd output files. They are repeated per +# subtree rather than living in the parent because bundle/dms asserts the stamp itself and +# would inherit them. +# +# The stamp as a change the plan reports on its own. It shows up at whatever depth the +# enclosing object sits at, so the indent is matched loosely; the body lines are matched as +# `"key": value` pairs rather than `.*` so the match stops at the entry's own closing brace +# instead of running into its siblings. (Go's regexp is RE2, so the indent cannot be +# captured and back-referenced.) +[[Repls]] +Old = '(?m)^ *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\},\n' +New = '' + +# Same entry when it is the last one in the object, so the comma is on the line before. +[[Repls]] +Old = '(?m),\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n' +New = "\n" + +# When the stamp is the only entry, the whole "changes" object exists because of recording. +# The trailing-comma form comes first: the rule after it would match the same text and +# leave the comma on the previous line dangling. +[[Repls]] +Old = '(?m),\n *"changes": \{\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n *\}\n' +New = "\n" + +[[Repls]] +Old = '(?m)^ *"changes": \{\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n *\},?\n' +New = '' + +# The stamp inside a serialized deployment block, as printed by `jobs get` / `pipelines get` +# and in state dumps. Both keys always sit alongside "kind" and "metadata_file_path", so +# each pattern anchors on one of those - that keeps it from matching an unrelated field +# named deployment_id elsewhere in the output. +# +# Order puts these after the root's numeric rules (Order = 10), which have by then turned +# the id into [NUMID]. +[[Repls]] +Old = '(?m)^ *"deployment_id": "\[NUMID\]",\n( *"kind": "BUNDLE")' +New = '${1}' +Order = 20 + +[[Repls]] +# The value is required to be non-empty: a terraform state dump carries +# "version_id": "" for a job it never stamped, and that line is not ours to drop. +Old = '(?m)(^ *"metadata_file_path": [^\n]*),\n *"version_id": "[^"\n]+"\n' +New = "$1\n" +Order = 20 + +# Same two keys in gron.py's flattened form, where each is its own line. +[[Repls]] +Old = '(?m)^json.*\.deployment\.(deployment_id|version_id) = [^\n]*\n' +New = '' +Order = 20 diff --git a/acceptance/bundle/state/test.toml b/acceptance/bundle/state/test.toml new file mode 100644 index 00000000000..abf7cc15484 --- /dev/null +++ b/acceptance/bundle/state/test.toml @@ -0,0 +1,55 @@ +# These normalize the deployment stamp recording adds to every job and pipeline, so a test +# asserts the same golden files whether or not recording is on - that is the point of the +# DMS run, rather than keeping a second copy of 600-odd output files. They are repeated per +# subtree rather than living in the parent because bundle/dms asserts the stamp itself and +# would inherit them. +# +# The stamp as a change the plan reports on its own. It shows up at whatever depth the +# enclosing object sits at, so the indent is matched loosely; the body lines are matched as +# `"key": value` pairs rather than `.*` so the match stops at the entry's own closing brace +# instead of running into its siblings. (Go's regexp is RE2, so the indent cannot be +# captured and back-referenced.) +[[Repls]] +Old = '(?m)^ *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\},\n' +New = '' + +# Same entry when it is the last one in the object, so the comma is on the line before. +[[Repls]] +Old = '(?m),\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n' +New = "\n" + +# When the stamp is the only entry, the whole "changes" object exists because of recording. +# The trailing-comma form comes first: the rule after it would match the same text and +# leave the comma on the previous line dangling. +[[Repls]] +Old = '(?m),\n *"changes": \{\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n *\}\n' +New = "\n" + +[[Repls]] +Old = '(?m)^ *"changes": \{\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n *\},?\n' +New = '' + +# The stamp inside a serialized deployment block, as printed by `jobs get` / `pipelines get` +# and in state dumps. Both keys always sit alongside "kind" and "metadata_file_path", so +# each pattern anchors on one of those - that keeps it from matching an unrelated field +# named deployment_id elsewhere in the output. +# +# Order puts these after the root's numeric rules (Order = 10), which have by then turned +# the id into [NUMID]. +[[Repls]] +Old = '(?m)^ *"deployment_id": "\[NUMID\]",\n( *"kind": "BUNDLE")' +New = '${1}' +Order = 20 + +[[Repls]] +# The value is required to be non-empty: a terraform state dump carries +# "version_id": "" for a job it never stamped, and that line is not ours to drop. +Old = '(?m)(^ *"metadata_file_path": [^\n]*),\n *"version_id": "[^"\n]+"\n' +New = "$1\n" +Order = 20 + +# Same two keys in gron.py's flattened form, where each is its own line. +[[Repls]] +Old = '(?m)^json.*\.deployment\.(deployment_id|version_id) = [^\n]*\n' +New = '' +Order = 20 diff --git a/acceptance/bundle/summary/test.toml b/acceptance/bundle/summary/test.toml new file mode 100644 index 00000000000..abf7cc15484 --- /dev/null +++ b/acceptance/bundle/summary/test.toml @@ -0,0 +1,55 @@ +# These normalize the deployment stamp recording adds to every job and pipeline, so a test +# asserts the same golden files whether or not recording is on - that is the point of the +# DMS run, rather than keeping a second copy of 600-odd output files. They are repeated per +# subtree rather than living in the parent because bundle/dms asserts the stamp itself and +# would inherit them. +# +# The stamp as a change the plan reports on its own. It shows up at whatever depth the +# enclosing object sits at, so the indent is matched loosely; the body lines are matched as +# `"key": value` pairs rather than `.*` so the match stops at the entry's own closing brace +# instead of running into its siblings. (Go's regexp is RE2, so the indent cannot be +# captured and back-referenced.) +[[Repls]] +Old = '(?m)^ *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\},\n' +New = '' + +# Same entry when it is the last one in the object, so the comma is on the line before. +[[Repls]] +Old = '(?m),\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n' +New = "\n" + +# When the stamp is the only entry, the whole "changes" object exists because of recording. +# The trailing-comma form comes first: the rule after it would match the same text and +# leave the comma on the previous line dangling. +[[Repls]] +Old = '(?m),\n *"changes": \{\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n *\}\n' +New = "\n" + +[[Repls]] +Old = '(?m)^ *"changes": \{\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n *\},?\n' +New = '' + +# The stamp inside a serialized deployment block, as printed by `jobs get` / `pipelines get` +# and in state dumps. Both keys always sit alongside "kind" and "metadata_file_path", so +# each pattern anchors on one of those - that keeps it from matching an unrelated field +# named deployment_id elsewhere in the output. +# +# Order puts these after the root's numeric rules (Order = 10), which have by then turned +# the id into [NUMID]. +[[Repls]] +Old = '(?m)^ *"deployment_id": "\[NUMID\]",\n( *"kind": "BUNDLE")' +New = '${1}' +Order = 20 + +[[Repls]] +# The value is required to be non-empty: a terraform state dump carries +# "version_id": "" for a job it never stamped, and that line is not ours to drop. +Old = '(?m)(^ *"metadata_file_path": [^\n]*),\n *"version_id": "[^"\n]+"\n' +New = "$1\n" +Order = 20 + +# Same two keys in gron.py's flattened form, where each is its own line. +[[Repls]] +Old = '(?m)^json.*\.deployment\.(deployment_id|version_id) = [^\n]*\n' +New = '' +Order = 20 diff --git a/acceptance/bundle/telemetry/config-remote-sync-error/out.test.toml b/acceptance/bundle/telemetry/config-remote-sync-error/out.test.toml index 4e97b0db661..ef2b279f225 100644 --- a/acceptance/bundle/telemetry/config-remote-sync-error/out.test.toml +++ b/acceptance/bundle/telemetry/config-remote-sync-error/out.test.toml @@ -1,5 +1,5 @@ Local = true Cloud = false GOOS.windows = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform"] diff --git a/acceptance/bundle/telemetry/config-remote-sync-recreate/out.test.toml b/acceptance/bundle/telemetry/config-remote-sync-recreate/out.test.toml index a3d9e265a64..6f6238f01b3 100644 --- a/acceptance/bundle/telemetry/config-remote-sync-recreate/out.test.toml +++ b/acceptance/bundle/telemetry/config-remote-sync-recreate/out.test.toml @@ -1,5 +1,5 @@ Local = true Cloud = false GOOS.windows = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/telemetry/config-remote-sync-save/out.test.toml b/acceptance/bundle/telemetry/config-remote-sync-save/out.test.toml index a3d9e265a64..6f6238f01b3 100644 --- a/acceptance/bundle/telemetry/config-remote-sync-save/out.test.toml +++ b/acceptance/bundle/telemetry/config-remote-sync-save/out.test.toml @@ -1,5 +1,5 @@ Local = true Cloud = false GOOS.windows = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/telemetry/config-remote-sync/out.test.toml b/acceptance/bundle/telemetry/config-remote-sync/out.test.toml index a3d9e265a64..6f6238f01b3 100644 --- a/acceptance/bundle/telemetry/config-remote-sync/out.test.toml +++ b/acceptance/bundle/telemetry/config-remote-sync/out.test.toml @@ -1,5 +1,5 @@ Local = true Cloud = false GOOS.windows = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/telemetry/deploy-app-lifecycle-started/out.test.toml b/acceptance/bundle/telemetry/deploy-app-lifecycle-started/out.test.toml index 96a25cb4752..2a52887146a 100644 --- a/acceptance/bundle/telemetry/deploy-app-lifecycle-started/out.test.toml +++ b/acceptance/bundle/telemetry/deploy-app-lifecycle-started/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/telemetry/deploy-artifact-path-type/out.test.toml b/acceptance/bundle/telemetry/deploy-artifact-path-type/out.test.toml index 92ced1275e4..48cc8d7c1c4 100644 --- a/acceptance/bundle/telemetry/deploy-artifact-path-type/out.test.toml +++ b/acceptance/bundle/telemetry/deploy-artifact-path-type/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/telemetry/deploy-artifacts-variables/out.test.toml b/acceptance/bundle/telemetry/deploy-artifacts-variables/out.test.toml index 92ced1275e4..48cc8d7c1c4 100644 --- a/acceptance/bundle/telemetry/deploy-artifacts-variables/out.test.toml +++ b/acceptance/bundle/telemetry/deploy-artifacts-variables/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/telemetry/deploy-compute-type/out.test.toml b/acceptance/bundle/telemetry/deploy-compute-type/out.test.toml index 92ced1275e4..48cc8d7c1c4 100644 --- a/acceptance/bundle/telemetry/deploy-compute-type/out.test.toml +++ b/acceptance/bundle/telemetry/deploy-compute-type/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/telemetry/deploy-config-file-count/out.test.toml b/acceptance/bundle/telemetry/deploy-config-file-count/out.test.toml index 92ced1275e4..48cc8d7c1c4 100644 --- a/acceptance/bundle/telemetry/deploy-config-file-count/out.test.toml +++ b/acceptance/bundle/telemetry/deploy-config-file-count/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/telemetry/deploy-error-message/out.test.toml b/acceptance/bundle/telemetry/deploy-error-message/out.test.toml index 92ced1275e4..48cc8d7c1c4 100644 --- a/acceptance/bundle/telemetry/deploy-error-message/out.test.toml +++ b/acceptance/bundle/telemetry/deploy-error-message/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/telemetry/deploy-error/out.test.toml b/acceptance/bundle/telemetry/deploy-error/out.test.toml index 92ced1275e4..48cc8d7c1c4 100644 --- a/acceptance/bundle/telemetry/deploy-error/out.test.toml +++ b/acceptance/bundle/telemetry/deploy-error/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/telemetry/deploy-experimental/out.test.toml b/acceptance/bundle/telemetry/deploy-experimental/out.test.toml index 92ced1275e4..48cc8d7c1c4 100644 --- a/acceptance/bundle/telemetry/deploy-experimental/out.test.toml +++ b/acceptance/bundle/telemetry/deploy-experimental/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/telemetry/deploy-mode/out.test.toml b/acceptance/bundle/telemetry/deploy-mode/out.test.toml index 92ced1275e4..48cc8d7c1c4 100644 --- a/acceptance/bundle/telemetry/deploy-mode/out.test.toml +++ b/acceptance/bundle/telemetry/deploy-mode/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/telemetry/deploy-name-prefix/custom/out.test.toml b/acceptance/bundle/telemetry/deploy-name-prefix/custom/out.test.toml index 92ced1275e4..48cc8d7c1c4 100644 --- a/acceptance/bundle/telemetry/deploy-name-prefix/custom/out.test.toml +++ b/acceptance/bundle/telemetry/deploy-name-prefix/custom/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/telemetry/deploy-name-prefix/mode-development/out.test.toml b/acceptance/bundle/telemetry/deploy-name-prefix/mode-development/out.test.toml index 92ced1275e4..48cc8d7c1c4 100644 --- a/acceptance/bundle/telemetry/deploy-name-prefix/mode-development/out.test.toml +++ b/acceptance/bundle/telemetry/deploy-name-prefix/mode-development/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/telemetry/deploy-no-uuid/out.test.toml b/acceptance/bundle/telemetry/deploy-no-uuid/out.test.toml index 92ced1275e4..48cc8d7c1c4 100644 --- a/acceptance/bundle/telemetry/deploy-no-uuid/out.test.toml +++ b/acceptance/bundle/telemetry/deploy-no-uuid/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/telemetry/deploy-run-as/out.test.toml b/acceptance/bundle/telemetry/deploy-run-as/out.test.toml index 92ced1275e4..48cc8d7c1c4 100644 --- a/acceptance/bundle/telemetry/deploy-run-as/out.test.toml +++ b/acceptance/bundle/telemetry/deploy-run-as/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/telemetry/deploy-target-count/out.test.toml b/acceptance/bundle/telemetry/deploy-target-count/out.test.toml index 92ced1275e4..48cc8d7c1c4 100644 --- a/acceptance/bundle/telemetry/deploy-target-count/out.test.toml +++ b/acceptance/bundle/telemetry/deploy-target-count/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/telemetry/deploy-variable-count/out.test.toml b/acceptance/bundle/telemetry/deploy-variable-count/out.test.toml index 92ced1275e4..48cc8d7c1c4 100644 --- a/acceptance/bundle/telemetry/deploy-variable-count/out.test.toml +++ b/acceptance/bundle/telemetry/deploy-variable-count/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/telemetry/deploy-whl-artifacts/out.test.toml b/acceptance/bundle/telemetry/deploy-whl-artifacts/out.test.toml index 92ced1275e4..48cc8d7c1c4 100644 --- a/acceptance/bundle/telemetry/deploy-whl-artifacts/out.test.toml +++ b/acceptance/bundle/telemetry/deploy-whl-artifacts/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/telemetry/deploy-workspace-folder-permissions/out.test.toml b/acceptance/bundle/telemetry/deploy-workspace-folder-permissions/out.test.toml index 92ced1275e4..48cc8d7c1c4 100644 --- a/acceptance/bundle/telemetry/deploy-workspace-folder-permissions/out.test.toml +++ b/acceptance/bundle/telemetry/deploy-workspace-folder-permissions/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/telemetry/deploy/out.test.toml b/acceptance/bundle/telemetry/deploy/out.test.toml index 92ced1275e4..48cc8d7c1c4 100644 --- a/acceptance/bundle/telemetry/deploy/out.test.toml +++ b/acceptance/bundle/telemetry/deploy/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/telemetry/test.toml b/acceptance/bundle/telemetry/test.toml index 14453c07f92..92804cc8ed9 100644 --- a/acceptance/bundle/telemetry/test.toml +++ b/acceptance/bundle/telemetry/test.toml @@ -1,6 +1,10 @@ RecordRequests = true IncludeRequestHeaders = ["User-Agent"] +# Telemetry reports the byte size of the serialized state, which the deployment stamp +# legitimately grows. The number is the assertion here, so there is nothing to normalize. +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] + [Env] DATABRICKS_CACHE_ENABLED = 'false' diff --git a/acceptance/bundle/templates/dbt-sql/out.test.toml b/acceptance/bundle/templates/dbt-sql/out.test.toml index 92ced1275e4..48cc8d7c1c4 100644 --- a/acceptance/bundle/templates/dbt-sql/out.test.toml +++ b/acceptance/bundle/templates/dbt-sql/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/templates/default-minimal/python/out.test.toml b/acceptance/bundle/templates/default-minimal/python/out.test.toml index 92ced1275e4..48cc8d7c1c4 100644 --- a/acceptance/bundle/templates/default-minimal/python/out.test.toml +++ b/acceptance/bundle/templates/default-minimal/python/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/templates/default-minimal/skip/out.test.toml b/acceptance/bundle/templates/default-minimal/skip/out.test.toml index 92ced1275e4..48cc8d7c1c4 100644 --- a/acceptance/bundle/templates/default-minimal/skip/out.test.toml +++ b/acceptance/bundle/templates/default-minimal/skip/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/templates/default-minimal/sql/out.test.toml b/acceptance/bundle/templates/default-minimal/sql/out.test.toml index 92ced1275e4..48cc8d7c1c4 100644 --- a/acceptance/bundle/templates/default-minimal/sql/out.test.toml +++ b/acceptance/bundle/templates/default-minimal/sql/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/templates/default-python/azure-government/out.test.toml b/acceptance/bundle/templates/default-python/azure-government/out.test.toml index 92ced1275e4..48cc8d7c1c4 100644 --- a/acceptance/bundle/templates/default-python/azure-government/out.test.toml +++ b/acceptance/bundle/templates/default-python/azure-government/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/templates/default-python/classic/out.test.toml b/acceptance/bundle/templates/default-python/classic/out.test.toml index 99483caeee6..8a113e1dfd4 100644 --- a/acceptance/bundle/templates/default-python/classic/out.test.toml +++ b/acceptance/bundle/templates/default-python/classic/out.test.toml @@ -1,6 +1,6 @@ Local = true Cloud = false Phase = 1 -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/templates/default-python/combinations/classic/out.test.toml b/acceptance/bundle/templates/default-python/combinations/classic/out.test.toml index 79230bd2367..2b02b68bfbb 100644 --- a/acceptance/bundle/templates/default-python/combinations/classic/out.test.toml +++ b/acceptance/bundle/templates/default-python/combinations/classic/out.test.toml @@ -1,6 +1,6 @@ Local = true Cloud = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] EnvMatrix.DLT = ["yes", "no"] EnvMatrix.NBOOK = ["yes", "no"] diff --git a/acceptance/bundle/templates/default-python/combinations/serverless/out.test.toml b/acceptance/bundle/templates/default-python/combinations/serverless/out.test.toml index 79230bd2367..2b02b68bfbb 100644 --- a/acceptance/bundle/templates/default-python/combinations/serverless/out.test.toml +++ b/acceptance/bundle/templates/default-python/combinations/serverless/out.test.toml @@ -1,6 +1,6 @@ Local = true Cloud = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] EnvMatrix.DLT = ["yes", "no"] EnvMatrix.NBOOK = ["yes", "no"] diff --git a/acceptance/bundle/templates/default-python/fail-missing-uv/out.test.toml b/acceptance/bundle/templates/default-python/fail-missing-uv/out.test.toml index 92ced1275e4..48cc8d7c1c4 100644 --- a/acceptance/bundle/templates/default-python/fail-missing-uv/out.test.toml +++ b/acceptance/bundle/templates/default-python/fail-missing-uv/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/templates/default-python/integration_classic/out.test.toml b/acceptance/bundle/templates/default-python/integration_classic/out.test.toml index ed19028b891..e5eb55f33db 100644 --- a/acceptance/bundle/templates/default-python/integration_classic/out.test.toml +++ b/acceptance/bundle/templates/default-python/integration_classic/out.test.toml @@ -1,6 +1,6 @@ Local = true Cloud = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] EnvMatrix.UV_PYTHON = [ "3.9", diff --git a/acceptance/bundle/templates/default-python/no-uc/out.test.toml b/acceptance/bundle/templates/default-python/no-uc/out.test.toml index 92ced1275e4..48cc8d7c1c4 100644 --- a/acceptance/bundle/templates/default-python/no-uc/out.test.toml +++ b/acceptance/bundle/templates/default-python/no-uc/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/templates/default-python/serverless-customcatalog/out.test.toml b/acceptance/bundle/templates/default-python/serverless-customcatalog/out.test.toml index b827ff3f062..40a9dbfa26b 100644 --- a/acceptance/bundle/templates/default-python/serverless-customcatalog/out.test.toml +++ b/acceptance/bundle/templates/default-python/serverless-customcatalog/out.test.toml @@ -1,5 +1,5 @@ Local = true Cloud = false Phase = 1 -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/templates/default-python/serverless/out.test.toml b/acceptance/bundle/templates/default-python/serverless/out.test.toml index 92ced1275e4..48cc8d7c1c4 100644 --- a/acceptance/bundle/templates/default-python/serverless/out.test.toml +++ b/acceptance/bundle/templates/default-python/serverless/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/templates/default-scala/out.test.toml b/acceptance/bundle/templates/default-scala/out.test.toml index 92ced1275e4..48cc8d7c1c4 100644 --- a/acceptance/bundle/templates/default-scala/out.test.toml +++ b/acceptance/bundle/templates/default-scala/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/templates/default-sql/out.test.toml b/acceptance/bundle/templates/default-sql/out.test.toml index 92ced1275e4..48cc8d7c1c4 100644 --- a/acceptance/bundle/templates/default-sql/out.test.toml +++ b/acceptance/bundle/templates/default-sql/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/templates/lakeflow-integrations/out.test.toml b/acceptance/bundle/templates/lakeflow-integrations/out.test.toml index 92ced1275e4..48cc8d7c1c4 100644 --- a/acceptance/bundle/templates/lakeflow-integrations/out.test.toml +++ b/acceptance/bundle/templates/lakeflow-integrations/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/templates/lakeflow-pipelines/python/out.test.toml b/acceptance/bundle/templates/lakeflow-pipelines/python/out.test.toml index 92ced1275e4..48cc8d7c1c4 100644 --- a/acceptance/bundle/templates/lakeflow-pipelines/python/out.test.toml +++ b/acceptance/bundle/templates/lakeflow-pipelines/python/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/templates/lakeflow-pipelines/sql/out.test.toml b/acceptance/bundle/templates/lakeflow-pipelines/sql/out.test.toml index 92ced1275e4..48cc8d7c1c4 100644 --- a/acceptance/bundle/templates/lakeflow-pipelines/sql/out.test.toml +++ b/acceptance/bundle/templates/lakeflow-pipelines/sql/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/templates/nested-output/out.test.toml b/acceptance/bundle/templates/nested-output/out.test.toml index 92ced1275e4..48cc8d7c1c4 100644 --- a/acceptance/bundle/templates/nested-output/out.test.toml +++ b/acceptance/bundle/templates/nested-output/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/templates/pydabs/check-consistency/out.test.toml b/acceptance/bundle/templates/pydabs/check-consistency/out.test.toml index 464dbdb3ab7..8f126618fbc 100644 --- a/acceptance/bundle/templates/pydabs/check-consistency/out.test.toml +++ b/acceptance/bundle/templates/pydabs/check-consistency/out.test.toml @@ -1,6 +1,6 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] EnvMatrix.INCLUDE_JOB = ["yes", "no"] EnvMatrix.INCLUDE_PIPELINE = ["yes", "no"] diff --git a/acceptance/bundle/templates/pydabs/check-formatting/out.test.toml b/acceptance/bundle/templates/pydabs/check-formatting/out.test.toml index 464dbdb3ab7..8f126618fbc 100644 --- a/acceptance/bundle/templates/pydabs/check-formatting/out.test.toml +++ b/acceptance/bundle/templates/pydabs/check-formatting/out.test.toml @@ -1,6 +1,6 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] EnvMatrix.INCLUDE_JOB = ["yes", "no"] EnvMatrix.INCLUDE_PIPELINE = ["yes", "no"] diff --git a/acceptance/bundle/templates/pydabs/deploy-classic/out.test.toml b/acceptance/bundle/templates/pydabs/deploy-classic/out.test.toml index 92ced1275e4..48cc8d7c1c4 100644 --- a/acceptance/bundle/templates/pydabs/deploy-classic/out.test.toml +++ b/acceptance/bundle/templates/pydabs/deploy-classic/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/templates/pydabs/init-classic/out.test.toml b/acceptance/bundle/templates/pydabs/init-classic/out.test.toml index 92ced1275e4..48cc8d7c1c4 100644 --- a/acceptance/bundle/templates/pydabs/init-classic/out.test.toml +++ b/acceptance/bundle/templates/pydabs/init-classic/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/templates/telemetry/custom-template/out.test.toml b/acceptance/bundle/templates/telemetry/custom-template/out.test.toml index 92ced1275e4..48cc8d7c1c4 100644 --- a/acceptance/bundle/templates/telemetry/custom-template/out.test.toml +++ b/acceptance/bundle/templates/telemetry/custom-template/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/templates/telemetry/dbt-sql/out.test.toml b/acceptance/bundle/templates/telemetry/dbt-sql/out.test.toml index 92ced1275e4..48cc8d7c1c4 100644 --- a/acceptance/bundle/templates/telemetry/dbt-sql/out.test.toml +++ b/acceptance/bundle/templates/telemetry/dbt-sql/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/templates/telemetry/default-python/out.test.toml b/acceptance/bundle/templates/telemetry/default-python/out.test.toml index 92ced1275e4..48cc8d7c1c4 100644 --- a/acceptance/bundle/templates/telemetry/default-python/out.test.toml +++ b/acceptance/bundle/templates/telemetry/default-python/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/templates/telemetry/default-sql/out.test.toml b/acceptance/bundle/templates/telemetry/default-sql/out.test.toml index 92ced1275e4..48cc8d7c1c4 100644 --- a/acceptance/bundle/templates/telemetry/default-sql/out.test.toml +++ b/acceptance/bundle/templates/telemetry/default-sql/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/templates/test.toml b/acceptance/bundle/templates/test.toml index 977f3725d01..5d7fc7bfa7a 100644 --- a/acceptance/bundle/templates/test.toml +++ b/acceptance/bundle/templates/test.toml @@ -1,5 +1,11 @@ # Local-only: At the moment, there are many differences across different envs w.r.t to catalog use, node type and so on. +# A template test materializes a whole project and deploys it, taking tens of seconds each, +# and some diff against a sibling test's output directory. Running all of that a second time +# for deployment history recording costs minutes and adds no coverage the rest of the suite +# does not already give, so these opt out. +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] + [[Server]] Pattern = "POST /telemetry-ext" Response.Body = ''' diff --git a/acceptance/bundle/test.toml b/acceptance/bundle/test.toml index 2ff57477252..32ccd18e452 100644 --- a/acceptance/bundle/test.toml +++ b/acceptance/bundle/test.toml @@ -11,6 +11,13 @@ EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrixExclude.dms_needs_direct = ["DATABRICKS_BUNDLE_DMS=true", "DATABRICKS_BUNDLE_ENGINE=terraform"] EnvMatrixExclude.dms_local_only = ["DATABRICKS_BUNDLE_DMS=true", "CONFIG_Cloud=true"] +# A saved plan does not carry the deployment stamp. On a first deploy there is no +# deployment to resolve when `bundle plan` runs, so the plan it writes leaves the field +# unset; `deploy --plan` then creates the resources without it and the next plan reports +# drift. Stamping at plan time would mean `bundle plan` creating the deployment record, +# which is a design decision, so the saved-plan path is left out of the DMS run for now. +EnvMatrixExclude.dms_no_readplan = ["DATABRICKS_BUNDLE_DMS=true", "READPLAN=1"] + # Recording is gated off for users (see validate.ValidateRecordDeploymentHistory) and # refuses a bundle whose state already tracks resources - which most tests here seed. # Both are forced on: these tests assert what a deploy does, so the resource duplication diff --git a/acceptance/bundle/user_agent/out.test.toml b/acceptance/bundle/user_agent/out.test.toml index b827ff3f062..40a9dbfa26b 100644 --- a/acceptance/bundle/user_agent/out.test.toml +++ b/acceptance/bundle/user_agent/out.test.toml @@ -1,5 +1,5 @@ Local = true Cloud = false Phase = 1 -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/user_agent/simple/out.test.toml b/acceptance/bundle/user_agent/simple/out.test.toml index 92ced1275e4..48cc8d7c1c4 100644 --- a/acceptance/bundle/user_agent/simple/out.test.toml +++ b/acceptance/bundle/user_agent/simple/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/user_agent/test.toml b/acceptance/bundle/user_agent/test.toml index a9f876b6b77..59b0f354509 100644 --- a/acceptance/bundle/user_agent/test.toml +++ b/acceptance/bundle/user_agent/test.toml @@ -3,5 +3,10 @@ RecordRequests = true Local = true IncludeRequestHeaders = ["User-Agent"] +# This test asserts the User-Agent on every single request the CLI makes, so recording's +# extra calls belong in the golden rather than being filtered out - but they are the same +# header the existing requests already cover, so the DMS run only adds entries to maintain. +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] + [Env] DATABRICKS_CACHE_ENABLED = 'false' From 754242735f09b233c40a89041e2f6374ddfdc219 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Sun, 9 Aug 2026 19:13:10 +0000 Subject: [PATCH 057/125] acceptance: filter the deployment stamp at the call site, not with global Repls Deployment history recording stamps deployment_id and version_id onto every job and pipeline, so a test that asserts JSON has to drop them to produce the same golden either way. That was done with seven regexes repeated across fifteen test.toml files, which applied to every output the subtree produced whether it needed it or not. Each test now says so where it prints: `| nostamp` for JSON output, or `print_requests.py --nostamp` for recorded requests. The four field paths live in one STAMP_FIELDS constant, and the jq lives in one bundle-scoped helper - bundle/script.prepare rather than the root, so no non-bundle test sees it. The filter is narrower than the regexes were: it only touches a block that also carries "kind" and "metadata_file_path", so an unrelated deployment_id survives, and it keeps an empty value, since a terraform state dump has "version_id": "" for a job it never stamped. Net -487 lines. Full bundle suite: 0 failures with recording on. Co-authored-by: Isaac --- acceptance/bin/print_requests.py | 18 ++++ .../local_code_source/output.txt | 4 +- .../ai_runtime_task/local_code_source/script | 6 +- acceptance/bundle/ai_runtime_task/test.toml | 55 ------------- acceptance/bundle/artifacts/test.toml | 56 ------------- .../bundle/artifacts/whl_dynamic/script | 6 +- .../artifacts/whl_implicit_custom_path/script | 2 +- .../artifacts/whl_via_environment_key/script | 2 +- acceptance/bundle/bundle_tag/id/output.txt | 4 +- acceptance/bundle/bundle_tag/id/script | 6 +- acceptance/bundle/bundle_tag/test.toml | 56 ------------- acceptance/bundle/bundle_tag/url/output.txt | 4 +- acceptance/bundle/bundle_tag/url/script | 6 +- .../bundle/deploy/experimental-python/script | 2 +- .../bundle/deploy/python-notebook/script | 2 +- acceptance/bundle/deploy/test.toml | 56 ------------- .../bundle/deploy/wal/chain-3-jobs/script | 2 +- .../deploy/wal/crash-after-create/script | 2 +- acceptance/bundle/deployment/test.toml | 56 ------------- .../bundle/destroy/jobs-and-pipeline/script | 2 +- acceptance/bundle/destroy/test.toml | 55 ------------- acceptance/bundle/empty_string_dropped/script | 6 +- .../bundle/empty_string_dropped/test.toml | 56 ------------- .../bundle/environments/dependencies/script | 8 +- acceptance/bundle/environments/test.toml | 55 ------------- acceptance/bundle/invariant/test.toml | 56 ------------- .../resource_deps/create_error/output.txt | 6 +- .../bundle/resource_deps/create_error/script | 6 +- .../bundle/resource_deps/id_chain/script | 6 +- .../bundle/resource_deps/job_id/output.txt | 6 +- acceptance/bundle/resource_deps/job_id/script | 10 +-- .../job_id_big_graph/delete_all/output.txt | 2 +- .../job_id_big_graph/delete_all/script | 2 +- .../job_id_big_graph/destroy/output.txt | 2 +- .../job_id_big_graph/destroy/script | 2 +- .../job_id_delete_bar/output.txt | 6 +- .../resource_deps/job_id_delete_bar/script | 8 +- .../job_id_delete_foo/output.txt | 6 +- .../resource_deps/job_id_delete_foo/script | 8 +- .../resource_deps/jobs_update/output.txt | 6 +- .../bundle/resource_deps/jobs_update/script | 10 +-- .../jobs_update_remote/output.txt | 4 +- .../resource_deps/jobs_update_remote/script | 8 +- .../missing_string_field/output.txt | 4 +- .../resource_deps/missing_string_field/script | 6 +- .../resource_deps/model_id_ref/output.txt | 2 +- .../bundle/resource_deps/model_id_ref/script | 2 +- .../pipelines_recreate/output.txt | 8 +- .../resource_deps/pipelines_recreate/script | 18 ++-- .../resource_deps/remote_app_url/output.txt | 6 +- .../resource_deps/remote_app_url/script | 6 +- .../resource_deps/remote_pipeline/script | 4 +- acceptance/bundle/resource_deps/test.toml | 56 ------------- .../volume_path_job_ref/output.txt | 2 +- .../resource_deps/volume_path_job_ref/script | 4 +- .../deploy/update-and-resize-autoscale/script | 4 +- .../clusters/deploy/update-and-resize/script | 4 +- .../bundle/resources/jobs/big_id/output.txt | 4 +- .../bundle/resources/jobs/big_id/script | 4 +- .../bundle/resources/jobs/delete_job/script | 2 +- .../resources/jobs/num_workers/output.txt | 2 +- .../bundle/resources/jobs/num_workers/script | 2 +- .../jobs/on_failure_empty_slice/script | 2 +- .../resources/jobs/remote_add_tag/script | 2 +- .../removed_from_config/output.txt | 2 +- .../remote_delete/removed_from_config/script | 2 +- .../jobs/remote_matches_config/output.txt | 2 +- .../jobs/remote_matches_config/script | 4 +- .../resources/jobs/tags_empty_map/script | 2 +- .../bundle/resources/jobs/update/output.txt | 8 +- .../bundle/resources/jobs/update/script | 8 +- .../jobs/update_single_node/output.txt | 6 +- .../resources/jobs/update_single_node/script | 12 +-- .../jobs/webhook-reorder-remote/output.txt | 2 +- .../jobs/webhook-reorder-remote/script | 4 +- .../bundle/resources/permissions/_script | 4 +- .../permissions/jobs/added_remotely/script | 2 +- .../jobs/current_can_manage_run/script | 8 +- .../permissions/jobs/delete_one/script | 8 +- .../jobs/deleted_remotely/output.txt | 6 +- .../permissions/jobs/deleted_remotely/script | 10 +-- .../jobs/other_can_manage_run/script | 4 +- .../permissions/jobs/update/output.txt | 14 ++-- .../resources/permissions/jobs/update/script | 28 +++---- .../permissions/pipelines/update/output.txt | 10 +-- .../permissions/pipelines/update/script | 23 +++--- .../permissions/target_permissions/script | 4 +- .../allow-duplicate-names/output.txt | 2 +- .../pipelines/allow-duplicate-names/script | 2 +- .../resources/pipelines/auto-approve/script | 8 +- .../pipelines/drift/parameters/output.txt | 2 +- .../pipelines/drift/parameters/script | 2 +- .../pipelines/lakeflow-pipeline/script | 2 +- .../num-workers-zero/out.requests.direct.txt | 2 - .../out.requests.terraform.txt | 2 - .../pipelines/num-workers-zero/output.txt | 2 + .../pipelines/num-workers-zero/script | 2 +- .../resources/pipelines/photon-true/script | 2 +- .../resources/pipelines/recreate-keys/_script | 8 +- .../resources/pipelines/recreate/script | 4 +- .../pipelines/remote_matches_config/script | 2 +- .../bundle/resources/pipelines/update/script | 4 +- .../pipelines/zero-value-fields/script | 2 +- .../postgres_branches/update_protected/script | 8 +- .../postgres_databases/update/script | 8 +- .../update_autoscaling/script | 8 +- .../update_display_name/script | 8 +- .../resources/postgres_roles/update/script | 8 +- .../resources/registered_models/basic/script | 2 +- .../resources/schemas/auto-approve/script | 4 +- acceptance/bundle/resources/test.toml | 56 ------------- .../bundle/run_as/job_default/output.txt | 4 +- acceptance/bundle/run_as/job_default/script | 4 +- acceptance/bundle/run_as/pipelines/_script | 4 +- acceptance/bundle/run_as/test.toml | 55 ------------- acceptance/bundle/script.prepare | 18 ++++ acceptance/bundle/select/basic/output.txt | 4 +- acceptance/bundle/select/basic/script | 8 +- acceptance/bundle/select/test.toml | 56 ------------- .../permission_level_migration/output.txt | 82 +++++++++---------- .../state/permission_level_migration/script | 2 +- acceptance/bundle/state/test.toml | 55 ------------- .../bundle/summary/modified_status/script | 8 +- acceptance/bundle/summary/test.toml | 55 ------------- 124 files changed, 352 insertions(+), 1151 deletions(-) delete mode 100644 acceptance/bundle/ai_runtime_task/test.toml delete mode 100644 acceptance/bundle/destroy/test.toml delete mode 100644 acceptance/bundle/environments/test.toml delete mode 100644 acceptance/bundle/run_as/test.toml create mode 100644 acceptance/bundle/script.prepare delete mode 100644 acceptance/bundle/state/test.toml delete mode 100644 acceptance/bundle/summary/test.toml diff --git a/acceptance/bin/print_requests.py b/acceptance/bin/print_requests.py index dbc7d5a6d90..9c8c6a8e7a9 100755 --- a/acceptance/bin/print_requests.py +++ b/acceptance/bin/print_requests.py @@ -90,6 +90,15 @@ ADD_PREFIX = "/" NEGATE_PREFIX = "^/" +# What --nostamp deletes. A job create sends the deployment block at the top level, an +# update nests it under new_settings. +STAMP_FIELDS = [ + "deployment.deployment_id", + "deployment.version_id", + "new_settings.deployment.deployment_id", + "new_settings.deployment.version_id", +] + def read_json_many(s): result = [] @@ -221,10 +230,19 @@ def main(): "--del-body, which edits the parsed JSON body, this drops a field of the request " "record itself, e.g. raw_body for a binary upload payload.", ) + parser.add_argument( + "--nostamp", + action="store_true", + help="Drop the deployment stamp (deployment_id, version_id) from job and pipeline " + "bodies, so a test asserts the same requests whether or not deployment history " + "recording is on. Shorthand for the --del-body fields it implies.", + ) parser.add_argument("--fname", default="out.requests.txt") args = parser.parse_args() del_body_fields = [field for group in args.del_body for field in group.split(",")] + if args.nostamp: + del_body_fields += STAMP_FIELDS del_fields = [field for group in args.del_field for field in group.split(",")] test_tmp_dir = os.environ.get("TEST_TMP_DIR") diff --git a/acceptance/bundle/ai_runtime_task/local_code_source/output.txt b/acceptance/bundle/ai_runtime_task/local_code_source/output.txt index 06cdf841c55..c540ae68caa 100644 --- a/acceptance/bundle/ai_runtime_task/local_code_source/output.txt +++ b/acceptance/bundle/ai_runtime_task/local_code_source/output.txt @@ -21,7 +21,7 @@ src/train.py === both code_source_paths point into the bundle .air_snapshots, command_paths rewritten, deps on environments spec ->>> print_requests.py --sort --del-field raw_body //.air_snapshots/ //jobs/create +>>> print_requests.py --nostamp --sort --del-field raw_body //.air_snapshots/ //jobs/create { "method": "POST", "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/ai-runtime-test/default/files/.air_snapshots/[SNAPSHOT].tar.gz", @@ -114,7 +114,7 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> print_requests.py --sort --del-field raw_body //.air_snapshots/ +>>> print_requests.py --nostamp --sort --del-field raw_body //.air_snapshots/ { "method": "POST", "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/ai-runtime-test/default/files/.air_snapshots/[SNAPSHOT].tar.gz", diff --git a/acceptance/bundle/ai_runtime_task/local_code_source/script b/acceptance/bundle/ai_runtime_task/local_code_source/script index fe5e707ce01..41949cf1c85 100644 --- a/acceptance/bundle/ai_runtime_task/local_code_source/script +++ b/acceptance/bundle/ai_runtime_task/local_code_source/script @@ -13,8 +13,8 @@ trace list_code_snapshot.py title "both code_source_paths point into the bundle .air_snapshots, command_paths rewritten, deps on environments spec\n" # --del-field raw_body drops the binary tarball upload payload (kept readable). Filters # use a leading // so Git Bash on Windows does not path-convert them. --keep is not -# passed, so print_requests.py consumes out.requests.txt. -trace print_requests.py --sort --del-field raw_body '//.air_snapshots/' '//jobs/create' +# passed, so print_requests.py --nostamp consumes out.requests.txt. +trace print_requests.py --nostamp --sort --del-field raw_body '//.air_snapshots/' '//jobs/create' title "re-planning unchanged code is a no-op (no changes)\n" trace $CLI bundle plan @@ -22,7 +22,7 @@ trace $CLI bundle plan title "editing a file changes the snapshot hash (content-addressed name changes)\n" update_file.py src/train.py 'print("training")' 'print("training v2")' trace $CLI bundle deploy -trace print_requests.py --sort --del-field raw_body '//.air_snapshots/' +trace print_requests.py --nostamp --sort --del-field raw_body '//.air_snapshots/' title "destroy removes the deployed bundle (including the synced snapshots)\n" trace $CLI bundle destroy --auto-approve diff --git a/acceptance/bundle/ai_runtime_task/test.toml b/acceptance/bundle/ai_runtime_task/test.toml deleted file mode 100644 index abf7cc15484..00000000000 --- a/acceptance/bundle/ai_runtime_task/test.toml +++ /dev/null @@ -1,55 +0,0 @@ -# These normalize the deployment stamp recording adds to every job and pipeline, so a test -# asserts the same golden files whether or not recording is on - that is the point of the -# DMS run, rather than keeping a second copy of 600-odd output files. They are repeated per -# subtree rather than living in the parent because bundle/dms asserts the stamp itself and -# would inherit them. -# -# The stamp as a change the plan reports on its own. It shows up at whatever depth the -# enclosing object sits at, so the indent is matched loosely; the body lines are matched as -# `"key": value` pairs rather than `.*` so the match stops at the entry's own closing brace -# instead of running into its siblings. (Go's regexp is RE2, so the indent cannot be -# captured and back-referenced.) -[[Repls]] -Old = '(?m)^ *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\},\n' -New = '' - -# Same entry when it is the last one in the object, so the comma is on the line before. -[[Repls]] -Old = '(?m),\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n' -New = "\n" - -# When the stamp is the only entry, the whole "changes" object exists because of recording. -# The trailing-comma form comes first: the rule after it would match the same text and -# leave the comma on the previous line dangling. -[[Repls]] -Old = '(?m),\n *"changes": \{\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n *\}\n' -New = "\n" - -[[Repls]] -Old = '(?m)^ *"changes": \{\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n *\},?\n' -New = '' - -# The stamp inside a serialized deployment block, as printed by `jobs get` / `pipelines get` -# and in state dumps. Both keys always sit alongside "kind" and "metadata_file_path", so -# each pattern anchors on one of those - that keeps it from matching an unrelated field -# named deployment_id elsewhere in the output. -# -# Order puts these after the root's numeric rules (Order = 10), which have by then turned -# the id into [NUMID]. -[[Repls]] -Old = '(?m)^ *"deployment_id": "\[NUMID\]",\n( *"kind": "BUNDLE")' -New = '${1}' -Order = 20 - -[[Repls]] -# The value is required to be non-empty: a terraform state dump carries -# "version_id": "" for a job it never stamped, and that line is not ours to drop. -Old = '(?m)(^ *"metadata_file_path": [^\n]*),\n *"version_id": "[^"\n]+"\n' -New = "$1\n" -Order = 20 - -# Same two keys in gron.py's flattened form, where each is its own line. -[[Repls]] -Old = '(?m)^json.*\.deployment\.(deployment_id|version_id) = [^\n]*\n' -New = '' -Order = 20 diff --git a/acceptance/bundle/artifacts/test.toml b/acceptance/bundle/artifacts/test.toml index a8051bc8ca6..61bf8345e7b 100644 --- a/acceptance/bundle/artifacts/test.toml +++ b/acceptance/bundle/artifacts/test.toml @@ -29,59 +29,3 @@ Response.Body = ''' "spark_version": "13.3.x-scala2.12" } ''' - -# These normalize the deployment stamp recording adds to every job and pipeline, so a test -# asserts the same golden files whether or not recording is on - that is the point of the -# DMS run, rather than keeping a second copy of 600-odd output files. They are repeated per -# subtree rather than living in the parent because bundle/dms asserts the stamp itself and -# would inherit them. -# -# The stamp as a change the plan reports on its own. It shows up at whatever depth the -# enclosing object sits at, so the indent is matched loosely; the body lines are matched as -# `"key": value` pairs rather than `.*` so the match stops at the entry's own closing brace -# instead of running into its siblings. (Go's regexp is RE2, so the indent cannot be -# captured and back-referenced.) -[[Repls]] -Old = '(?m)^ *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\},\n' -New = '' - -# Same entry when it is the last one in the object, so the comma is on the line before. -[[Repls]] -Old = '(?m),\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n' -New = "\n" - -# When the stamp is the only entry, the whole "changes" object exists because of recording. -# The trailing-comma form comes first: the rule after it would match the same text and -# leave the comma on the previous line dangling. -[[Repls]] -Old = '(?m),\n *"changes": \{\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n *\}\n' -New = "\n" - -[[Repls]] -Old = '(?m)^ *"changes": \{\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n *\},?\n' -New = '' - -# The stamp inside a serialized deployment block, as printed by `jobs get` / `pipelines get` -# and in state dumps. Both keys always sit alongside "kind" and "metadata_file_path", so -# each pattern anchors on one of those - that keeps it from matching an unrelated field -# named deployment_id elsewhere in the output. -# -# Order puts these after the root's numeric rules (Order = 10), which have by then turned -# the id into [NUMID]. -[[Repls]] -Old = '(?m)^ *"deployment_id": "\[NUMID\]",\n( *"kind": "BUNDLE")' -New = '${1}' -Order = 20 - -[[Repls]] -# The value is required to be non-empty: a terraform state dump carries -# "version_id": "" for a job it never stamped, and that line is not ours to drop. -Old = '(?m)(^ *"metadata_file_path": [^\n]*),\n *"version_id": "[^"\n]+"\n' -New = "$1\n" -Order = 20 - -# Same two keys in gron.py's flattened form, where each is its own line. -[[Repls]] -Old = '(?m)^json.*\.deployment\.(deployment_id|version_id) = [^\n]*\n' -New = '' -Order = 20 diff --git a/acceptance/bundle/artifacts/whl_dynamic/script b/acceptance/bundle/artifacts/whl_dynamic/script index 068db16caf8..ddb6fcd55ec 100644 --- a/acceptance/bundle/artifacts/whl_dynamic/script +++ b/acceptance/bundle/artifacts/whl_dynamic/script @@ -4,9 +4,9 @@ cp -r $TESTDIR/../whl_explicit/my_test_code . mkdir prebuilt cp -r $TESTDIR/../whl_prebuilt_multiple/dist/lib/other_test_code-0.0.1-py3-none-any.whl prebuilt -trace $CLI bundle validate -o json | jq .artifacts +trace $CLI bundle validate -o json | nostamp | jq .artifacts -$CLI bundle plan -o json > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy title "There are 2 original wheels and 2 patched ones" @@ -25,7 +25,7 @@ rm out.requests.txt title "Updating the local wheel and deploying again\n" touch my_test_code/src/new_module.py -$CLI bundle plan -o json > out.plan_update.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp > out.plan_update.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy title "Verify contents, it should now have new_module.py" diff --git a/acceptance/bundle/artifacts/whl_implicit_custom_path/script b/acceptance/bundle/artifacts/whl_implicit_custom_path/script index fdc0723f594..623849d3bb5 100644 --- a/acceptance/bundle/artifacts/whl_implicit_custom_path/script +++ b/acceptance/bundle/artifacts/whl_implicit_custom_path/script @@ -3,7 +3,7 @@ trace $CLI bundle deploy trace find.py --expect 1 whl title "Expecting 1 wheel in libraries section in /jobs/create" -trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body' out.requests.txt +trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body' out.requests.txt | nostamp title "Expecting 1 wheel to be uploaded" trace jq .path < out.requests.txt | grep import | grep whl | sort diff --git a/acceptance/bundle/artifacts/whl_via_environment_key/script b/acceptance/bundle/artifacts/whl_via_environment_key/script index 3a0dd929c09..f82c5d7eccf 100644 --- a/acceptance/bundle/artifacts/whl_via_environment_key/script +++ b/acceptance/bundle/artifacts/whl_via_environment_key/script @@ -3,7 +3,7 @@ trace $CLI bundle deploy trace find.py --expect 1 whl title "Expecting 1 wheel in environments section in /jobs/create" -trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body' out.requests.txt +trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body' out.requests.txt | nostamp title "Expecting 1 wheel to be uploaded" trace jq .path < out.requests.txt | grep import | grep whl | sort diff --git a/acceptance/bundle/bundle_tag/id/output.txt b/acceptance/bundle/bundle_tag/id/output.txt index acfd8c3d1a1..c405eb26af7 100644 --- a/acceptance/bundle/bundle_tag/id/output.txt +++ b/acceptance/bundle/bundle_tag/id/output.txt @@ -39,7 +39,7 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> print_requests.py //jobs +>>> print_requests.py --nostamp //jobs { "method": "POST", "path": "/api/2.2/jobs/create", @@ -79,7 +79,7 @@ All files and directories at the following location will be deleted: /Workspace/ Deleting files... Destroy complete! ->>> print_requests.py //jobs +>>> print_requests.py --nostamp //jobs { "method": "POST", "path": "/api/2.2/jobs/delete", diff --git a/acceptance/bundle/bundle_tag/id/script b/acceptance/bundle/bundle_tag/id/script index dbc595a277f..759cc89eba1 100644 --- a/acceptance/bundle/bundle_tag/id/script +++ b/acceptance/bundle/bundle_tag/id/script @@ -1,8 +1,8 @@ trace $CLI bundle validate -trace $CLI bundle validate -o json | jq .resources +trace $CLI bundle validate -o json | nostamp | jq .resources trace $CLI bundle plan trace $CLI bundle deploy -trace print_requests.py //jobs +trace print_requests.py --nostamp //jobs trace $CLI bundle summary trace $CLI bundle destroy --auto-approve -trace print_requests.py //jobs +trace print_requests.py --nostamp //jobs diff --git a/acceptance/bundle/bundle_tag/test.toml b/acceptance/bundle/bundle_tag/test.toml index ea76209cc5a..8540f9500e6 100644 --- a/acceptance/bundle/bundle_tag/test.toml +++ b/acceptance/bundle/bundle_tag/test.toml @@ -1,57 +1 @@ Badness = "configs with id and url should be rejected" - -# These normalize the deployment stamp recording adds to every job and pipeline, so a test -# asserts the same golden files whether or not recording is on - that is the point of the -# DMS run, rather than keeping a second copy of 600-odd output files. They are repeated per -# subtree rather than living in the parent because bundle/dms asserts the stamp itself and -# would inherit them. -# -# The stamp as a change the plan reports on its own. It shows up at whatever depth the -# enclosing object sits at, so the indent is matched loosely; the body lines are matched as -# `"key": value` pairs rather than `.*` so the match stops at the entry's own closing brace -# instead of running into its siblings. (Go's regexp is RE2, so the indent cannot be -# captured and back-referenced.) -[[Repls]] -Old = '(?m)^ *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\},\n' -New = '' - -# Same entry when it is the last one in the object, so the comma is on the line before. -[[Repls]] -Old = '(?m),\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n' -New = "\n" - -# When the stamp is the only entry, the whole "changes" object exists because of recording. -# The trailing-comma form comes first: the rule after it would match the same text and -# leave the comma on the previous line dangling. -[[Repls]] -Old = '(?m),\n *"changes": \{\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n *\}\n' -New = "\n" - -[[Repls]] -Old = '(?m)^ *"changes": \{\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n *\},?\n' -New = '' - -# The stamp inside a serialized deployment block, as printed by `jobs get` / `pipelines get` -# and in state dumps. Both keys always sit alongside "kind" and "metadata_file_path", so -# each pattern anchors on one of those - that keeps it from matching an unrelated field -# named deployment_id elsewhere in the output. -# -# Order puts these after the root's numeric rules (Order = 10), which have by then turned -# the id into [NUMID]. -[[Repls]] -Old = '(?m)^ *"deployment_id": "\[NUMID\]",\n( *"kind": "BUNDLE")' -New = '${1}' -Order = 20 - -[[Repls]] -# The value is required to be non-empty: a terraform state dump carries -# "version_id": "" for a job it never stamped, and that line is not ours to drop. -Old = '(?m)(^ *"metadata_file_path": [^\n]*),\n *"version_id": "[^"\n]+"\n' -New = "$1\n" -Order = 20 - -# Same two keys in gron.py's flattened form, where each is its own line. -[[Repls]] -Old = '(?m)^json.*\.deployment\.(deployment_id|version_id) = [^\n]*\n' -New = '' -Order = 20 diff --git a/acceptance/bundle/bundle_tag/url/output.txt b/acceptance/bundle/bundle_tag/url/output.txt index 76d46c68fa7..c3d00fe765f 100644 --- a/acceptance/bundle/bundle_tag/url/output.txt +++ b/acceptance/bundle/bundle_tag/url/output.txt @@ -39,7 +39,7 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> print_requests.py //jobs +>>> print_requests.py --nostamp //jobs { "method": "POST", "path": "/api/2.2/jobs/create", @@ -79,7 +79,7 @@ All files and directories at the following location will be deleted: /Workspace/ Deleting files... Destroy complete! ->>> print_requests.py //jobs +>>> print_requests.py --nostamp //jobs { "method": "POST", "path": "/api/2.2/jobs/delete", diff --git a/acceptance/bundle/bundle_tag/url/script b/acceptance/bundle/bundle_tag/url/script index dbc595a277f..759cc89eba1 100644 --- a/acceptance/bundle/bundle_tag/url/script +++ b/acceptance/bundle/bundle_tag/url/script @@ -1,8 +1,8 @@ trace $CLI bundle validate -trace $CLI bundle validate -o json | jq .resources +trace $CLI bundle validate -o json | nostamp | jq .resources trace $CLI bundle plan trace $CLI bundle deploy -trace print_requests.py //jobs +trace print_requests.py --nostamp //jobs trace $CLI bundle summary trace $CLI bundle destroy --auto-approve -trace print_requests.py //jobs +trace print_requests.py --nostamp //jobs diff --git a/acceptance/bundle/deploy/experimental-python/script b/acceptance/bundle/deploy/experimental-python/script index 3e52e9c2a7a..09b90b28e2d 100644 --- a/acceptance/bundle/deploy/experimental-python/script +++ b/acceptance/bundle/deploy/experimental-python/script @@ -1,3 +1,3 @@ trace uv run --quiet --with $DATABRICKS_BUNDLES_WHEEL -- $CLI bundle deploy -trace $CLI jobs list --output json +trace $CLI jobs list --output json | nostamp diff --git a/acceptance/bundle/deploy/python-notebook/script b/acceptance/bundle/deploy/python-notebook/script index 601ecd36366..21318c58b66 100644 --- a/acceptance/bundle/deploy/python-notebook/script +++ b/acceptance/bundle/deploy/python-notebook/script @@ -1,3 +1,3 @@ trace $CLI bundle deploy -trace $CLI jobs list --output json +trace $CLI jobs list --output json | nostamp diff --git a/acceptance/bundle/deploy/test.toml b/acceptance/bundle/deploy/test.toml index 10c2541173a..84e8a4a1990 100644 --- a/acceptance/bundle/deploy/test.toml +++ b/acceptance/bundle/deploy/test.toml @@ -3,59 +3,3 @@ Ignore = [ '.databricks', '__pycache__', ] - -# These normalize the deployment stamp recording adds to every job and pipeline, so a test -# asserts the same golden files whether or not recording is on - that is the point of the -# DMS run, rather than keeping a second copy of 600-odd output files. They are repeated per -# subtree rather than living in the parent because bundle/dms asserts the stamp itself and -# would inherit them. -# -# The stamp as a change the plan reports on its own. It shows up at whatever depth the -# enclosing object sits at, so the indent is matched loosely; the body lines are matched as -# `"key": value` pairs rather than `.*` so the match stops at the entry's own closing brace -# instead of running into its siblings. (Go's regexp is RE2, so the indent cannot be -# captured and back-referenced.) -[[Repls]] -Old = '(?m)^ *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\},\n' -New = '' - -# Same entry when it is the last one in the object, so the comma is on the line before. -[[Repls]] -Old = '(?m),\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n' -New = "\n" - -# When the stamp is the only entry, the whole "changes" object exists because of recording. -# The trailing-comma form comes first: the rule after it would match the same text and -# leave the comma on the previous line dangling. -[[Repls]] -Old = '(?m),\n *"changes": \{\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n *\}\n' -New = "\n" - -[[Repls]] -Old = '(?m)^ *"changes": \{\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n *\},?\n' -New = '' - -# The stamp inside a serialized deployment block, as printed by `jobs get` / `pipelines get` -# and in state dumps. Both keys always sit alongside "kind" and "metadata_file_path", so -# each pattern anchors on one of those - that keeps it from matching an unrelated field -# named deployment_id elsewhere in the output. -# -# Order puts these after the root's numeric rules (Order = 10), which have by then turned -# the id into [NUMID]. -[[Repls]] -Old = '(?m)^ *"deployment_id": "\[NUMID\]",\n( *"kind": "BUNDLE")' -New = '${1}' -Order = 20 - -[[Repls]] -# The value is required to be non-empty: a terraform state dump carries -# "version_id": "" for a job it never stamped, and that line is not ours to drop. -Old = '(?m)(^ *"metadata_file_path": [^\n]*),\n *"version_id": "[^"\n]+"\n' -New = "$1\n" -Order = 20 - -# Same two keys in gron.py's flattened form, where each is its own line. -[[Repls]] -Old = '(?m)^json.*\.deployment\.(deployment_id|version_id) = [^\n]*\n' -New = '' -Order = 20 diff --git a/acceptance/bundle/deploy/wal/chain-3-jobs/script b/acceptance/bundle/deploy/wal/chain-3-jobs/script index a5afc6f51d5..c41e8e7f995 100644 --- a/acceptance/bundle/deploy/wal/chain-3-jobs/script +++ b/acceptance/bundle/deploy/wal/chain-3-jobs/script @@ -7,7 +7,7 @@ trace errcode $CLI bundle deploy echo "" echo "=== WAL content after crash ===" -jq -S . .databricks/bundle/default/resources.json.wal 2>/dev/null || echo "No WAL file" +jq -S . .databricks/bundle/default/resources.json.wal 2>/dev/null | nostamp || echo "No WAL file" echo "" echo "=== Number of jobs saved in WAL ===" diff --git a/acceptance/bundle/deploy/wal/crash-after-create/script b/acceptance/bundle/deploy/wal/crash-after-create/script index 264d84648d3..5647c83e387 100644 --- a/acceptance/bundle/deploy/wal/crash-after-create/script +++ b/acceptance/bundle/deploy/wal/crash-after-create/script @@ -8,7 +8,7 @@ trace errcode $CLI bundle deploy trace assert_exists.py .databricks/bundle/default/resources.json.wal trace assert_not_exists.py .databricks/bundle/default/resources.json -trace cat .databricks/bundle/default/resources.json.wal | jq +trace cat .databricks/bundle/default/resources.json.wal | jq | nostamp title "Any other command recovers state" $CLI bundle $COMMAND &> LOG.COMMAND.txt diff --git a/acceptance/bundle/deployment/test.toml b/acceptance/bundle/deployment/test.toml index 32ecf0fa454..c7c6f58ed6e 100644 --- a/acceptance/bundle/deployment/test.toml +++ b/acceptance/bundle/deployment/test.toml @@ -1,57 +1 @@ Cloud = true - -# These normalize the deployment stamp recording adds to every job and pipeline, so a test -# asserts the same golden files whether or not recording is on - that is the point of the -# DMS run, rather than keeping a second copy of 600-odd output files. They are repeated per -# subtree rather than living in the parent because bundle/dms asserts the stamp itself and -# would inherit them. -# -# The stamp as a change the plan reports on its own. It shows up at whatever depth the -# enclosing object sits at, so the indent is matched loosely; the body lines are matched as -# `"key": value` pairs rather than `.*` so the match stops at the entry's own closing brace -# instead of running into its siblings. (Go's regexp is RE2, so the indent cannot be -# captured and back-referenced.) -[[Repls]] -Old = '(?m)^ *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\},\n' -New = '' - -# Same entry when it is the last one in the object, so the comma is on the line before. -[[Repls]] -Old = '(?m),\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n' -New = "\n" - -# When the stamp is the only entry, the whole "changes" object exists because of recording. -# The trailing-comma form comes first: the rule after it would match the same text and -# leave the comma on the previous line dangling. -[[Repls]] -Old = '(?m),\n *"changes": \{\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n *\}\n' -New = "\n" - -[[Repls]] -Old = '(?m)^ *"changes": \{\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n *\},?\n' -New = '' - -# The stamp inside a serialized deployment block, as printed by `jobs get` / `pipelines get` -# and in state dumps. Both keys always sit alongside "kind" and "metadata_file_path", so -# each pattern anchors on one of those - that keeps it from matching an unrelated field -# named deployment_id elsewhere in the output. -# -# Order puts these after the root's numeric rules (Order = 10), which have by then turned -# the id into [NUMID]. -[[Repls]] -Old = '(?m)^ *"deployment_id": "\[NUMID\]",\n( *"kind": "BUNDLE")' -New = '${1}' -Order = 20 - -[[Repls]] -# The value is required to be non-empty: a terraform state dump carries -# "version_id": "" for a job it never stamped, and that line is not ours to drop. -Old = '(?m)(^ *"metadata_file_path": [^\n]*),\n *"version_id": "[^"\n]+"\n' -New = "$1\n" -Order = 20 - -# Same two keys in gron.py's flattened form, where each is its own line. -[[Repls]] -Old = '(?m)^json.*\.deployment\.(deployment_id|version_id) = [^\n]*\n' -New = '' -Order = 20 diff --git a/acceptance/bundle/destroy/jobs-and-pipeline/script b/acceptance/bundle/destroy/jobs-and-pipeline/script index 57ee878f12f..b02d93bcecb 100644 --- a/acceptance/bundle/destroy/jobs-and-pipeline/script +++ b/acceptance/bundle/destroy/jobs-and-pipeline/script @@ -45,7 +45,7 @@ trace $CLI workspace get-status "${DEPLOYMENT_PATH}" | jq '{path, object_type}' title "Assert the pipeline is created" PIPELINE_ID=$($CLI bundle summary -o json | jq -r '.resources.pipelines.bar.id') -trace $CLI pipelines get "${PIPELINE_ID}" | jq "{spec}" +trace $CLI pipelines get "${PIPELINE_ID}" | nostamp | jq "{spec}" title "Assert the job is created:\n" JOB_ID=$($CLI bundle summary -o json | jq -r '.resources.jobs.foo.id') diff --git a/acceptance/bundle/destroy/test.toml b/acceptance/bundle/destroy/test.toml deleted file mode 100644 index abf7cc15484..00000000000 --- a/acceptance/bundle/destroy/test.toml +++ /dev/null @@ -1,55 +0,0 @@ -# These normalize the deployment stamp recording adds to every job and pipeline, so a test -# asserts the same golden files whether or not recording is on - that is the point of the -# DMS run, rather than keeping a second copy of 600-odd output files. They are repeated per -# subtree rather than living in the parent because bundle/dms asserts the stamp itself and -# would inherit them. -# -# The stamp as a change the plan reports on its own. It shows up at whatever depth the -# enclosing object sits at, so the indent is matched loosely; the body lines are matched as -# `"key": value` pairs rather than `.*` so the match stops at the entry's own closing brace -# instead of running into its siblings. (Go's regexp is RE2, so the indent cannot be -# captured and back-referenced.) -[[Repls]] -Old = '(?m)^ *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\},\n' -New = '' - -# Same entry when it is the last one in the object, so the comma is on the line before. -[[Repls]] -Old = '(?m),\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n' -New = "\n" - -# When the stamp is the only entry, the whole "changes" object exists because of recording. -# The trailing-comma form comes first: the rule after it would match the same text and -# leave the comma on the previous line dangling. -[[Repls]] -Old = '(?m),\n *"changes": \{\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n *\}\n' -New = "\n" - -[[Repls]] -Old = '(?m)^ *"changes": \{\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n *\},?\n' -New = '' - -# The stamp inside a serialized deployment block, as printed by `jobs get` / `pipelines get` -# and in state dumps. Both keys always sit alongside "kind" and "metadata_file_path", so -# each pattern anchors on one of those - that keeps it from matching an unrelated field -# named deployment_id elsewhere in the output. -# -# Order puts these after the root's numeric rules (Order = 10), which have by then turned -# the id into [NUMID]. -[[Repls]] -Old = '(?m)^ *"deployment_id": "\[NUMID\]",\n( *"kind": "BUNDLE")' -New = '${1}' -Order = 20 - -[[Repls]] -# The value is required to be non-empty: a terraform state dump carries -# "version_id": "" for a job it never stamped, and that line is not ours to drop. -Old = '(?m)(^ *"metadata_file_path": [^\n]*),\n *"version_id": "[^"\n]+"\n' -New = "$1\n" -Order = 20 - -# Same two keys in gron.py's flattened form, where each is its own line. -[[Repls]] -Old = '(?m)^json.*\.deployment\.(deployment_id|version_id) = [^\n]*\n' -New = '' -Order = 20 diff --git a/acceptance/bundle/empty_string_dropped/script b/acceptance/bundle/empty_string_dropped/script index f856c966e20..a03c479c071 100644 --- a/acceptance/bundle/empty_string_dropped/script +++ b/acceptance/bundle/empty_string_dropped/script @@ -8,14 +8,14 @@ # Resolved config the engines see. Today every "" field is still present here; a # fix in the initialize phase would drop them, and this golden would show that. -$CLI bundle validate -o json -t direct | jq .resources > out.validate.json +$CLI bundle validate -o json -t direct | nostamp | jq .resources > out.validate.json # Exclude non-create traffic: workspace file ops, telemetry (nondeterministic), and the # deployment history calls the DMS run adds. trace DATABRICKS_BUNDLE_ENGINE=terraform $CLI bundle deploy -t tf -print_requests.py ^//api/2.0/workspace ^//telemetry ^//api/2.0/bundle --sort > out.requests.terraform.json +print_requests.py --nostamp ^//api/2.0/workspace ^//telemetry ^//api/2.0/bundle --sort > out.requests.terraform.json trace DATABRICKS_BUNDLE_ENGINE=direct $CLI bundle deploy -t direct -print_requests.py ^//api/2.0/workspace ^//telemetry ^//api/2.0/bundle --sort > out.requests.direct.json +print_requests.py --nostamp ^//api/2.0/workspace ^//telemetry ^//api/2.0/bundle --sort > out.requests.direct.json $TESTDIR/empty_sent.py diff --git a/acceptance/bundle/empty_string_dropped/test.toml b/acceptance/bundle/empty_string_dropped/test.toml index ebc74f5b64f..51e7bc13e23 100644 --- a/acceptance/bundle/empty_string_dropped/test.toml +++ b/acceptance/bundle/empty_string_dropped/test.toml @@ -10,59 +10,3 @@ EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] Ignore = [ ".databricks", ] - -# These normalize the deployment stamp recording adds to every job and pipeline, so a test -# asserts the same golden files whether or not recording is on - that is the point of the -# DMS run, rather than keeping a second copy of 600-odd output files. They are repeated per -# subtree rather than living in the parent because bundle/dms asserts the stamp itself and -# would inherit them. -# -# The stamp as a change the plan reports on its own. It shows up at whatever depth the -# enclosing object sits at, so the indent is matched loosely; the body lines are matched as -# `"key": value` pairs rather than `.*` so the match stops at the entry's own closing brace -# instead of running into its siblings. (Go's regexp is RE2, so the indent cannot be -# captured and back-referenced.) -[[Repls]] -Old = '(?m)^ *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\},\n' -New = '' - -# Same entry when it is the last one in the object, so the comma is on the line before. -[[Repls]] -Old = '(?m),\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n' -New = "\n" - -# When the stamp is the only entry, the whole "changes" object exists because of recording. -# The trailing-comma form comes first: the rule after it would match the same text and -# leave the comma on the previous line dangling. -[[Repls]] -Old = '(?m),\n *"changes": \{\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n *\}\n' -New = "\n" - -[[Repls]] -Old = '(?m)^ *"changes": \{\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n *\},?\n' -New = '' - -# The stamp inside a serialized deployment block, as printed by `jobs get` / `pipelines get` -# and in state dumps. Both keys always sit alongside "kind" and "metadata_file_path", so -# each pattern anchors on one of those - that keeps it from matching an unrelated field -# named deployment_id elsewhere in the output. -# -# Order puts these after the root's numeric rules (Order = 10), which have by then turned -# the id into [NUMID]. -[[Repls]] -Old = '(?m)^ *"deployment_id": "\[NUMID\]",\n( *"kind": "BUNDLE")' -New = '${1}' -Order = 20 - -[[Repls]] -# The value is required to be non-empty: a terraform state dump carries -# "version_id": "" for a job it never stamped, and that line is not ours to drop. -Old = '(?m)(^ *"metadata_file_path": [^\n]*),\n *"version_id": "[^"\n]+"\n' -New = "$1\n" -Order = 20 - -# Same two keys in gron.py's flattened form, where each is its own line. -[[Repls]] -Old = '(?m)^json.*\.deployment\.(deployment_id|version_id) = [^\n]*\n' -New = '' -Order = 20 diff --git a/acceptance/bundle/environments/dependencies/script b/acceptance/bundle/environments/dependencies/script index 54d408a8db2..00e5fc846cc 100644 --- a/acceptance/bundle/environments/dependencies/script +++ b/acceptance/bundle/environments/dependencies/script @@ -4,11 +4,11 @@ set -euo pipefail trace $CLI bundle validate trace $CLI bundle deploy -trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body.environments' out.requests.txt +trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body.environments' out.requests.txt | nostamp -trace jq -s '.[] | select(.path=="/api/2.0/pipelines")' out.requests.txt +trace jq -s '.[] | select(.path=="/api/2.0/pipelines")' out.requests.txt | nostamp -trace $CLI bundle validate -o json | jq '.resources.jobs.test_job.environments' -trace $CLI bundle validate -o json | jq '.resources.pipelines.test_pipeline.environment' +trace $CLI bundle validate -o json | nostamp | jq '.resources.jobs.test_job.environments' +trace $CLI bundle validate -o json | nostamp | jq '.resources.pipelines.test_pipeline.environment' rm out.requests.txt diff --git a/acceptance/bundle/environments/test.toml b/acceptance/bundle/environments/test.toml deleted file mode 100644 index abf7cc15484..00000000000 --- a/acceptance/bundle/environments/test.toml +++ /dev/null @@ -1,55 +0,0 @@ -# These normalize the deployment stamp recording adds to every job and pipeline, so a test -# asserts the same golden files whether or not recording is on - that is the point of the -# DMS run, rather than keeping a second copy of 600-odd output files. They are repeated per -# subtree rather than living in the parent because bundle/dms asserts the stamp itself and -# would inherit them. -# -# The stamp as a change the plan reports on its own. It shows up at whatever depth the -# enclosing object sits at, so the indent is matched loosely; the body lines are matched as -# `"key": value` pairs rather than `.*` so the match stops at the entry's own closing brace -# instead of running into its siblings. (Go's regexp is RE2, so the indent cannot be -# captured and back-referenced.) -[[Repls]] -Old = '(?m)^ *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\},\n' -New = '' - -# Same entry when it is the last one in the object, so the comma is on the line before. -[[Repls]] -Old = '(?m),\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n' -New = "\n" - -# When the stamp is the only entry, the whole "changes" object exists because of recording. -# The trailing-comma form comes first: the rule after it would match the same text and -# leave the comma on the previous line dangling. -[[Repls]] -Old = '(?m),\n *"changes": \{\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n *\}\n' -New = "\n" - -[[Repls]] -Old = '(?m)^ *"changes": \{\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n *\},?\n' -New = '' - -# The stamp inside a serialized deployment block, as printed by `jobs get` / `pipelines get` -# and in state dumps. Both keys always sit alongside "kind" and "metadata_file_path", so -# each pattern anchors on one of those - that keeps it from matching an unrelated field -# named deployment_id elsewhere in the output. -# -# Order puts these after the root's numeric rules (Order = 10), which have by then turned -# the id into [NUMID]. -[[Repls]] -Old = '(?m)^ *"deployment_id": "\[NUMID\]",\n( *"kind": "BUNDLE")' -New = '${1}' -Order = 20 - -[[Repls]] -# The value is required to be non-empty: a terraform state dump carries -# "version_id": "" for a job it never stamped, and that line is not ours to drop. -Old = '(?m)(^ *"metadata_file_path": [^\n]*),\n *"version_id": "[^"\n]+"\n' -New = "$1\n" -Order = 20 - -# Same two keys in gron.py's flattened form, where each is its own line. -[[Repls]] -Old = '(?m)^json.*\.deployment\.(deployment_id|version_id) = [^\n]*\n' -New = '' -Order = 20 diff --git a/acceptance/bundle/invariant/test.toml b/acceptance/bundle/invariant/test.toml index d20dc220ff5..1d0d883f6d5 100644 --- a/acceptance/bundle/invariant/test.toml +++ b/acceptance/bundle/invariant/test.toml @@ -125,59 +125,3 @@ Response.Body = '{"status": {"state": "SUCCEEDED"}, "manifest": {"schema": {"col [[Server]] Pattern = "DELETE /api/2.1/unity-catalog/tables/{full_name}" Response.Body = '{"status": "OK"}' - -# These normalize the deployment stamp recording adds to every job and pipeline, so a test -# asserts the same golden files whether or not recording is on - that is the point of the -# DMS run, rather than keeping a second copy of 600-odd output files. They are repeated per -# subtree rather than living in the parent because bundle/dms asserts the stamp itself and -# would inherit them. -# -# The stamp as a change the plan reports on its own. It shows up at whatever depth the -# enclosing object sits at, so the indent is matched loosely; the body lines are matched as -# `"key": value` pairs rather than `.*` so the match stops at the entry's own closing brace -# instead of running into its siblings. (Go's regexp is RE2, so the indent cannot be -# captured and back-referenced.) -[[Repls]] -Old = '(?m)^ *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\},\n' -New = '' - -# Same entry when it is the last one in the object, so the comma is on the line before. -[[Repls]] -Old = '(?m),\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n' -New = "\n" - -# When the stamp is the only entry, the whole "changes" object exists because of recording. -# The trailing-comma form comes first: the rule after it would match the same text and -# leave the comma on the previous line dangling. -[[Repls]] -Old = '(?m),\n *"changes": \{\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n *\}\n' -New = "\n" - -[[Repls]] -Old = '(?m)^ *"changes": \{\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n *\},?\n' -New = '' - -# The stamp inside a serialized deployment block, as printed by `jobs get` / `pipelines get` -# and in state dumps. Both keys always sit alongside "kind" and "metadata_file_path", so -# each pattern anchors on one of those - that keeps it from matching an unrelated field -# named deployment_id elsewhere in the output. -# -# Order puts these after the root's numeric rules (Order = 10), which have by then turned -# the id into [NUMID]. -[[Repls]] -Old = '(?m)^ *"deployment_id": "\[NUMID\]",\n( *"kind": "BUNDLE")' -New = '${1}' -Order = 20 - -[[Repls]] -# The value is required to be non-empty: a terraform state dump carries -# "version_id": "" for a job it never stamped, and that line is not ours to drop. -Old = '(?m)(^ *"metadata_file_path": [^\n]*),\n *"version_id": "[^"\n]+"\n' -New = "$1\n" -Order = 20 - -# Same two keys in gron.py's flattened form, where each is its own line. -[[Repls]] -Old = '(?m)^json.*\.deployment\.(deployment_id|version_id) = [^\n]*\n' -New = '' -Order = 20 diff --git a/acceptance/bundle/resource_deps/create_error/output.txt b/acceptance/bundle/resource_deps/create_error/output.txt index f8f3c819fa0..525ee93f60a 100644 --- a/acceptance/bundle/resource_deps/create_error/output.txt +++ b/acceptance/bundle/resource_deps/create_error/output.txt @@ -7,7 +7,7 @@ create jobs.independent Plan: 4 to add, 0 to change, 0 to delete, 0 unchanged ->>> print_requests.py --sort //jobs +>>> print_requests.py --nostamp --sort //jobs { "method": "POST", "path": "/api/2.2/jobs/create", @@ -75,7 +75,7 @@ create jobs.foo Plan: 3 to add, 0 to change, 0 to delete, 1 unchanged === Expecting no difference in the output between first and second deploy ->>> print_requests.py //jobs +>>> print_requests.py --nostamp //jobs { "method": "POST", "path": "/api/2.2/jobs/create", @@ -104,7 +104,7 @@ All files and directories at the following location will be deleted: /Workspace/ Deleting files... Destroy complete! ->>> print_requests.py //jobs +>>> print_requests.py --nostamp //jobs { "method": "POST", "path": "/api/2.2/jobs/delete", diff --git a/acceptance/bundle/resource_deps/create_error/script b/acceptance/bundle/resource_deps/create_error/script index d475fdaea78..ea426e1876a 100644 --- a/acceptance/bundle/resource_deps/create_error/script +++ b/acceptance/bundle/resource_deps/create_error/script @@ -1,7 +1,7 @@ echo "*" > .gitignore trace $CLI bundle plan 2>&1 musterr $CLI bundle deploy &> out.deploy.$DATABRICKS_BUNDLE_ENGINE.txt -trace print_requests.py --sort //jobs +trace print_requests.py --nostamp --sort //jobs trace $CLI bundle summary @@ -16,7 +16,7 @@ musterr $CLI bundle deploy &> out.deploy2.$DATABRICKS_BUNDLE_ENGINE.txt title "Expecting no difference in the output between first and second deploy" diff.py out.deploy.$DATABRICKS_BUNDLE_ENGINE.txt out.deploy2.$DATABRICKS_BUNDLE_ENGINE.txt rm out.deploy2.$DATABRICKS_BUNDLE_ENGINE.txt -trace print_requests.py //jobs +trace print_requests.py --nostamp //jobs trace $CLI bundle destroy --auto-approve -trace print_requests.py //jobs +trace print_requests.py --nostamp //jobs diff --git a/acceptance/bundle/resource_deps/id_chain/script b/acceptance/bundle/resource_deps/id_chain/script index 244a5973158..bcd103ac2a5 100644 --- a/acceptance/bundle/resource_deps/id_chain/script +++ b/acceptance/bundle/resource_deps/id_chain/script @@ -16,7 +16,7 @@ print_requests_short() { trace $CLI bundle plan trace print_requests -$CLI bundle plan -o json > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json trace print_requests trace $CLI bundle deploy @@ -25,7 +25,7 @@ trace print_requests_short trace update_file.py databricks.yml aa_desc aa_new_desc trace update_file.py databricks.yml prefix new_prefix -$CLI bundle plan -o json > out.plan_update.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp > out.plan_update.$DATABRICKS_BUNDLE_ENGINE.json trace print_requests trace $CLI bundle plan @@ -34,5 +34,5 @@ trace print_requests trace $CLI bundle deploy trace print_requests_short -$CLI bundle plan -o json > out.plan_skip.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp > out.plan_skip.$DATABRICKS_BUNDLE_ENGINE.json trace print_requests diff --git a/acceptance/bundle/resource_deps/job_id/output.txt b/acceptance/bundle/resource_deps/job_id/output.txt index 89f0656f907..96961a93e59 100644 --- a/acceptance/bundle/resource_deps/job_id/output.txt +++ b/acceptance/bundle/resource_deps/job_id/output.txt @@ -13,7 +13,7 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> print_requests.py //jobs +>>> print_requests.py --nostamp //jobs { "method": "POST", "path": "/api/2.2/jobs/create", @@ -65,7 +65,7 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> print_requests.py //jobs +>>> print_requests.py --nostamp //jobs { "method": "POST", "path": "/api/2.2/jobs/delete", @@ -87,4 +87,4 @@ All files and directories at the following location will be deleted: /Workspace/ Deleting files... Destroy complete! ->>> print_requests.py //jobs +>>> print_requests.py --nostamp //jobs diff --git a/acceptance/bundle/resource_deps/job_id/script b/acceptance/bundle/resource_deps/job_id/script index 3b033fe390a..f8d30917d1c 100644 --- a/acceptance/bundle/resource_deps/job_id/script +++ b/acceptance/bundle/resource_deps/job_id/script @@ -1,16 +1,16 @@ trace $CLI bundle plan -trace $CLI bundle plan -o json > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy -trace print_requests.py //jobs +trace print_requests.py --nostamp //jobs foo_id=`read_id.py foo` bar_id=`read_id.py bar` cp empty.yml databricks.yml -trace $CLI bundle plan -o json > out.plan_delete.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp > out.plan_delete.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy -trace print_requests.py //jobs +trace print_requests.py --nostamp //jobs trace $CLI bundle destroy --auto-approve -trace print_requests.py //jobs +trace print_requests.py --nostamp //jobs diff --git a/acceptance/bundle/resource_deps/job_id_big_graph/delete_all/output.txt b/acceptance/bundle/resource_deps/job_id_big_graph/delete_all/output.txt index b6c98142421..3c66857e591 100644 --- a/acceptance/bundle/resource_deps/job_id_big_graph/delete_all/output.txt +++ b/acceptance/bundle/resource_deps/job_id_big_graph/delete_all/output.txt @@ -5,7 +5,7 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> print_requests.py --sort //jobs +>>> print_requests.py --nostamp --sort //jobs { "method": "POST", "path": "/api/2.2/jobs/create", diff --git a/acceptance/bundle/resource_deps/job_id_big_graph/delete_all/script b/acceptance/bundle/resource_deps/job_id_big_graph/delete_all/script index 4ff07b8b69e..ee4b36cd14a 100644 --- a/acceptance/bundle/resource_deps/job_id_big_graph/delete_all/script +++ b/acceptance/bundle/resource_deps/job_id_big_graph/delete_all/script @@ -3,7 +3,7 @@ cp $TESTDIR/../empty.yml . echo "*" > .gitignore trace $CLI bundle deploy -trace print_requests.py --sort //jobs +trace print_requests.py --nostamp --sort //jobs replace_ids.py diff --git a/acceptance/bundle/resource_deps/job_id_big_graph/destroy/output.txt b/acceptance/bundle/resource_deps/job_id_big_graph/destroy/output.txt index 72b7277ec76..cab493417b6 100644 --- a/acceptance/bundle/resource_deps/job_id_big_graph/destroy/output.txt +++ b/acceptance/bundle/resource_deps/job_id_big_graph/destroy/output.txt @@ -5,7 +5,7 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> print_requests.py --sort //jobs +>>> print_requests.py --nostamp --sort //jobs { "method": "POST", "path": "/api/2.2/jobs/create", diff --git a/acceptance/bundle/resource_deps/job_id_big_graph/destroy/script b/acceptance/bundle/resource_deps/job_id_big_graph/destroy/script index 090f2bd806f..1b6e368ec0f 100644 --- a/acceptance/bundle/resource_deps/job_id_big_graph/destroy/script +++ b/acceptance/bundle/resource_deps/job_id_big_graph/destroy/script @@ -2,7 +2,7 @@ cp $TESTDIR/../databricks.yml . echo "*" > .gitignore trace $CLI bundle deploy -trace print_requests.py --sort //jobs +trace print_requests.py --nostamp --sort //jobs replace_ids.py diff --git a/acceptance/bundle/resource_deps/job_id_delete_bar/output.txt b/acceptance/bundle/resource_deps/job_id_delete_bar/output.txt index fdba7e0c427..8c720215c93 100644 --- a/acceptance/bundle/resource_deps/job_id_delete_bar/output.txt +++ b/acceptance/bundle/resource_deps/job_id_delete_bar/output.txt @@ -5,7 +5,7 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> print_requests.py //jobs +>>> print_requests.py --nostamp //jobs { "method": "POST", "path": "/api/2.2/jobs/create", @@ -58,7 +58,7 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> print_requests.py --sort //jobs +>>> print_requests.py --nostamp --sort //jobs >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: @@ -69,7 +69,7 @@ All files and directories at the following location will be deleted: /Workspace/ Deleting files... Destroy complete! ->>> print_requests.py //jobs +>>> print_requests.py --nostamp //jobs { "method": "POST", "path": "/api/2.2/jobs/delete", diff --git a/acceptance/bundle/resource_deps/job_id_delete_bar/script b/acceptance/bundle/resource_deps/job_id_delete_bar/script index 62bdc1e3fe9..03b9a087810 100644 --- a/acceptance/bundle/resource_deps/job_id_delete_bar/script +++ b/acceptance/bundle/resource_deps/job_id_delete_bar/script @@ -1,6 +1,6 @@ echo "*" > .gitignore trace $CLI bundle deploy -trace print_requests.py //jobs +trace print_requests.py --nostamp //jobs foo_id=`read_id.py foo` @@ -8,11 +8,11 @@ bar_id=`read_id.py bar` title "Delete bar, keep foo (foo config updated to not depend on bar)" cp only_foo.yml databricks.yml -trace $CLI bundle plan -o json > out.plan_delete.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp > out.plan_delete.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy # Sort requests because foo's update and bar's delete have no dependency edge # (foo's dependency on bar was removed in only_foo.yml), so execution order is non-deterministic. -trace print_requests.py --sort //jobs > out.deploy2.requests.$DATABRICKS_BUNDLE_ENGINE.json +trace print_requests.py --nostamp --sort //jobs > out.deploy2.requests.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle destroy --auto-approve -trace print_requests.py //jobs +trace print_requests.py --nostamp //jobs diff --git a/acceptance/bundle/resource_deps/job_id_delete_foo/output.txt b/acceptance/bundle/resource_deps/job_id_delete_foo/output.txt index dd604c51486..c021dfe8fc8 100644 --- a/acceptance/bundle/resource_deps/job_id_delete_foo/output.txt +++ b/acceptance/bundle/resource_deps/job_id_delete_foo/output.txt @@ -5,7 +5,7 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> print_requests.py //jobs +>>> print_requests.py --nostamp //jobs { "method": "POST", "path": "/api/2.2/jobs/create", @@ -58,7 +58,7 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> print_requests.py //jobs +>>> print_requests.py --nostamp //jobs { "method": "POST", "path": "/api/2.2/jobs/delete", @@ -76,7 +76,7 @@ All files and directories at the following location will be deleted: /Workspace/ Deleting files... Destroy complete! ->>> print_requests.py //jobs +>>> print_requests.py --nostamp //jobs { "method": "POST", "path": "/api/2.2/jobs/delete", diff --git a/acceptance/bundle/resource_deps/job_id_delete_foo/script b/acceptance/bundle/resource_deps/job_id_delete_foo/script index 89e1d97bd38..f5efbd22609 100644 --- a/acceptance/bundle/resource_deps/job_id_delete_foo/script +++ b/acceptance/bundle/resource_deps/job_id_delete_foo/script @@ -1,6 +1,6 @@ echo "*" > .gitignore trace $CLI bundle deploy -trace print_requests.py //jobs +trace print_requests.py --nostamp //jobs foo_id=`read_id.py foo` @@ -8,9 +8,9 @@ bar_id=`read_id.py bar` title "Delete foo, keep bar" cp only_bar.yml databricks.yml -trace $CLI bundle plan -o json > out.plan_delete.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp > out.plan_delete.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy -trace print_requests.py //jobs +trace print_requests.py --nostamp //jobs trace $CLI bundle destroy --auto-approve -trace print_requests.py //jobs +trace print_requests.py --nostamp //jobs diff --git a/acceptance/bundle/resource_deps/jobs_update/output.txt b/acceptance/bundle/resource_deps/jobs_update/output.txt index e37e9f115c1..ce5e99399e1 100644 --- a/acceptance/bundle/resource_deps/jobs_update/output.txt +++ b/acceptance/bundle/resource_deps/jobs_update/output.txt @@ -11,7 +11,7 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> print_requests.py //jobs +>>> print_requests.py --nostamp //jobs >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged @@ -30,7 +30,7 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> print_requests.py //jobs +>>> print_requests.py --nostamp //jobs >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged @@ -73,7 +73,7 @@ All files and directories at the following location will be deleted: /Workspace/ Deleting files... Destroy complete! ->>> print_requests.py //jobs +>>> print_requests.py --nostamp //jobs { "method": "POST", "path": "/api/2.2/jobs/delete", diff --git a/acceptance/bundle/resource_deps/jobs_update/script b/acceptance/bundle/resource_deps/jobs_update/script index 3cb808b187f..f806f2db846 100644 --- a/acceptance/bundle/resource_deps/jobs_update/script +++ b/acceptance/bundle/resource_deps/jobs_update/script @@ -2,7 +2,7 @@ echo "*" > .gitignore trace $CLI bundle plan trace $CLI bundle deploy -trace print_requests.py //jobs > out.deploy1.requests.json +trace print_requests.py --nostamp //jobs > out.deploy1.requests.json foo_id=`read_id.py foo` @@ -15,18 +15,18 @@ trace update_file.py databricks.yml DAYS HOURS trace $CLI bundle plan trace $CLI bundle deploy # per-engine output file because terraform adds output-only and server-side defaults fields into request and direct does not -trace print_requests.py //jobs > out.deploy2.requests.$DATABRICKS_BUNDLE_ENGINE.json +trace print_requests.py --nostamp //jobs > out.deploy2.requests.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle plan title "Fetch job ID and verify remote state" # Badness: output should not be different per engine; investigate if it's the same on cloud -trace $CLI jobs get $foo_id > out.get_foo.$DATABRICKS_BUNDLE_ENGINE.json -trace $CLI jobs get $bar_id +trace $CLI jobs get $foo_id | nostamp > out.get_foo.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI jobs get $bar_id | nostamp rm out.requests.txt trace $CLI bundle destroy --auto-approve -trace print_requests.py //jobs +trace print_requests.py --nostamp //jobs trace musterr $CLI jobs get $foo_id trace musterr $CLI jobs get $bar_id diff --git a/acceptance/bundle/resource_deps/jobs_update_remote/output.txt b/acceptance/bundle/resource_deps/jobs_update_remote/output.txt index 8cd27e9c352..f51fc29befa 100644 --- a/acceptance/bundle/resource_deps/jobs_update_remote/output.txt +++ b/acceptance/bundle/resource_deps/jobs_update_remote/output.txt @@ -5,7 +5,7 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> print_requests.py //jobs +>>> print_requests.py --nostamp //jobs { "method": "POST", "path": "/api/2.2/jobs/create", @@ -76,7 +76,7 @@ All files and directories at the following location will be deleted: /Workspace/ Deleting files... Destroy complete! ->>> print_requests.py //jobs +>>> print_requests.py --nostamp //jobs { "method": "POST", "path": "/api/2.2/jobs/reset", diff --git a/acceptance/bundle/resource_deps/jobs_update_remote/script b/acceptance/bundle/resource_deps/jobs_update_remote/script index 09572ae04be..d456ed8b023 100644 --- a/acceptance/bundle/resource_deps/jobs_update_remote/script +++ b/acceptance/bundle/resource_deps/jobs_update_remote/script @@ -1,7 +1,7 @@ echo "*" > .gitignore -$CLI bundle plan -o json > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy -trace print_requests.py //jobs +trace print_requests.py --nostamp //jobs trace $CLI bundle plan @@ -13,7 +13,7 @@ bar_id=`read_id.py bar` title "Update trigger.periodic.unit remotely and re-deploy; jobs.bar is unchanged" trace envsubst < job_update.json > tmp.json && mv tmp.json job_update.json trace $CLI jobs reset --json @job_update.json -$CLI bundle plan -o json > out.plan_update.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp > out.plan_update.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle destroy --auto-approve -trace print_requests.py //jobs +trace print_requests.py --nostamp //jobs diff --git a/acceptance/bundle/resource_deps/missing_string_field/output.txt b/acceptance/bundle/resource_deps/missing_string_field/output.txt index 83bf6f7168d..66ff307b069 100644 --- a/acceptance/bundle/resource_deps/missing_string_field/output.txt +++ b/acceptance/bundle/resource_deps/missing_string_field/output.txt @@ -28,7 +28,7 @@ create pipelines.foo Plan: 2 to add, 0 to change, 0 to delete, 0 unchanged ->>> print_requests.py //pipeline +>>> print_requests.py --nostamp //pipeline >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... @@ -36,7 +36,7 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> print_requests.py //pipeline +>>> print_requests.py --nostamp //pipeline { "method": "POST", "path": "/api/2.0/pipelines", diff --git a/acceptance/bundle/resource_deps/missing_string_field/script b/acceptance/bundle/resource_deps/missing_string_field/script index a234d07b080..c526e980444 100644 --- a/acceptance/bundle/resource_deps/missing_string_field/script +++ b/acceptance/bundle/resource_deps/missing_string_field/script @@ -1,6 +1,6 @@ -trace $CLI bundle validate -o json | jq .resources +trace $CLI bundle validate -o json | nostamp | jq .resources errcode $CLI bundle plan -trace print_requests.py //pipeline +trace print_requests.py --nostamp //pipeline trace $CLI bundle deploy -trace print_requests.py //pipeline +trace print_requests.py --nostamp //pipeline diff --git a/acceptance/bundle/resource_deps/model_id_ref/output.txt b/acceptance/bundle/resource_deps/model_id_ref/output.txt index 3a6306babf8..fa8ab3cdc02 100644 --- a/acceptance/bundle/resource_deps/model_id_ref/output.txt +++ b/acceptance/bundle/resource_deps/model_id_ref/output.txt @@ -13,7 +13,7 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> print_requests.py //mlflow/registered-models/create //jobs/create --sort +>>> print_requests.py --nostamp //mlflow/registered-models/create //jobs/create --sort { "method": "POST", "path": "/api/2.0/mlflow/registered-models/create", diff --git a/acceptance/bundle/resource_deps/model_id_ref/script b/acceptance/bundle/resource_deps/model_id_ref/script index 35dfa5beed1..7931ccec74b 100644 --- a/acceptance/bundle/resource_deps/model_id_ref/script +++ b/acceptance/bundle/resource_deps/model_id_ref/script @@ -10,4 +10,4 @@ trace $CLI bundle deploy model_id=$($CLI model-registry get-model my-model | jq -r '.registered_model_databricks.id') add_repl.py "$model_id" MY_MODEL_ID -trace print_requests.py //mlflow/registered-models/create //jobs/create --sort +trace print_requests.py --nostamp //mlflow/registered-models/create //jobs/create --sort diff --git a/acceptance/bundle/resource_deps/pipelines_recreate/output.txt b/acceptance/bundle/resource_deps/pipelines_recreate/output.txt index b7eec30702b..f5d8d159b31 100644 --- a/acceptance/bundle/resource_deps/pipelines_recreate/output.txt +++ b/acceptance/bundle/resource_deps/pipelines_recreate/output.txt @@ -11,7 +11,7 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> print_requests.py //jobs //pipelines +>>> print_requests.py --nostamp //jobs //pipelines >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged @@ -37,7 +37,7 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> print_requests.py //jobs //pipelines +>>> print_requests.py --nostamp //jobs //pipelines === Fetch resource IDs and verify remote state >>> musterr [CLI] pipelines get [FOO_ID] @@ -100,7 +100,7 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> print_requests.py //jobs //pipelines +>>> print_requests.py --nostamp //jobs //pipelines >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: @@ -116,7 +116,7 @@ All files and directories at the following location will be deleted: /Workspace/ Deleting files... Destroy complete! ->>> print_requests.py //jobs //pipelines +>>> print_requests.py --nostamp //jobs //pipelines { "method": "POST", "path": "/api/2.2/jobs/delete", diff --git a/acceptance/bundle/resource_deps/pipelines_recreate/script b/acceptance/bundle/resource_deps/pipelines_recreate/script index 1a06b13bc24..9d31e809254 100644 --- a/acceptance/bundle/resource_deps/pipelines_recreate/script +++ b/acceptance/bundle/resource_deps/pipelines_recreate/script @@ -1,9 +1,9 @@ echo "*" > .gitignore trace $CLI bundle plan -$CLI bundle plan -o json > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy -trace print_requests.py //jobs //pipelines > out.create.requests.json +trace print_requests.py --nostamp //jobs //pipelines > out.create.requests.json foo_id=`read_id.py foo` @@ -14,27 +14,27 @@ trace $CLI bundle plan # empty title "Update storage, triggering recreate for pipeline; this means updating downstream deps" trace update_file.py databricks.yml "storage: dbfs:/my-storage" "storage: dbfs:/my-new-storage" trace $CLI bundle plan -$CLI bundle plan -o json > out.plan_update.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp > out.plan_update.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy --auto-approve -trace print_requests.py //jobs //pipelines > out.update.requests.$DATABRICKS_BUNDLE_ENGINE.json +trace print_requests.py --nostamp //jobs //pipelines > out.update.requests.$DATABRICKS_BUNDLE_ENGINE.json foo_id_2=`read_id.py foo` title "Fetch resource IDs and verify remote state" trace musterr $CLI pipelines get $foo_id -trace $CLI pipelines get $foo_id_2 -trace $CLI jobs get $bar_id | jq 'del(.settings.run_as)' +trace $CLI pipelines get $foo_id_2 | nostamp +trace $CLI jobs get $bar_id | nostamp| jq 'del(.settings.run_as)' rm out.requests.txt title "Follow up plan & deploy do nothing" trace $CLI bundle plan -$CLI bundle plan -o json > out.plan_noop.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp > out.plan_noop.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy -trace print_requests.py //jobs //pipelines +trace print_requests.py --nostamp //jobs //pipelines trace $CLI bundle destroy --auto-approve -trace print_requests.py //jobs //pipelines +trace print_requests.py --nostamp //jobs //pipelines trace musterr $CLI pipelines get $foo_id trace musterr $CLI pipelines get $foo_id_2 diff --git a/acceptance/bundle/resource_deps/remote_app_url/output.txt b/acceptance/bundle/resource_deps/remote_app_url/output.txt index 2f48d384b3f..a1d3314dab1 100644 --- a/acceptance/bundle/resource_deps/remote_app_url/output.txt +++ b/acceptance/bundle/resource_deps/remote_app_url/output.txt @@ -14,7 +14,7 @@ create pipelines.mypipeline Plan: 2 to add, 0 to change, 0 to delete, 0 unchanged ->>> print_requests.py ^//import-file/ ^//api/2.0/bundle +>>> print_requests.py --nostamp ^//import-file/ ^//api/2.0/bundle { "method": "POST", "path": "/api/2.0/workspace/mkdirs", @@ -29,7 +29,7 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> print_requests.py ^//import-file/ ^//api/2.0/bundle +>>> print_requests.py --nostamp ^//import-file/ ^//api/2.0/bundle { "method": "POST", "path": "/api/2.0/workspace/mkdirs", @@ -98,7 +98,7 @@ All files and directories at the following location will be deleted: /Workspace/ Deleting files... Destroy complete! ->>> print_requests.py --sort ^//import-file/ ^//api/2.0/bundle +>>> print_requests.py --nostamp --sort ^//import-file/ ^//api/2.0/bundle { "method": "DELETE", "path": "/api/2.0/apps/myapp" diff --git a/acceptance/bundle/resource_deps/remote_app_url/script b/acceptance/bundle/resource_deps/remote_app_url/script index 2ce8c3e3c5e..6d1e4fd5839 100644 --- a/acceptance/bundle/resource_deps/remote_app_url/script +++ b/acceptance/bundle/resource_deps/remote_app_url/script @@ -1,9 +1,9 @@ trace $CLI bundle validate trace $CLI bundle plan -trace print_requests.py '^//import-file/' '^//api/2.0/bundle' +trace print_requests.py --nostamp '^//import-file/' '^//api/2.0/bundle' trace $CLI bundle deploy -trace print_requests.py '^//import-file/' '^//api/2.0/bundle' +trace print_requests.py --nostamp '^//import-file/' '^//api/2.0/bundle' trace $CLI bundle destroy --auto-approve -trace print_requests.py --sort '^//import-file/' '^//api/2.0/bundle' +trace print_requests.py --nostamp --sort '^//import-file/' '^//api/2.0/bundle' diff --git a/acceptance/bundle/resource_deps/remote_pipeline/script b/acceptance/bundle/resource_deps/remote_pipeline/script index ae44dfd1229..b8683429aac 100644 --- a/acceptance/bundle/resource_deps/remote_pipeline/script +++ b/acceptance/bundle/resource_deps/remote_pipeline/script @@ -3,8 +3,8 @@ print_requests() { rm out.requests.txt } -$CLI bundle plan -o json > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests -$CLI bundle plan -o json > out.plan_skip.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp > out.plan_skip.$DATABRICKS_BUNDLE_ENGINE.json trace print_requests diff --git a/acceptance/bundle/resource_deps/test.toml b/acceptance/bundle/resource_deps/test.toml index 405ffae696a..dc29b70c320 100644 --- a/acceptance/bundle/resource_deps/test.toml +++ b/acceptance/bundle/resource_deps/test.toml @@ -4,59 +4,3 @@ Ignore = [ ".databricks", ".gitignore", ] - -# These normalize the deployment stamp recording adds to every job and pipeline, so a test -# asserts the same golden files whether or not recording is on - that is the point of the -# DMS run, rather than keeping a second copy of 600-odd output files. They are repeated per -# subtree rather than living in the parent because bundle/dms asserts the stamp itself and -# would inherit them. -# -# The stamp as a change the plan reports on its own. It shows up at whatever depth the -# enclosing object sits at, so the indent is matched loosely; the body lines are matched as -# `"key": value` pairs rather than `.*` so the match stops at the entry's own closing brace -# instead of running into its siblings. (Go's regexp is RE2, so the indent cannot be -# captured and back-referenced.) -[[Repls]] -Old = '(?m)^ *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\},\n' -New = '' - -# Same entry when it is the last one in the object, so the comma is on the line before. -[[Repls]] -Old = '(?m),\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n' -New = "\n" - -# When the stamp is the only entry, the whole "changes" object exists because of recording. -# The trailing-comma form comes first: the rule after it would match the same text and -# leave the comma on the previous line dangling. -[[Repls]] -Old = '(?m),\n *"changes": \{\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n *\}\n' -New = "\n" - -[[Repls]] -Old = '(?m)^ *"changes": \{\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n *\},?\n' -New = '' - -# The stamp inside a serialized deployment block, as printed by `jobs get` / `pipelines get` -# and in state dumps. Both keys always sit alongside "kind" and "metadata_file_path", so -# each pattern anchors on one of those - that keeps it from matching an unrelated field -# named deployment_id elsewhere in the output. -# -# Order puts these after the root's numeric rules (Order = 10), which have by then turned -# the id into [NUMID]. -[[Repls]] -Old = '(?m)^ *"deployment_id": "\[NUMID\]",\n( *"kind": "BUNDLE")' -New = '${1}' -Order = 20 - -[[Repls]] -# The value is required to be non-empty: a terraform state dump carries -# "version_id": "" for a job it never stamped, and that line is not ours to drop. -Old = '(?m)(^ *"metadata_file_path": [^\n]*),\n *"version_id": "[^"\n]+"\n' -New = "$1\n" -Order = 20 - -# Same two keys in gron.py's flattened form, where each is its own line. -[[Repls]] -Old = '(?m)^json.*\.deployment\.(deployment_id|version_id) = [^\n]*\n' -New = '' -Order = 20 diff --git a/acceptance/bundle/resource_deps/volume_path_job_ref/output.txt b/acceptance/bundle/resource_deps/volume_path_job_ref/output.txt index 63edb8decb4..b4cbd222f25 100644 --- a/acceptance/bundle/resource_deps/volume_path_job_ref/output.txt +++ b/acceptance/bundle/resource_deps/volume_path_job_ref/output.txt @@ -44,4 +44,4 @@ } } ->>> print_requests.py //jobs +>>> print_requests.py --nostamp //jobs diff --git a/acceptance/bundle/resource_deps/volume_path_job_ref/script b/acceptance/bundle/resource_deps/volume_path_job_ref/script index bcc1cf937cb..6c7250cc564 100644 --- a/acceptance/bundle/resource_deps/volume_path_job_ref/script +++ b/acceptance/bundle/resource_deps/volume_path_job_ref/script @@ -1,5 +1,5 @@ envsubst < databricks.yml.tmpl > databricks.yml -trace $CLI bundle validate -o json | jq '.resources.jobs, .resources.volumes' +trace $CLI bundle validate -o json | nostamp | jq '.resources.jobs, .resources.volumes' # The job's data_path default references the volume's computed volume_path. # Record the JSON plan per engine to confirm the reference is resolved into the @@ -9,4 +9,4 @@ trace errcode $CLI bundle plan -o json &> out.plan.$DATABRICKS_BUNDLE_ENGINE.jso # Deploy and inspect the create-job request: the parameter default sent to # the Jobs API must be the interpolated volume_path. trace errcode $CLI bundle deploy &> out.deploy.$DATABRICKS_BUNDLE_ENGINE.txt -trace print_requests.py //jobs > out.deploy.requests.$DATABRICKS_BUNDLE_ENGINE.json +trace print_requests.py --nostamp //jobs > out.deploy.requests.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/script b/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/script index 16928c71889..eb9645721ec 100755 --- a/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/script +++ b/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/script @@ -7,11 +7,11 @@ cleanup() { trap cleanup EXIT $CLI bundle plan > out.plan_.$DATABRICKS_BUNDLE_ENGINE.txt -$CLI bundle plan -o json | jq 'del(.. | objects | .deployment_id?, .version_id?)' > out.plan_.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp | jq 'del(.. | objects | .deployment_id?, .version_id?)' > out.plan_.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy title "Cluster should exist with num_workers after bundle deployment:\n" -CLUSTER_ID=$($CLI bundle summary -o json | jq -r '.resources.clusters.test_cluster.id') +CLUSTER_ID=$($CLI bundle summary -o json | nostamp | jq -r '.resources.clusters.test_cluster.id') echo "$CLUSTER_ID:CLUSTER_ID" >> ACC_REPLS $CLI clusters get "${CLUSTER_ID}" | jq '{cluster_name,num_workers,autoscale}' diff --git a/acceptance/bundle/resources/clusters/deploy/update-and-resize/script b/acceptance/bundle/resources/clusters/deploy/update-and-resize/script index 06cdfc7caaf..0ef5d80c68b 100644 --- a/acceptance/bundle/resources/clusters/deploy/update-and-resize/script +++ b/acceptance/bundle/resources/clusters/deploy/update-and-resize/script @@ -7,11 +7,11 @@ cleanup() { trap cleanup EXIT $CLI bundle plan > out.plan_.$DATABRICKS_BUNDLE_ENGINE.txt -$CLI bundle plan -o json | jq 'del(.. | objects | .deployment_id?, .version_id?)' > out.plan_.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp | jq 'del(.. | objects | .deployment_id?, .version_id?)' > out.plan_.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy title "Cluster should exist after bundle deployment:\n" -CLUSTER_ID=$($CLI bundle summary -o json | jq -r '.resources.clusters.test_cluster.id') +CLUSTER_ID=$($CLI bundle summary -o json | nostamp | jq -r '.resources.clusters.test_cluster.id') echo "$CLUSTER_ID:CLUSTER_ID" >> ACC_REPLS $CLI clusters get "${CLUSTER_ID}" | jq '{cluster_name,num_workers}' diff --git a/acceptance/bundle/resources/jobs/big_id/output.txt b/acceptance/bundle/resources/jobs/big_id/output.txt index c539d037f79..638996fd058 100644 --- a/acceptance/bundle/resources/jobs/big_id/output.txt +++ b/acceptance/bundle/resources/jobs/big_id/output.txt @@ -7,7 +7,7 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id,new_settings.deployment.deployment_id,new_settings.deployment.version_id +>>> print_requests.py //jobs --nostamp { "method": "POST", "path": "/api/2.2/jobs/create", @@ -70,7 +70,7 @@ All files and directories at the following location will be deleted: /Workspace/ Deleting files... Destroy complete! ->>> print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id,new_settings.deployment.deployment_id,new_settings.deployment.version_id +>>> print_requests.py //jobs --nostamp { "method": "POST", "path": "/api/2.2/jobs/delete", diff --git a/acceptance/bundle/resources/jobs/big_id/script b/acceptance/bundle/resources/jobs/big_id/script index 6f0e1215c6a..b5823022572 100644 --- a/acceptance/bundle/resources/jobs/big_id/script +++ b/acceptance/bundle/resources/jobs/big_id/script @@ -1,8 +1,8 @@ trace $CLI bundle validate -o json | jq .resources > out.validate.json trace $CLI bundle plan -o json > out.plan.$DATABRICKS_BUNDLE_ENGINE.json $CLI bundle deploy $(readplanarg out.plan.direct.json) -trace print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id,new_settings.deployment.deployment_id,new_settings.deployment.version_id +trace print_requests.py //jobs --nostamp print_state.py > out.state.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle plan | contains.py '0 to add, 0 to change, 0 to delete' trace $CLI bundle destroy --auto-approve -trace print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id,new_settings.deployment.deployment_id,new_settings.deployment.version_id +trace print_requests.py //jobs --nostamp diff --git a/acceptance/bundle/resources/jobs/delete_job/script b/acceptance/bundle/resources/jobs/delete_job/script index c3ce3c66802..a0910145023 100644 --- a/acceptance/bundle/resources/jobs/delete_job/script +++ b/acceptance/bundle/resources/jobs/delete_job/script @@ -2,4 +2,4 @@ trace $CLI bundle deploy cp empty.yml databricks.yml $CLI bundle plan -o json | jq 'del(.. | objects | .deployment_id?, .version_id?)' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy -print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id,new_settings.deployment.deployment_id,new_settings.deployment.version_id +print_requests.py //jobs --nostamp diff --git a/acceptance/bundle/resources/jobs/num_workers/output.txt b/acceptance/bundle/resources/jobs/num_workers/output.txt index 702558444e9..7e5725e474f 100644 --- a/acceptance/bundle/resources/jobs/num_workers/output.txt +++ b/acceptance/bundle/resources/jobs/num_workers/output.txt @@ -21,7 +21,7 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id,new_settings.deployment.deployment_id,new_settings.deployment.version_id +>>> print_requests.py //jobs --nostamp { "method": "POST", "path": "/api/2.2/jobs/create", diff --git a/acceptance/bundle/resources/jobs/num_workers/script b/acceptance/bundle/resources/jobs/num_workers/script index 674a820061c..a9f612ccbdb 100644 --- a/acceptance/bundle/resources/jobs/num_workers/script +++ b/acceptance/bundle/resources/jobs/num_workers/script @@ -1,6 +1,6 @@ envsubst < databricks.yml.tmpl > databricks.yml trace $CLI bundle deploy -trace print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id,new_settings.deployment.deployment_id,new_settings.deployment.version_id +trace print_requests.py //jobs --nostamp trace $CLI bundle plan rm out.requests.txt diff --git a/acceptance/bundle/resources/jobs/on_failure_empty_slice/script b/acceptance/bundle/resources/jobs/on_failure_empty_slice/script index e69f9fb5bab..6f0606bc346 100644 --- a/acceptance/bundle/resources/jobs/on_failure_empty_slice/script +++ b/acceptance/bundle/resources/jobs/on_failure_empty_slice/script @@ -1,3 +1,3 @@ trace $CLI bundle plan trace $CLI bundle deploy -trace $CLI bundle plan -o json | jq '.plan."resources.jobs.refresh_usage_logs".changes // .' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp | jq '.plan."resources.jobs.refresh_usage_logs".changes // .' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/jobs/remote_add_tag/script b/acceptance/bundle/resources/jobs/remote_add_tag/script index 37a37b0059f..79de8e89f48 100644 --- a/acceptance/bundle/resources/jobs/remote_add_tag/script +++ b/acceptance/bundle/resources/jobs/remote_add_tag/script @@ -8,4 +8,4 @@ r["tags"]["new_tag"] = "new_value" EOF $CLI bundle plan -$CLI bundle plan -o json | jq 'del(.. | objects | .deployment_id?, .version_id?)' > out.plan_post_update.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp | jq 'del(.. | objects | .deployment_id?, .version_id?)' > out.plan_post_update.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/jobs/remote_delete/removed_from_config/output.txt b/acceptance/bundle/resources/jobs/remote_delete/removed_from_config/output.txt index 45a761af80e..34e214013ab 100644 --- a/acceptance/bundle/resources/jobs/remote_delete/removed_from_config/output.txt +++ b/acceptance/bundle/resources/jobs/remote_delete/removed_from_config/output.txt @@ -21,4 +21,4 @@ Updating deployment state... Deployment complete! === No delete API calls for resources that are already gone remotely ->>> print_requests.py //jobs/delete //pipelines/ --del-body deployment.deployment_id,deployment.version_id,new_settings.deployment.deployment_id,new_settings.deployment.version_id +>>> print_requests.py //jobs/delete //pipelines/ --nostamp diff --git a/acceptance/bundle/resources/jobs/remote_delete/removed_from_config/script b/acceptance/bundle/resources/jobs/remote_delete/removed_from_config/script index 45098c8a690..89525ab713e 100644 --- a/acceptance/bundle/resources/jobs/remote_delete/removed_from_config/script +++ b/acceptance/bundle/resources/jobs/remote_delete/removed_from_config/script @@ -18,4 +18,4 @@ trace $CLI bundle deploy trace $CLI bundle summary &> out.summary.$DATABRICKS_BUNDLE_ENGINE.txt title "No delete API calls for resources that are already gone remotely" -trace print_requests.py //jobs/delete //pipelines/ --del-body deployment.deployment_id,deployment.version_id,new_settings.deployment.deployment_id,new_settings.deployment.version_id +trace print_requests.py //jobs/delete //pipelines/ --nostamp diff --git a/acceptance/bundle/resources/jobs/remote_matches_config/output.txt b/acceptance/bundle/resources/jobs/remote_matches_config/output.txt index d1f2a5cf7d5..6cfc9158e2c 100644 --- a/acceptance/bundle/resources/jobs/remote_matches_config/output.txt +++ b/acceptance/bundle/resources/jobs/remote_matches_config/output.txt @@ -23,4 +23,4 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id,new_settings.deployment.deployment_id,new_settings.deployment.version_id +>>> print_requests.py //jobs --nostamp diff --git a/acceptance/bundle/resources/jobs/remote_matches_config/script b/acceptance/bundle/resources/jobs/remote_matches_config/script index b528539bf92..1402dc86189 100755 --- a/acceptance/bundle/resources/jobs/remote_matches_config/script +++ b/acceptance/bundle/resources/jobs/remote_matches_config/script @@ -13,9 +13,9 @@ r["max_concurrent_runs"] = 2 EOF trace $CLI bundle plan -$CLI bundle plan -o json | jq 'del(.. | objects | .deployment_id?, .version_id?)' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp | jq 'del(.. | objects | .deployment_id?, .version_id?)' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json rm out.requests.txt # XXX READPLAN trace $CLI bundle deploy -trace print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id,new_settings.deployment.deployment_id,new_settings.deployment.version_id +trace print_requests.py //jobs --nostamp diff --git a/acceptance/bundle/resources/jobs/tags_empty_map/script b/acceptance/bundle/resources/jobs/tags_empty_map/script index 61ebb0a9139..4a232723bd0 100644 --- a/acceptance/bundle/resources/jobs/tags_empty_map/script +++ b/acceptance/bundle/resources/jobs/tags_empty_map/script @@ -1,3 +1,3 @@ trace $CLI bundle plan trace $CLI bundle deploy -trace $CLI bundle plan -o json | jq '.plan."resources.jobs.test_job".changes // .' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp | jq '.plan."resources.jobs.test_job".changes // .' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/jobs/update/output.txt b/acceptance/bundle/resources/jobs/update/output.txt index f6a44af17c6..b448d92768c 100644 --- a/acceptance/bundle/resources/jobs/update/output.txt +++ b/acceptance/bundle/resources/jobs/update/output.txt @@ -8,7 +8,7 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id,new_settings.deployment.deployment_id,new_settings.deployment.version_id +>>> print_requests.py //jobs --nostamp >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged @@ -19,7 +19,7 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id,new_settings.deployment.deployment_id,new_settings.deployment.version_id +>>> print_requests.py //jobs --nostamp === Update trigger.periodic.unit and re-deploy >>> update_file.py databricks.yml DAYS HOURS @@ -33,7 +33,7 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id,new_settings.deployment.deployment_id,new_settings.deployment.version_id +>>> print_requests.py //jobs --nostamp >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged @@ -89,7 +89,7 @@ All files and directories at the following location will be deleted: /Workspace/ Deleting files... Destroy complete! ->>> print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id,new_settings.deployment.deployment_id,new_settings.deployment.version_id +>>> print_requests.py //jobs --nostamp { "method": "POST", "path": "/api/2.2/jobs/delete", diff --git a/acceptance/bundle/resources/jobs/update/script b/acceptance/bundle/resources/jobs/update/script index c15b425741c..1be869aa509 100644 --- a/acceptance/bundle/resources/jobs/update/script +++ b/acceptance/bundle/resources/jobs/update/script @@ -2,14 +2,14 @@ echo "*" > .gitignore trace $CLI bundle plan $CLI bundle plan -o json > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json $CLI bundle deploy $(readplanarg out.plan_create.direct.json) -trace print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id,new_settings.deployment.deployment_id,new_settings.deployment.version_id > out.create.requests.json +trace print_requests.py //jobs --nostamp > out.create.requests.json print_state.py > out.state.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle plan trace $CLI bundle plan -o json > out.plan_skip.$DATABRICKS_BUNDLE_ENGINE.json $CLI bundle deploy $(readplanarg out.plan_skip.direct.json) -trace print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id,new_settings.deployment.deployment_id,new_settings.deployment.version_id +trace print_requests.py //jobs --nostamp title "Update trigger.periodic.unit and re-deploy" trace update_file.py databricks.yml DAYS HOURS @@ -17,7 +17,7 @@ trace $CLI bundle plan $CLI bundle plan -o json > out.plan_update.$DATABRICKS_BUNDLE_ENGINE.json $CLI bundle deploy $(readplanarg out.plan_update.direct.json) -trace print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id,new_settings.deployment.deployment_id,new_settings.deployment.version_id | jq 'del(.body.new_settings.run_as, .body.new_settings.webhook_notifications, .body.new_settings.email_notifications)' > out.update.requests.json +trace print_requests.py //jobs --nostamp | jq 'del(.body.new_settings.run_as, .body.new_settings.webhook_notifications, .body.new_settings.email_notifications)' > out.update.requests.json trace $CLI bundle plan @@ -30,7 +30,7 @@ rm out.requests.txt title "Destroy the job and verify that it's removed from the state and from remote" trace $CLI bundle destroy --auto-approve -trace print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id,new_settings.deployment.deployment_id,new_settings.deployment.version_id +trace print_requests.py //jobs --nostamp trace musterr $CLI jobs get $ppid rm out.requests.txt diff --git a/acceptance/bundle/resources/jobs/update_single_node/output.txt b/acceptance/bundle/resources/jobs/update_single_node/output.txt index ec195ebc965..61bba9a6808 100644 --- a/acceptance/bundle/resources/jobs/update_single_node/output.txt +++ b/acceptance/bundle/resources/jobs/update_single_node/output.txt @@ -10,7 +10,7 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id,new_settings.deployment.deployment_id,new_settings.deployment.version_id +>>> print_requests.py //jobs --nostamp === Update trigger.periodic.unit and re-deploy >>> update_file.py databricks.yml DAYS HOURS @@ -26,7 +26,7 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id,new_settings.deployment.deployment_id,new_settings.deployment.version_id +>>> print_requests.py //jobs --nostamp >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged @@ -89,7 +89,7 @@ All files and directories at the following location will be deleted: /Workspace/ Deleting files... Destroy complete! ->>> print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id,new_settings.deployment.deployment_id,new_settings.deployment.version_id +>>> print_requests.py //jobs --nostamp { "method": "POST", "path": "/api/2.2/jobs/delete", diff --git a/acceptance/bundle/resources/jobs/update_single_node/script b/acceptance/bundle/resources/jobs/update_single_node/script index d822881fe20..ada0b239589 100644 --- a/acceptance/bundle/resources/jobs/update_single_node/script +++ b/acceptance/bundle/resources/jobs/update_single_node/script @@ -1,17 +1,17 @@ echo "*" > .gitignore trace $CLI bundle plan -$CLI bundle plan -o json | jq 'del(.. | objects | .deployment_id?, .version_id?)' > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp | jq 'del(.. | objects | .deployment_id?, .version_id?)' > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy -$CLI bundle plan -o json | jq 'del(.. | objects | .deployment_id?, .version_id?)' > out.plan_skip.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp | jq 'del(.. | objects | .deployment_id?, .version_id?)' > out.plan_skip.$DATABRICKS_BUNDLE_ENGINE.json -trace print_requests.py //jobs > out.create.requests.txt --del-body deployment.deployment_id,deployment.version_id,new_settings.deployment.deployment_id,new_settings.deployment.version_id +trace print_requests.py //jobs > out.create.requests.txt --nostamp title "Update trigger.periodic.unit and re-deploy" trace update_file.py databricks.yml DAYS HOURS trace $CLI bundle plan -$CLI bundle plan -o json | jq 'del(.. | objects | .deployment_id?, .version_id?)' > out.plan_update.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp | jq 'del(.. | objects | .deployment_id?, .version_id?)' > out.plan_update.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy -trace print_requests.py //jobs > out.update.requests.$DATABRICKS_BUNDLE_ENGINE.json --del-body deployment.deployment_id,deployment.version_id,new_settings.deployment.deployment_id,new_settings.deployment.version_id +trace print_requests.py //jobs > out.update.requests.$DATABRICKS_BUNDLE_ENGINE.json --nostamp trace $CLI bundle plan @@ -24,7 +24,7 @@ rm out.requests.txt title "Destroy the job and verify that it's removed from the state and from remote" trace $CLI bundle destroy --auto-approve -trace print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id,new_settings.deployment.deployment_id,new_settings.deployment.version_id +trace print_requests.py //jobs --nostamp trace musterr $CLI jobs get $ppid rm out.requests.txt diff --git a/acceptance/bundle/resources/jobs/webhook-reorder-remote/output.txt b/acceptance/bundle/resources/jobs/webhook-reorder-remote/output.txt index 2185aedcef6..0f1e0ba0a72 100644 --- a/acceptance/bundle/resources/jobs/webhook-reorder-remote/output.txt +++ b/acceptance/bundle/resources/jobs/webhook-reorder-remote/output.txt @@ -16,7 +16,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged ->>> print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id,new_settings.deployment.deployment_id,new_settings.deployment.version_id +>>> print_requests.py //jobs --nostamp { "method": "POST", "path": "/api/2.2/jobs/create", diff --git a/acceptance/bundle/resources/jobs/webhook-reorder-remote/script b/acceptance/bundle/resources/jobs/webhook-reorder-remote/script index 7d9103aca52..e355b877ec3 100644 --- a/acceptance/bundle/resources/jobs/webhook-reorder-remote/script +++ b/acceptance/bundle/resources/jobs/webhook-reorder-remote/script @@ -14,6 +14,6 @@ EOF # The reordered remote must not produce a phantom diff: on_* lists are diffed by id. trace $CLI bundle plan -$CLI bundle plan -o json | jq '.plan."resources.jobs.my_job".changes | del(.. | objects | .deployment_id?, .version_id?)' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp | jq '.plan."resources.jobs.my_job".changes | del(.. | objects | .deployment_id?, .version_id?)' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json -trace print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id,new_settings.deployment.deployment_id,new_settings.deployment.version_id +trace print_requests.py //jobs --nostamp diff --git a/acceptance/bundle/resources/permissions/_script b/acceptance/bundle/resources/permissions/_script index 90e6596c9a3..1ba0dd3bf8a 100644 --- a/acceptance/bundle/resources/permissions/_script +++ b/acceptance/bundle/resources/permissions/_script @@ -1,7 +1,7 @@ -trace $CLI bundle validate -o json | jq .resources.$RESOURCE.foo.permissions +trace $CLI bundle validate -o json | nostamp | jq .resources.$RESOURCE.foo.permissions rm out.requests.txt -$CLI bundle plan -o json > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp > out.plan.$DATABRICKS_BUNDLE_ENGINE.json print_requests() { jq -c < out.requests.txt | jq 'select(.method != "GET" and (.path | contains("permissions")))' diff --git a/acceptance/bundle/resources/permissions/jobs/added_remotely/script b/acceptance/bundle/resources/permissions/jobs/added_remotely/script index 6e8f759cbb8..e85bcde5379 100644 --- a/acceptance/bundle/resources/permissions/jobs/added_remotely/script +++ b/acceptance/bundle/resources/permissions/jobs/added_remotely/script @@ -10,7 +10,7 @@ title "Add permissions out of band" trace $CLI permissions set jobs "$job_id" --json @remote_add.json trace $CLI bundle plan -trace $CLI bundle plan -o json > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp > out.plan.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace $CLI bundle plan diff --git a/acceptance/bundle/resources/permissions/jobs/current_can_manage_run/script b/acceptance/bundle/resources/permissions/jobs/current_can_manage_run/script index 3cfdd63e806..3c8217c195c 100644 --- a/acceptance/bundle/resources/permissions/jobs/current_can_manage_run/script +++ b/acceptance/bundle/resources/permissions/jobs/current_can_manage_run/script @@ -1,11 +1,11 @@ envsubst < databricks.yml.tmpl > databricks.yml -trace $CLI bundle validate -t green -o json | jq .resources +trace $CLI bundle validate -t green -o json | nostamp | jq .resources trace errcode $CLI bundle deploy -t green -print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id,new_settings.deployment.deployment_id,new_settings.deployment.version_id &> out.deploy.requests.json +print_requests.py //jobs --nostamp &> out.deploy.requests.json # check plan to ensure there is not drift -trace $CLI bundle plan -o json -t green > out.plan.$DATABRICKS_BUNDLE_ENGINE.txt +trace $CLI bundle plan -o json -t green | nostamp > out.plan.$DATABRICKS_BUNDLE_ENGINE.txt trace errcode $CLI bundle destroy -t green --auto-approve -print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id,new_settings.deployment.deployment_id,new_settings.deployment.version_id &> out.destroy.requests.$DATABRICKS_BUNDLE_ENGINE.json +print_requests.py //jobs --nostamp &> out.destroy.requests.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/permissions/jobs/delete_one/script b/acceptance/bundle/resources/permissions/jobs/delete_one/script index b51fed3b894..1f0aa1c80a7 100644 --- a/acceptance/bundle/resources/permissions/jobs/delete_one/script +++ b/acceptance/bundle/resources/permissions/jobs/delete_one/script @@ -8,16 +8,16 @@ if [ -n "$CLOUD_ENV" ]; then fi rm -f out.requests.txt -trace $CLI bundle plan -o json > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json cleanup() { trace errcode $CLI bundle destroy --auto-approve - print_requests.py //jobs/ > out.requests_destroy.$DATABRICKS_BUNDLE_ENGINE.json + print_requests.py --nostamp //jobs/ > out.requests_destroy.$DATABRICKS_BUNDLE_ENGINE.json } trap cleanup EXIT trace $CLI bundle deploy -print_requests.py //jobs/ | gron.py --sort-arrays access_control_list > out.requests_create.txt +print_requests.py --nostamp //jobs/ | gron.py --sort-arrays access_control_list > out.requests_create.txt trace $CLI bundle plan @@ -29,7 +29,7 @@ rm -f out.requests.txt title "Delete one permission and deploy again\n" grep -v DELETE databricks.yml > tmp.yml && mv tmp.yml databricks.yml trace $CLI bundle deploy -print_requests.py //jobs/ | gron.py --sort-arrays access_control_list > out.requests_update.txt +print_requests.py --nostamp //jobs/ | gron.py --sort-arrays access_control_list > out.requests_update.txt $CLI permissions get jobs "$job_id" | gron.py --sort-arrays access_control_list | grep -v display_name > out.permissions_update.txt rm -f out.requests.txt diff --git a/acceptance/bundle/resources/permissions/jobs/deleted_remotely/output.txt b/acceptance/bundle/resources/permissions/jobs/deleted_remotely/output.txt index a9d817fec27..740aa92fd19 100644 --- a/acceptance/bundle/resources/permissions/jobs/deleted_remotely/output.txt +++ b/acceptance/bundle/resources/permissions/jobs/deleted_remotely/output.txt @@ -7,7 +7,7 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> print_requests.py //jobs/ +>>> print_requests.py --nostamp //jobs/ >>> [CLI] permissions get jobs [JOB_WITH_PERMISSIONS_ID] { @@ -89,7 +89,7 @@ Deployment complete! "object_type": "job" } ->>> print_requests.py //jobs/ +>>> print_requests.py --nostamp //jobs/ { "method": "PUT", "path": "/api/2.0/permissions/jobs/[JOB_WITH_PERMISSIONS_ID]", @@ -111,7 +111,7 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> print_requests.py //jobs/ +>>> print_requests.py --nostamp //jobs/ >>> [CLI] permissions get jobs [JOB_WITH_PERMISSIONS_ID] { diff --git a/acceptance/bundle/resources/permissions/jobs/deleted_remotely/script b/acceptance/bundle/resources/permissions/jobs/deleted_remotely/script index 28369ea37cb..cce4d71d0ce 100644 --- a/acceptance/bundle/resources/permissions/jobs/deleted_remotely/script +++ b/acceptance/bundle/resources/permissions/jobs/deleted_remotely/script @@ -1,6 +1,6 @@ -trace $CLI bundle plan -o json > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy -trace print_requests.py //jobs/ > out.requests_create.json +trace print_requests.py --nostamp //jobs/ > out.requests_create.json job_id="$(read_id.py job_with_permissions)" @@ -9,11 +9,11 @@ rm -f out.requests.txt title "Delete permissions remotely" trace $CLI permissions set jobs "$job_id" --json @remote_delete.json -trace print_requests.py //jobs/ +trace print_requests.py --nostamp //jobs/ -trace $CLI bundle plan -o json > out.plan_restore.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp > out.plan_restore.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy -trace print_requests.py //jobs/ > out.requests_restore.json +trace print_requests.py --nostamp //jobs/ > out.requests_restore.json trace $CLI permissions get jobs "$job_id" rm -f out.requests.txt diff --git a/acceptance/bundle/resources/permissions/jobs/other_can_manage_run/script b/acceptance/bundle/resources/permissions/jobs/other_can_manage_run/script index 6d9f9c0453f..3e8d8d7c69c 100644 --- a/acceptance/bundle/resources/permissions/jobs/other_can_manage_run/script +++ b/acceptance/bundle/resources/permissions/jobs/other_can_manage_run/script @@ -2,7 +2,7 @@ envsubst < databricks.yml.tmpl > databricks.yml trace $CLI bundle validate -t green -o json | jq .resources trace errcode $CLI bundle deploy -t green -print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id,new_settings.deployment.deployment_id,new_settings.deployment.version_id &> out.deploy.requests.json +print_requests.py //jobs --nostamp &> out.deploy.requests.json trace errcode $CLI bundle destroy -t green --auto-approve -print_requests.py //jobs --del-body deployment.deployment_id,deployment.version_id,new_settings.deployment.deployment_id,new_settings.deployment.version_id &> out.destroy.requests.$DATABRICKS_BUNDLE_ENGINE.json +print_requests.py //jobs --nostamp &> out.destroy.requests.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/permissions/jobs/update/output.txt b/acceptance/bundle/resources/permissions/jobs/update/output.txt index 24f9debf98c..bdd92a681d2 100644 --- a/acceptance/bundle/resources/permissions/jobs/update/output.txt +++ b/acceptance/bundle/resources/permissions/jobs/update/output.txt @@ -13,7 +13,7 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> print_requests.py //jobs/ +>>> print_requests.py --nostamp //jobs/ >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged @@ -79,7 +79,7 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> print_requests.py //jobs/ +>>> print_requests.py --nostamp //jobs/ >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged @@ -94,7 +94,7 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> print_requests.py //jobs/ +>>> print_requests.py --nostamp //jobs/ >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged @@ -109,7 +109,7 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> print_requests.py //jobs/ +>>> print_requests.py --nostamp //jobs/ >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged @@ -137,7 +137,7 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> print_requests.py //jobs/ +>>> print_requests.py --nostamp //jobs/ >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged @@ -152,7 +152,7 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> print_requests.py //jobs/ +>>> print_requests.py --nostamp //jobs/ >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: @@ -163,4 +163,4 @@ All files and directories at the following location will be deleted: /Workspace/ Deleting files... Destroy complete! ->>> print_requests.py //jobs/ +>>> print_requests.py --nostamp //jobs/ diff --git a/acceptance/bundle/resources/permissions/jobs/update/script b/acceptance/bundle/resources/permissions/jobs/update/script index 38ac1a5b62e..28f2c121f2b 100644 --- a/acceptance/bundle/resources/permissions/jobs/update/script +++ b/acceptance/bundle/resources/permissions/jobs/update/script @@ -1,10 +1,10 @@ cp databricks.yml databricks.yml.saved trace $CLI bundle plan -trace $CLI bundle plan -o json > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy -trace print_requests.py //jobs/ > out.requests_create.json +trace print_requests.py --nostamp //jobs/ > out.requests_create.json trace $CLI bundle plan -trace $CLI bundle plan -o json > out.plan_post_create.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp > out.plan_post_create.$DATABRICKS_BUNDLE_ENGINE.json job_id="$(read_id.py job_with_permissions)" @@ -13,41 +13,41 @@ rm -f out.requests.txt title "Update one permission and deploy again\n" update_file.py databricks.yml CAN_VIEW CAN_MANAGE -trace $CLI bundle plan -o json > out.plan_update.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp > out.plan_update.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy -trace print_requests.py //jobs/ > out.requests_update.json +trace print_requests.py --nostamp //jobs/ > out.requests_update.json trace $CLI bundle plan title "Delete one permission and deploy again\n" grep -v DELETE_ONE databricks.yml > tmp.yml && mv tmp.yml databricks.yml -trace $CLI bundle plan -o json > out.plan_delete_one.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp > out.plan_delete_one.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy -trace print_requests.py //jobs/ > out.requests_delete_one.json +trace print_requests.py --nostamp //jobs/ > out.requests_delete_one.json trace $CLI bundle plan title "Delete the whole block and deploy again\n" grep -v PERMISSIONS databricks.yml > tmp.yml && mv tmp.yml databricks.yml -trace $CLI bundle plan -o json > out.plan_delete_all.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp > out.plan_delete_all.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy -trace print_requests.py //jobs/ > out.requests_delete_all.$DATABRICKS_BUNDLE_ENGINE.json +trace print_requests.py --nostamp //jobs/ > out.requests_delete_all.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle plan title "Restore original config\n" trace diff.py databricks.yml.saved databricks.yml mv databricks.yml.saved databricks.yml -trace $CLI bundle plan -o json > out.plan_restore.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp > out.plan_restore.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy -trace print_requests.py //jobs/ > out.requests_restore_original.json +trace print_requests.py --nostamp //jobs/ > out.requests_restore_original.json trace $CLI bundle plan title "Set permissions: []\n" grep -vE '(PERMISSIONS|DELETE_ONE)' databricks.yml > tmp.yml && mv tmp.yml databricks.yml update_file.py databricks.yml '# permissions: [] # EXPLICIT_EMPTY' 'permissions: [] # EXPLICIT_EMPTY' -trace $CLI bundle plan -o json > out.plan_set_empty.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp > out.plan_set_empty.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy -trace print_requests.py //jobs/ > out.requests_set_empty.$DATABRICKS_BUNDLE_ENGINE.json +trace print_requests.py --nostamp //jobs/ > out.requests_set_empty.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle destroy --auto-approve -trace print_requests.py //jobs/ > out.requests_destroy.$DATABRICKS_BUNDLE_ENGINE.json +trace print_requests.py --nostamp //jobs/ > out.requests_destroy.$DATABRICKS_BUNDLE_ENGINE.json #rm -fr .databricks diff --git a/acceptance/bundle/resources/permissions/pipelines/update/output.txt b/acceptance/bundle/resources/permissions/pipelines/update/output.txt index e5395eeceea..6394c59b40b 100644 --- a/acceptance/bundle/resources/permissions/pipelines/update/output.txt +++ b/acceptance/bundle/resources/permissions/pipelines/update/output.txt @@ -7,7 +7,7 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> print_requests.py //pipeline +>>> print_requests.py --nostamp //pipeline >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged @@ -59,7 +59,7 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> print_requests.py //pipeline +>>> print_requests.py --nostamp //pipeline >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged @@ -74,7 +74,7 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> print_requests.py //pipeline +>>> print_requests.py --nostamp //pipeline >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged @@ -98,7 +98,7 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> print_requests.py //pipeline --sort +>>> print_requests.py --nostamp //pipeline --sort >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged @@ -113,7 +113,7 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> print_requests.py //pipeline +>>> print_requests.py --nostamp //pipeline >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged diff --git a/acceptance/bundle/resources/permissions/pipelines/update/script b/acceptance/bundle/resources/permissions/pipelines/update/script index 3fc2edd239c..b3e9d1ca539 100644 --- a/acceptance/bundle/resources/permissions/pipelines/update/script +++ b/acceptance/bundle/resources/permissions/pipelines/update/script @@ -1,7 +1,7 @@ cp databricks.yml databricks.yml.saved -trace $CLI bundle plan -o json > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy -trace print_requests.py //pipeline > out.requests_create.json +trace print_requests.py --nostamp //pipeline > out.requests_create.json trace $CLI bundle plan pipeline_id="$(read_id.py foo)" @@ -11,32 +11,33 @@ rm -f out.requests.txt title "Update one permission and deploy again\n" update_file.py databricks.yml CAN_VIEW CAN_MANAGE -trace $CLI bundle plan -o json > out.plan_update.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp > out.plan_update.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy -trace print_requests.py //pipeline > out.requests_update.json +trace print_requests.py --nostamp //pipeline > out.requests_update.json trace $CLI bundle plan title "Delete one permission and deploy again\n" grep -v DELETE_ONE databricks.yml > tmp.yml && mv tmp.yml databricks.yml -trace $CLI bundle plan -o json > out.plan_delete_one.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp > out.plan_delete_one.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy -trace print_requests.py //pipeline > out.requests_delete_one.json +trace print_requests.py --nostamp //pipeline > out.requests_delete_one.json trace $CLI bundle plan title "Delete the whole block and deploy again\n" grep -v PERMISSIONS databricks.yml > tmp.yml && mv tmp.yml databricks.yml trace cat databricks.yml -trace $CLI bundle plan -o json > out.plan_delete_all.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp > out.plan_delete_all.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy -trace print_requests.py //pipeline --sort > out.requests_delete_all.json +trace print_requests.py --nostamp //pipeline --sort > out.requests_delete_all.json trace $CLI bundle plan title "Restore original config\n" mv databricks.yml.saved databricks.yml -trace $CLI bundle plan -o json > out.plan_restore.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp > out.plan_restore.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy -trace print_requests.py //pipeline > out.requests_restore_original.json +trace print_requests.py --nostamp //pipeline > out.requests_restore_original.json trace $CLI bundle plan trace $CLI bundle destroy --auto-approve -print_requests.py //pipeline > out.requests_destroy.json +print_requests.py --nostamp //pipeline > out.requests_destroy.json +rm -f out.requests.txt diff --git a/acceptance/bundle/resources/permissions/target_permissions/script b/acceptance/bundle/resources/permissions/target_permissions/script index 67a775cfb49..2c63ad82cd2 100644 --- a/acceptance/bundle/resources/permissions/target_permissions/script +++ b/acceptance/bundle/resources/permissions/target_permissions/script @@ -1,7 +1,7 @@ trace $CLI bundle plan $CLI bundle plan -o json | jq 'del(.. | objects | .deployment_id?, .version_id?)' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy -print_requests.py //jobs/ > out.requests_create.$DATABRICKS_BUNDLE_ENGINE.json +print_requests.py --nostamp //jobs/ > out.requests_create.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle destroy --auto-approve -print_requests.py //jobs/ > out.requests_delete.$DATABRICKS_BUNDLE_ENGINE.json +print_requests.py --nostamp //jobs/ > out.requests_delete.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/pipelines/allow-duplicate-names/output.txt b/acceptance/bundle/resources/pipelines/allow-duplicate-names/output.txt index 7f517420608..d620b6d36bf 100644 --- a/acceptance/bundle/resources/pipelines/allow-duplicate-names/output.txt +++ b/acceptance/bundle/resources/pipelines/allow-duplicate-names/output.txt @@ -5,7 +5,7 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> print_requests.py //pipelines --del-body deployment.deployment_id,deployment.version_id,new_settings.deployment.deployment_id,new_settings.deployment.version_id +>>> print_requests.py //pipelines --nostamp { "method": "POST", "path": "/api/2.0/pipelines", diff --git a/acceptance/bundle/resources/pipelines/allow-duplicate-names/script b/acceptance/bundle/resources/pipelines/allow-duplicate-names/script index 9055e4ba02b..ec26ce55407 100644 --- a/acceptance/bundle/resources/pipelines/allow-duplicate-names/script +++ b/acceptance/bundle/resources/pipelines/allow-duplicate-names/script @@ -16,4 +16,4 @@ export PIPELINE_ID # Deploy the bundle that has a pipeline with the same name: trace $CLI bundle deploy -trace print_requests.py //pipelines --del-body deployment.deployment_id,deployment.version_id,new_settings.deployment.deployment_id,new_settings.deployment.version_id +trace print_requests.py //pipelines --nostamp diff --git a/acceptance/bundle/resources/pipelines/auto-approve/script b/acceptance/bundle/resources/pipelines/auto-approve/script index 7c68356ea7c..f0b6c4f9044 100644 --- a/acceptance/bundle/resources/pipelines/auto-approve/script +++ b/acceptance/bundle/resources/pipelines/auto-approve/script @@ -9,12 +9,12 @@ trap cleanup EXIT trace $CLI bundle deploy title "Assert the pipeline is created" -PIPELINE_ID=$($CLI bundle summary -o json | jq -r '.resources.pipelines.bar.id') -trace $CLI pipelines get "${PIPELINE_ID}" | jq "{spec}" +PIPELINE_ID=$($CLI bundle summary -o json | nostamp | jq -r '.resources.pipelines.bar.id') +trace $CLI pipelines get "${PIPELINE_ID}" | nostamp| jq "{spec}" title "Assert the job is created" -JOB_ID=$($CLI bundle summary -o json | jq -r '.resources.jobs.foo.id') -$CLI jobs get "${JOB_ID}" | jq '{name: .settings.name}' +JOB_ID=$($CLI bundle summary -o json | nostamp | jq -r '.resources.jobs.foo.id') +$CLI jobs get "${JOB_ID}" | nostamp| jq '{name: .settings.name}' title "Remove resources from configuration." trace rm resources.yml diff --git a/acceptance/bundle/resources/pipelines/drift/parameters/output.txt b/acceptance/bundle/resources/pipelines/drift/parameters/output.txt index 9043ddc35b5..66e3e709e5d 100644 --- a/acceptance/bundle/resources/pipelines/drift/parameters/output.txt +++ b/acceptance/bundle/resources/pipelines/drift/parameters/output.txt @@ -17,7 +17,7 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> print_requests.py //api/2.0/pipelines +>>> print_requests.py --nostamp //api/2.0/pipelines json.method = "POST"; json.path = "/api/2.0/pipelines"; json.body.channel = "CURRENT"; diff --git a/acceptance/bundle/resources/pipelines/drift/parameters/script b/acceptance/bundle/resources/pipelines/drift/parameters/script index 6773f248917..1d9fd0f2c02 100644 --- a/acceptance/bundle/resources/pipelines/drift/parameters/script +++ b/acceptance/bundle/resources/pipelines/drift/parameters/script @@ -8,4 +8,4 @@ trace $CLI bundle plan | contains.py "Plan: 0 to add, 0 to change, 0 to delete, title "Redeploy is a no-op (no update call)" trace $CLI bundle deploy -trace print_requests.py //api/2.0/pipelines | gron.py | contains.py '!json.method = "PUT"' +trace print_requests.py --nostamp //api/2.0/pipelines | gron.py | contains.py '!json.method = "PUT"' diff --git a/acceptance/bundle/resources/pipelines/lakeflow-pipeline/script b/acceptance/bundle/resources/pipelines/lakeflow-pipeline/script index d380b57c974..a319f62fb18 100644 --- a/acceptance/bundle/resources/pipelines/lakeflow-pipeline/script +++ b/acceptance/bundle/resources/pipelines/lakeflow-pipeline/script @@ -8,5 +8,5 @@ cleanup() { trap cleanup EXIT trace $CLI bundle deploy -trace jq -s '.[] | select(.path=="/api/2.0/pipelines" and .method == "POST") | .body' out.requests.txt +trace jq -s '.[] | select(.path=="/api/2.0/pipelines" and .method == "POST") | .body' out.requests.txt | nostamp rm out.requests.txt diff --git a/acceptance/bundle/resources/pipelines/num-workers-zero/out.requests.direct.txt b/acceptance/bundle/resources/pipelines/num-workers-zero/out.requests.direct.txt index 62341cc543f..547928989d9 100644 --- a/acceptance/bundle/resources/pipelines/num-workers-zero/out.requests.direct.txt +++ b/acceptance/bundle/resources/pipelines/num-workers-zero/out.requests.direct.txt @@ -1,5 +1,3 @@ - ->>> errcode jq -s .[] | select(.method == "POST" and (.path | contains("/pipelines"))) out.requests.txt { "method": "POST", "path": "/api/2.0/pipelines", diff --git a/acceptance/bundle/resources/pipelines/num-workers-zero/out.requests.terraform.txt b/acceptance/bundle/resources/pipelines/num-workers-zero/out.requests.terraform.txt index 7bf55181288..e1ec34db0bb 100644 --- a/acceptance/bundle/resources/pipelines/num-workers-zero/out.requests.terraform.txt +++ b/acceptance/bundle/resources/pipelines/num-workers-zero/out.requests.terraform.txt @@ -1,5 +1,3 @@ - ->>> errcode jq -s .[] | select(.method == "POST" and (.path | contains("/pipelines"))) out.requests.txt { "method": "POST", "path": "/api/2.0/pipelines", diff --git a/acceptance/bundle/resources/pipelines/num-workers-zero/output.txt b/acceptance/bundle/resources/pipelines/num-workers-zero/output.txt index e69de29bb2d..825621b4559 100644 --- a/acceptance/bundle/resources/pipelines/num-workers-zero/output.txt +++ b/acceptance/bundle/resources/pipelines/num-workers-zero/output.txt @@ -0,0 +1,2 @@ + +>>> errcode jq -s .[] | select(.method == "POST" and (.path | contains("/pipelines"))) out.requests.txt diff --git a/acceptance/bundle/resources/pipelines/num-workers-zero/script b/acceptance/bundle/resources/pipelines/num-workers-zero/script index a33c1771e96..20a48929475 100644 --- a/acceptance/bundle/resources/pipelines/num-workers-zero/script +++ b/acceptance/bundle/resources/pipelines/num-workers-zero/script @@ -1,3 +1,3 @@ trace $CLI bundle deploy > out.deploy.$DATABRICKS_BUNDLE_ENGINE.txt 2>&1 -trace errcode jq -s '.[] | select(.method == "POST" and (.path | contains("/pipelines")))' out.requests.txt > out.requests.$DATABRICKS_BUNDLE_ENGINE.txt 2>&1 +trace errcode jq -s '.[] | select(.method == "POST" and (.path | contains("/pipelines")))' out.requests.txt | nostamp > out.requests.$DATABRICKS_BUNDLE_ENGINE.txt 2>&1 rm out.requests.txt diff --git a/acceptance/bundle/resources/pipelines/photon-true/script b/acceptance/bundle/resources/pipelines/photon-true/script index 73eba500cd0..3fe12431e65 100644 --- a/acceptance/bundle/resources/pipelines/photon-true/script +++ b/acceptance/bundle/resources/pipelines/photon-true/script @@ -1,2 +1,2 @@ trace $CLI bundle deploy -print_requests.py //api/2.0/pipelines +print_requests.py --nostamp //api/2.0/pipelines diff --git a/acceptance/bundle/resources/pipelines/recreate-keys/_script b/acceptance/bundle/resources/pipelines/recreate-keys/_script index d1cb49b382c..2ab649947da 100644 --- a/acceptance/bundle/resources/pipelines/recreate-keys/_script +++ b/acceptance/bundle/resources/pipelines/recreate-keys/_script @@ -2,13 +2,13 @@ trace cat databricks.yml touch foo.py touch bar.py trace $CLI bundle plan # should show 'create' -$CLI bundle plan -o json > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy ppid1=`read_id.py my` print_requests() { - jq --sort-keys 'select(.method != "GET" and (.path | contains("/pipelines")))' < out.requests.txt + jq --sort-keys 'select(.method != "GET" and (.path | contains("/pipelines")))' < out.requests.txt | nostamp rm -f out.requests.txt } @@ -16,14 +16,14 @@ trace print_requests trace update_file.py databricks.yml $CONFIG_UPDATE trace $CLI bundle plan # should show 'recreate' -$CLI bundle plan -o json > out.plan_recreate.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp > out.plan_recreate.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy --auto-approve trace print_requests title "Fetch pipeline ID and verify remote state" ppid2=`read_id.py my` -trace $CLI pipelines get $ppid2 +trace $CLI pipelines get $ppid2 | nostamp title "Verify that original pipeline is gone" trace musterr $CLI pipelines get $ppid1 diff --git a/acceptance/bundle/resources/pipelines/recreate/script b/acceptance/bundle/resources/pipelines/recreate/script index 08423e1d323..0a814d08985 100644 --- a/acceptance/bundle/resources/pipelines/recreate/script +++ b/acceptance/bundle/resources/pipelines/recreate/script @@ -8,8 +8,8 @@ trap cleanup EXIT trace $CLI bundle deploy title "Assert the pipeline is created with catalog" -PIPELINE_ID=$($CLI bundle summary -o json | jq -r '.resources.pipelines.foo.id') -trace $CLI pipelines get "${PIPELINE_ID}" | jq "{spec}" +PIPELINE_ID=$($CLI bundle summary -o json | nostamp | jq -r '.resources.pipelines.foo.id') +trace $CLI pipelines get "${PIPELINE_ID}" | nostamp| jq "{spec}" # Note: In Terraform provider v1.98.0+, changing catalog no longer triggers recreation. # We switch to using storage location instead, which still triggers recreation. diff --git a/acceptance/bundle/resources/pipelines/remote_matches_config/script b/acceptance/bundle/resources/pipelines/remote_matches_config/script index ccbc1936b57..9896fb8e4f1 100644 --- a/acceptance/bundle/resources/pipelines/remote_matches_config/script +++ b/acceptance/bundle/resources/pipelines/remote_matches_config/script @@ -15,6 +15,6 @@ r["run_as"] = {"user_name": "changed@example.test"} EOF trace $CLI bundle plan -$CLI bundle plan -o json | jq 'del(.. | objects | .deployment_id?, .version_id?)' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp | jq 'del(.. | objects | .deployment_id?, .version_id?)' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json rm out.requests.txt diff --git a/acceptance/bundle/resources/pipelines/update/script b/acceptance/bundle/resources/pipelines/update/script index f48e025aba2..d9eb259cbb9 100644 --- a/acceptance/bundle/resources/pipelines/update/script +++ b/acceptance/bundle/resources/pipelines/update/script @@ -4,7 +4,7 @@ touch bar.py trace $CLI bundle deploy print_requests() { - print_requests.py //pipelines --del-body deployment.deployment_id,deployment.version_id,new_settings.deployment.deployment_id,new_settings.deployment.version_id + print_requests.py //pipelines --nostamp read_state.py pipelines my id name } @@ -21,7 +21,7 @@ rm out.requests.txt title "Fetch pipeline ID and verify remote state" ppid=`read_id.py my` -trace $CLI pipelines get $ppid +trace $CLI pipelines get $ppid | nostamp rm out.requests.txt title "Destroy the pipeline and verify that it's removed from the state and from remote" diff --git a/acceptance/bundle/resources/pipelines/zero-value-fields/script b/acceptance/bundle/resources/pipelines/zero-value-fields/script index 06f64fc53fe..ef7e886ea01 100644 --- a/acceptance/bundle/resources/pipelines/zero-value-fields/script +++ b/acceptance/bundle/resources/pipelines/zero-value-fields/script @@ -1,5 +1,5 @@ trace $CLI bundle deploy -print_requests.py //api/2.0/pipelines > out.requests.$DATABRICKS_BUNDLE_ENGINE.txt 2>&1 +print_requests.py --nostamp //api/2.0/pipelines > out.requests.$DATABRICKS_BUNDLE_ENGINE.txt 2>&1 if [ -f .databricks/bundle/default/terraform/bundle.tf.json ]; then jq .resource.databricks_pipeline .databricks/bundle/default/terraform/bundle.tf.json > out.tf.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/postgres_branches/update_protected/script b/acceptance/bundle/resources/postgres_branches/update_protected/script index 28d21b69ecf..1566c885831 100755 --- a/acceptance/bundle/resources/postgres_branches/update_protected/script +++ b/acceptance/bundle/resources/postgres_branches/update_protected/script @@ -15,7 +15,7 @@ print_requests() { title "Initial deployment" trace $CLI bundle validate trace $CLI bundle plan -trace $CLI bundle plan -o json | jq '.plan."resources.postgres_branches.dev_branch" // .' > out.plan.create.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp | jq '.plan."resources.postgres_branches.dev_branch" // .' > out.plan.create.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt trace $CLI bundle deploy @@ -28,7 +28,7 @@ trace $CLI postgres get-branch "${branch_name}" | branch_fields title "Verify no_change (no changes)" trace $CLI bundle plan -trace $CLI bundle plan -o json | jq '(.plan."resources.postgres_branches.dev_branch" // .) | del(.remote_state.status.logical_size_bytes)' > out.plan.no_change.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp | jq '(.plan."resources.postgres_branches.dev_branch" // .) | del(.remote_state.status.logical_size_bytes)' > out.plan.no_change.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt trace $CLI bundle deploy @@ -39,7 +39,7 @@ trace update_file.py databricks.yml "is_protected: false" "is_protected: true" trace cat databricks.yml trace $CLI bundle plan -trace $CLI bundle plan -o json | jq '(.plan."resources.postgres_branches.dev_branch" // .) | del(.remote_state.status.logical_size_bytes)' > out.plan.update.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp | jq '(.plan."resources.postgres_branches.dev_branch" // .) | del(.remote_state.status.logical_size_bytes)' > out.plan.update.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt trace $CLI bundle deploy @@ -51,7 +51,7 @@ trace $CLI postgres get-branch "${branch_name}" | branch_fields title "Restore is_protected to false" trace update_file.py databricks.yml "is_protected: true" "is_protected: false" trace $CLI bundle plan -trace $CLI bundle plan -o json | jq '(.plan."resources.postgres_branches.dev_branch" // .) | del(.remote_state.status.logical_size_bytes)' > out.plan.restore.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp | jq '(.plan."resources.postgres_branches.dev_branch" // .) | del(.remote_state.status.logical_size_bytes)' > out.plan.restore.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt trace $CLI bundle deploy diff --git a/acceptance/bundle/resources/postgres_databases/update/script b/acceptance/bundle/resources/postgres_databases/update/script index e23fea3bbaa..bdbad2b67c6 100644 --- a/acceptance/bundle/resources/postgres_databases/update/script +++ b/acceptance/bundle/resources/postgres_databases/update/script @@ -15,7 +15,7 @@ print_requests() { title "Initial deployment" trace $CLI bundle validate trace $CLI bundle plan -trace $CLI bundle plan -o json | jq '.plan."resources.postgres_databases.my_database" // .' > out.plan.create.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp | jq '.plan."resources.postgres_databases.my_database" // .' > out.plan.create.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt trace $CLI bundle deploy @@ -29,7 +29,7 @@ trace $CLI postgres get-database "${database_name}" | database_fields title "Verify no changes" trace $CLI bundle plan -trace $CLI bundle plan -o json | jq '.plan."resources.postgres_databases.my_database" // .' > out.plan.no_change.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp | jq '.plan."resources.postgres_databases.my_database" // .' > out.plan.no_change.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt trace $CLI bundle deploy @@ -40,7 +40,7 @@ trace update_file.py databricks.yml "postgres_database: initial_db_name" "postgr trace cat databricks.yml trace $CLI bundle plan -trace $CLI bundle plan -o json | jq '.plan."resources.postgres_databases.my_database" // .' > out.plan.update.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp | jq '.plan."resources.postgres_databases.my_database" // .' > out.plan.update.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt trace $CLI bundle deploy @@ -52,7 +52,7 @@ trace $CLI postgres get-database "${database_name}" | database_fields title "Restore postgres_database to original value" trace update_file.py databricks.yml "postgres_database: renamed_db_name" "postgres_database: initial_db_name" trace $CLI bundle plan -trace $CLI bundle plan -o json | jq '.plan."resources.postgres_databases.my_database" // .' > out.plan.restore.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp | jq '.plan."resources.postgres_databases.my_database" // .' > out.plan.restore.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt trace $CLI bundle deploy diff --git a/acceptance/bundle/resources/postgres_endpoints/update_autoscaling/script b/acceptance/bundle/resources/postgres_endpoints/update_autoscaling/script index 1df081df0a6..f2506c21a01 100755 --- a/acceptance/bundle/resources/postgres_endpoints/update_autoscaling/script +++ b/acceptance/bundle/resources/postgres_endpoints/update_autoscaling/script @@ -15,7 +15,7 @@ print_requests() { title "Initial deployment" trace $CLI bundle validate trace $CLI bundle plan -trace $CLI bundle plan -o json | jq '.plan."resources.postgres_endpoints.my_endpoint" // . | del(.remote_state)' > out.plan.create.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp | jq '.plan."resources.postgres_endpoints.my_endpoint" // . | del(.remote_state)' > out.plan.create.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt trace $CLI bundle deploy @@ -29,7 +29,7 @@ trace $CLI postgres get-endpoint "${endpoint_name}" | endpoint_fields title "Verify no changes" trace $CLI bundle plan -trace $CLI bundle plan -o json | jq '.plan."resources.postgres_endpoints.my_endpoint" // . | del(.remote_state)' > out.plan.no_change.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp | jq '.plan."resources.postgres_endpoints.my_endpoint" // . | del(.remote_state)' > out.plan.no_change.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt trace $CLI bundle deploy @@ -40,7 +40,7 @@ trace update_file.py databricks.yml "autoscaling_limit_max_cu: 8" "autoscaling_l trace cat databricks.yml trace $CLI bundle plan -trace $CLI bundle plan -o json | jq '.plan."resources.postgres_endpoints.my_endpoint" // . | del(.remote_state)' > out.plan.update.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp | jq '.plan."resources.postgres_endpoints.my_endpoint" // . | del(.remote_state)' > out.plan.update.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt trace $CLI bundle deploy @@ -52,7 +52,7 @@ trace $CLI postgres get-endpoint "${endpoint_name}" | endpoint_fields title "Restore endpoint autoscaling to original value" trace update_file.py databricks.yml "autoscaling_limit_max_cu: 4" "autoscaling_limit_max_cu: 8" trace $CLI bundle plan -trace $CLI bundle plan -o json | jq '.plan."resources.postgres_endpoints.my_endpoint" // . | del(.remote_state)' > out.plan.restore.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp | jq '.plan."resources.postgres_endpoints.my_endpoint" // . | del(.remote_state)' > out.plan.restore.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt trace $CLI bundle deploy diff --git a/acceptance/bundle/resources/postgres_projects/update_display_name/script b/acceptance/bundle/resources/postgres_projects/update_display_name/script index 5c6ac02445e..657eca21869 100755 --- a/acceptance/bundle/resources/postgres_projects/update_display_name/script +++ b/acceptance/bundle/resources/postgres_projects/update_display_name/script @@ -15,7 +15,7 @@ print_requests() { title "Initial deployment" trace $CLI bundle validate trace $CLI bundle plan -trace $CLI bundle plan -o json | jq '.plan."resources.postgres_projects.my_project" // .' > out.plan.create.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp | jq '.plan."resources.postgres_projects.my_project" // .' > out.plan.create.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt trace $CLI bundle deploy @@ -27,7 +27,7 @@ trace $CLI postgres get-project "${project_name}" | project_fields title "Verify no changes" trace $CLI bundle plan -trace $CLI bundle plan -o json | jq '.plan."resources.postgres_projects.my_project" // .' > out.plan.no_change.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp | jq '.plan."resources.postgres_projects.my_project" // .' > out.plan.no_change.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt trace $CLI bundle deploy @@ -38,7 +38,7 @@ trace update_file.py databricks.yml "Original Name" "Updated Name" trace cat databricks.yml trace $CLI bundle plan -trace $CLI bundle plan -o json | jq '.plan."resources.postgres_projects.my_project" // .' > out.plan.update.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp | jq '.plan."resources.postgres_projects.my_project" // .' > out.plan.update.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt trace $CLI bundle deploy @@ -50,7 +50,7 @@ trace $CLI postgres get-project "${project_name}" | project_fields title "Restore display_name to original value" trace update_file.py databricks.yml "Updated Name" "Original Name" trace $CLI bundle plan -trace $CLI bundle plan -o json | jq '.plan."resources.postgres_projects.my_project" // .' > out.plan.restore.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp | jq '.plan."resources.postgres_projects.my_project" // .' > out.plan.restore.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt trace $CLI bundle deploy diff --git a/acceptance/bundle/resources/postgres_roles/update/script b/acceptance/bundle/resources/postgres_roles/update/script index 6c727a34536..5eea84136a6 100644 --- a/acceptance/bundle/resources/postgres_roles/update/script +++ b/acceptance/bundle/resources/postgres_roles/update/script @@ -15,7 +15,7 @@ print_requests() { title "Initial deployment" trace $CLI bundle validate trace $CLI bundle plan -trace $CLI bundle plan -o json | jq '.plan."resources.postgres_roles.my_role" // .' > out.plan.create.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp | jq '.plan."resources.postgres_roles.my_role" // .' > out.plan.create.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt trace $CLI bundle deploy @@ -28,7 +28,7 @@ trace $CLI postgres get-role "${role_name}" | role_fields title "Verify no changes" trace $CLI bundle plan -trace $CLI bundle plan -o json | jq '.plan."resources.postgres_roles.my_role" // .' > out.plan.no_change.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp | jq '.plan."resources.postgres_roles.my_role" // .' > out.plan.no_change.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt trace $CLI bundle deploy @@ -39,7 +39,7 @@ trace update_file.py databricks.yml "createdb: false" "createdb: true" trace cat databricks.yml trace $CLI bundle plan -trace $CLI bundle plan -o json | jq '.plan."resources.postgres_roles.my_role" // .' > out.plan.update.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp | jq '.plan."resources.postgres_roles.my_role" // .' > out.plan.update.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt trace $CLI bundle deploy @@ -50,7 +50,7 @@ trace $CLI postgres get-role "${role_name}" | role_fields title "Restore attributes.createdb to original value" trace update_file.py databricks.yml "createdb: true" "createdb: false" trace $CLI bundle plan -trace $CLI bundle plan -o json | jq '.plan."resources.postgres_roles.my_role" // .' > out.plan.restore.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp | jq '.plan."resources.postgres_roles.my_role" // .' > out.plan.restore.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt trace $CLI bundle deploy diff --git a/acceptance/bundle/resources/registered_models/basic/script b/acceptance/bundle/resources/registered_models/basic/script index 42313057e0b..61dff74182c 100644 --- a/acceptance/bundle/resources/registered_models/basic/script +++ b/acceptance/bundle/resources/registered_models/basic/script @@ -20,7 +20,7 @@ trap cleanup EXIT deploy_registered_model() { trace $CLI bundle plan trace $CLI bundle deploy - registered_model_id=$($CLI bundle summary --output json | jq -r '.resources.registered_models.my_registered_model.id') + registered_model_id=$($CLI bundle summary --output json | nostamp | jq -r '.resources.registered_models.my_registered_model.id') trace $CLI registered-models get "${registered_model_id}" | jq '{name, comment, catalog_name, schema_name}' } diff --git a/acceptance/bundle/resources/schemas/auto-approve/script b/acceptance/bundle/resources/schemas/auto-approve/script index 9ea9f4fc410..f2f2b6cbade 100644 --- a/acceptance/bundle/resources/schemas/auto-approve/script +++ b/acceptance/bundle/resources/schemas/auto-approve/script @@ -20,8 +20,8 @@ title "Assert the schema is created" trace $CLI schemas get "${CATALOG_NAME}.${SCHEMA_NAME}" | jq "{full_name, comment}" title "Assert the pipeline is created and uses the schema" -PIPELINE_ID=$($CLI bundle summary -o json | jq -r '.resources.pipelines.foo.id') -trace $CLI pipelines get "${PIPELINE_ID}" | jq "{spec}" +PIPELINE_ID=$($CLI bundle summary -o json | nostamp | jq -r '.resources.pipelines.foo.id') +trace $CLI pipelines get "${PIPELINE_ID}" | nostamp| jq "{spec}" title "Create a volume in the schema, and add a file to it. This ensures that the schema has some data in it and deletion will fail unless the generated diff --git a/acceptance/bundle/resources/test.toml b/acceptance/bundle/resources/test.toml index 3d68723dbe7..159efe02696 100644 --- a/acceptance/bundle/resources/test.toml +++ b/acceptance/bundle/resources/test.toml @@ -1,57 +1 @@ RecordRequests = true - -# These normalize the deployment stamp recording adds to every job and pipeline, so a test -# asserts the same golden files whether or not recording is on - that is the point of the -# DMS run, rather than keeping a second copy of 600-odd output files. They are repeated per -# subtree rather than living in the parent because bundle/dms asserts the stamp itself and -# would inherit them. -# -# The stamp as a change the plan reports on its own. It shows up at whatever depth the -# enclosing object sits at, so the indent is matched loosely; the body lines are matched as -# `"key": value` pairs rather than `.*` so the match stops at the entry's own closing brace -# instead of running into its siblings. (Go's regexp is RE2, so the indent cannot be -# captured and back-referenced.) -[[Repls]] -Old = '(?m)^ *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\},\n' -New = '' - -# Same entry when it is the last one in the object, so the comma is on the line before. -[[Repls]] -Old = '(?m),\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n' -New = "\n" - -# When the stamp is the only entry, the whole "changes" object exists because of recording. -# The trailing-comma form comes first: the rule after it would match the same text and -# leave the comma on the previous line dangling. -[[Repls]] -Old = '(?m),\n *"changes": \{\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n *\}\n' -New = "\n" - -[[Repls]] -Old = '(?m)^ *"changes": \{\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n *\},?\n' -New = '' - -# The stamp inside a serialized deployment block, as printed by `jobs get` / `pipelines get` -# and in state dumps. Both keys always sit alongside "kind" and "metadata_file_path", so -# each pattern anchors on one of those - that keeps it from matching an unrelated field -# named deployment_id elsewhere in the output. -# -# Order puts these after the root's numeric rules (Order = 10), which have by then turned -# the id into [NUMID]. -[[Repls]] -Old = '(?m)^ *"deployment_id": "\[NUMID\]",\n( *"kind": "BUNDLE")' -New = '${1}' -Order = 20 - -[[Repls]] -# The value is required to be non-empty: a terraform state dump carries -# "version_id": "" for a job it never stamped, and that line is not ours to drop. -Old = '(?m)(^ *"metadata_file_path": [^\n]*),\n *"version_id": "[^"\n]+"\n' -New = "$1\n" -Order = 20 - -# Same two keys in gron.py's flattened form, where each is its own line. -[[Repls]] -Old = '(?m)^json.*\.deployment\.(deployment_id|version_id) = [^\n]*\n' -New = '' -Order = 20 diff --git a/acceptance/bundle/run_as/job_default/output.txt b/acceptance/bundle/run_as/job_default/output.txt index 137a49cbd82..b7f3fcc5128 100644 --- a/acceptance/bundle/run_as/job_default/output.txt +++ b/acceptance/bundle/run_as/job_default/output.txt @@ -6,7 +6,7 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> print_requests.py //jobs +>>> print_requests.py --nostamp //jobs { "method": "POST", "path": "/api/2.2/jobs/create", @@ -58,7 +58,7 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> print_requests.py //jobs +>>> print_requests.py --nostamp //jobs { "method": "POST", "path": "/api/2.2/jobs/reset", diff --git a/acceptance/bundle/run_as/job_default/script b/acceptance/bundle/run_as/job_default/script index 3e80f98838e..be96b205477 100644 --- a/acceptance/bundle/run_as/job_default/script +++ b/acceptance/bundle/run_as/job_default/script @@ -8,7 +8,7 @@ trap cleanup EXIT title "Deploy with run_as" trace $CLI bundle deploy -trace print_requests.py //jobs | contains.py "!GET" "POST" +trace print_requests.py --nostamp //jobs | contains.py "!GET" "POST" JOB_ID=$($CLI bundle summary -o json | jq -r '.resources.jobs.job_with_run_as.id') trace $CLI jobs get $JOB_ID | jq -r '.settings.run_as' @@ -18,5 +18,5 @@ update_file.py databricks.yml "run_as: title "Remove run_as and redeploy" trace $CLI bundle plan trace $CLI bundle deploy -trace print_requests.py //jobs | contains.py "!GET" "POST" +trace print_requests.py --nostamp //jobs | contains.py "!GET" "POST" trace $CLI jobs get $JOB_ID | jq -r '.settings.run_as' diff --git a/acceptance/bundle/run_as/pipelines/_script b/acceptance/bundle/run_as/pipelines/_script index ad875a39a75..a4ae022ac22 100644 --- a/acceptance/bundle/run_as/pipelines/_script +++ b/acceptance/bundle/run_as/pipelines/_script @@ -1,5 +1,5 @@ print_requests() { - jq --sort-keys 'select(.method != "GET" and (.path | contains("/pipelines")))' < out.requests.txt + jq --sort-keys 'select(.method != "GET" and (.path | contains("/pipelines")))' < out.requests.txt | nostamp rm out.requests.txt } @@ -15,7 +15,7 @@ for target in "${targets[@]}"; do trace $CLI bundle plan -t $target # Debug plan - $CLI bundle plan -o json -t $target > out.plan_$target.$DATABRICKS_BUNDLE_ENGINE.json + $CLI bundle plan -o json -t $target | nostamp > out.plan_$target.$DATABRICKS_BUNDLE_ENGINE.json # Deploy rm -f out.requests.txt diff --git a/acceptance/bundle/run_as/test.toml b/acceptance/bundle/run_as/test.toml deleted file mode 100644 index abf7cc15484..00000000000 --- a/acceptance/bundle/run_as/test.toml +++ /dev/null @@ -1,55 +0,0 @@ -# These normalize the deployment stamp recording adds to every job and pipeline, so a test -# asserts the same golden files whether or not recording is on - that is the point of the -# DMS run, rather than keeping a second copy of 600-odd output files. They are repeated per -# subtree rather than living in the parent because bundle/dms asserts the stamp itself and -# would inherit them. -# -# The stamp as a change the plan reports on its own. It shows up at whatever depth the -# enclosing object sits at, so the indent is matched loosely; the body lines are matched as -# `"key": value` pairs rather than `.*` so the match stops at the entry's own closing brace -# instead of running into its siblings. (Go's regexp is RE2, so the indent cannot be -# captured and back-referenced.) -[[Repls]] -Old = '(?m)^ *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\},\n' -New = '' - -# Same entry when it is the last one in the object, so the comma is on the line before. -[[Repls]] -Old = '(?m),\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n' -New = "\n" - -# When the stamp is the only entry, the whole "changes" object exists because of recording. -# The trailing-comma form comes first: the rule after it would match the same text and -# leave the comma on the previous line dangling. -[[Repls]] -Old = '(?m),\n *"changes": \{\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n *\}\n' -New = "\n" - -[[Repls]] -Old = '(?m)^ *"changes": \{\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n *\},?\n' -New = '' - -# The stamp inside a serialized deployment block, as printed by `jobs get` / `pipelines get` -# and in state dumps. Both keys always sit alongside "kind" and "metadata_file_path", so -# each pattern anchors on one of those - that keeps it from matching an unrelated field -# named deployment_id elsewhere in the output. -# -# Order puts these after the root's numeric rules (Order = 10), which have by then turned -# the id into [NUMID]. -[[Repls]] -Old = '(?m)^ *"deployment_id": "\[NUMID\]",\n( *"kind": "BUNDLE")' -New = '${1}' -Order = 20 - -[[Repls]] -# The value is required to be non-empty: a terraform state dump carries -# "version_id": "" for a job it never stamped, and that line is not ours to drop. -Old = '(?m)(^ *"metadata_file_path": [^\n]*),\n *"version_id": "[^"\n]+"\n' -New = "$1\n" -Order = 20 - -# Same two keys in gron.py's flattened form, where each is its own line. -[[Repls]] -Old = '(?m)^json.*\.deployment\.(deployment_id|version_id) = [^\n]*\n' -New = '' -Order = 20 diff --git a/acceptance/bundle/script.prepare b/acceptance/bundle/script.prepare new file mode 100644 index 00000000000..340d360e2c8 --- /dev/null +++ b/acceptance/bundle/script.prepare @@ -0,0 +1,18 @@ +nostamp() { + # Drop the deployment stamp (deployment_id / version_id) that history recording adds to + # every job and pipeline, so a test asserts the same JSON with recording on or off. Pipe + # a plan, a state dump, or a resource payload through this. See DATABRICKS_BUNDLE_DMS in + # bundle/test.toml; bundle/dms asserts the stamp and does not use this. + # + # Only inside a block that also has "kind" and "metadata_file_path", so an unrelated + # deployment_id survives. An empty value survives too: a terraform state dump carries + # "version_id": "" for a job it never stamped. + # + # A plan reports the stamp as its own change entry, keyed by field path, so those go as + # well - and with them a "changes" object left empty, which recording alone created. + jq '((.. | objects | select(has("kind") and has("metadata_file_path"))) + |= with_entries(select((.key | IN("deployment_id", "version_id")) == false or .value == ""))) + | ((.. | objects | .changes? | objects) + |= with_entries(select(.key | IN("deployment.deployment_id", "deployment.version_id") | not))) + | del(.. | objects | select(.changes == {}) | .changes)' +} diff --git a/acceptance/bundle/select/basic/output.txt b/acceptance/bundle/select/basic/output.txt index 56c20a404d4..6d3af9307f1 100644 --- a/acceptance/bundle/select/basic/output.txt +++ b/acceptance/bundle/select/basic/output.txt @@ -31,7 +31,7 @@ Deployment complete! === Telemetry: select_used true ->>> print_requests.py --sort //jobs +>>> print_requests.py --nostamp --sort //jobs { "method": "POST", "path": "/api/2.2/jobs/create", @@ -105,7 +105,7 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> print_requests.py --sort //jobs +>>> print_requests.py --nostamp --sort //jobs { "method": "POST", "path": "/api/2.2/jobs/create", diff --git a/acceptance/bundle/select/basic/script b/acceptance/bundle/select/basic/script index 88c61a537b7..12db6ca0ba0 100644 --- a/acceptance/bundle/select/basic/script +++ b/acceptance/bundle/select/basic/script @@ -22,14 +22,14 @@ trace $CLI bundle plan --select jobs.foo # JSON embeds remote state that differs between local and cloud), then deploy it: # inline, or via --plan (READPLAN=1). The deploy is not traced because readplanarg # varies the command line between READPLAN variants, which must produce identical output. -$CLI bundle plan --select jobs.foo -o json > plan.json +$CLI bundle plan --select jobs.foo -o json | nostamp > plan.json title "bundle deploy --select jobs.foo\n" $CLI bundle deploy --select jobs.foo $(readplanarg plan.json) # The deploy reports that --select was used via telemetry. title "Telemetry:\n" print_telemetry_bool_values | grep '^select_used ' # Only bar and foo were created, never baz. -trace print_requests.py --sort //jobs +trace print_requests.py --nostamp --sort //jobs # Summary after the partial deploy: foo and bar are deployed, baz is not. trace $CLI bundle summary @@ -37,11 +37,11 @@ trace $CLI bundle summary # foo and bar are already deployed, so only baz remains to create. title "Full plan after partial deploy" trace $CLI bundle plan -$CLI bundle plan -o json > plan-full.json +$CLI bundle plan -o json | nostamp > plan-full.json title "Full deploy\n" $CLI bundle deploy $(readplanarg plan-full.json) # Only baz is created this time. -trace print_requests.py --sort //jobs +trace print_requests.py --nostamp --sort //jobs # Everything is deployed now: no changes. title "Full plan again" trace $CLI bundle plan diff --git a/acceptance/bundle/select/test.toml b/acceptance/bundle/select/test.toml index 792257f226f..85ce448afd3 100644 --- a/acceptance/bundle/select/test.toml +++ b/acceptance/bundle/select/test.toml @@ -1,59 +1,3 @@ Local = true Cloud = false Ignore = [".databricks", ".gitignore"] - -# These normalize the deployment stamp recording adds to every job and pipeline, so a test -# asserts the same golden files whether or not recording is on - that is the point of the -# DMS run, rather than keeping a second copy of 600-odd output files. They are repeated per -# subtree rather than living in the parent because bundle/dms asserts the stamp itself and -# would inherit them. -# -# The stamp as a change the plan reports on its own. It shows up at whatever depth the -# enclosing object sits at, so the indent is matched loosely; the body lines are matched as -# `"key": value` pairs rather than `.*` so the match stops at the entry's own closing brace -# instead of running into its siblings. (Go's regexp is RE2, so the indent cannot be -# captured and back-referenced.) -[[Repls]] -Old = '(?m)^ *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\},\n' -New = '' - -# Same entry when it is the last one in the object, so the comma is on the line before. -[[Repls]] -Old = '(?m),\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n' -New = "\n" - -# When the stamp is the only entry, the whole "changes" object exists because of recording. -# The trailing-comma form comes first: the rule after it would match the same text and -# leave the comma on the previous line dangling. -[[Repls]] -Old = '(?m),\n *"changes": \{\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n *\}\n' -New = "\n" - -[[Repls]] -Old = '(?m)^ *"changes": \{\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n *\},?\n' -New = '' - -# The stamp inside a serialized deployment block, as printed by `jobs get` / `pipelines get` -# and in state dumps. Both keys always sit alongside "kind" and "metadata_file_path", so -# each pattern anchors on one of those - that keeps it from matching an unrelated field -# named deployment_id elsewhere in the output. -# -# Order puts these after the root's numeric rules (Order = 10), which have by then turned -# the id into [NUMID]. -[[Repls]] -Old = '(?m)^ *"deployment_id": "\[NUMID\]",\n( *"kind": "BUNDLE")' -New = '${1}' -Order = 20 - -[[Repls]] -# The value is required to be non-empty: a terraform state dump carries -# "version_id": "" for a job it never stamped, and that line is not ours to drop. -Old = '(?m)(^ *"metadata_file_path": [^\n]*),\n *"version_id": "[^"\n]+"\n' -New = "$1\n" -Order = 20 - -# Same two keys in gron.py's flattened form, where each is its own line. -[[Repls]] -Old = '(?m)^json.*\.deployment\.(deployment_id|version_id) = [^\n]*\n' -New = '' -Order = 20 diff --git a/acceptance/bundle/state/permission_level_migration/output.txt b/acceptance/bundle/state/permission_level_migration/output.txt index d6d2e3f71ca..4a57e4065da 100644 --- a/acceptance/bundle/state/permission_level_migration/output.txt +++ b/acceptance/bundle/state/permission_level_migration/output.txt @@ -12,48 +12,48 @@ Deployment complete! === Print state after deploy >>> print_state.py { - "state_version": 2, - "cli_version": "[CLI_VERSION]", - "lineage": "test-lineage", - "serial": 2, - "state": { - "resources.jobs.my_job": { - "__id__": "[NUMID]", - "state": { - "deployment": { - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" + "state_version": 2, + "cli_version": "[CLI_VERSION]", + "lineage": "test-lineage", + "serial": 2, + "state": { + "resources.jobs.my_job": { + "__id__": "[NUMID]", + "state": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "my job", + "queue": { + "enabled": true + } + } }, - "edit_mode": "UI_LOCKED", - "format": "MULTI_TASK", - "max_concurrent_runs": 1, - "name": "my job", - "queue": { - "enabled": true + "resources.jobs.my_job.permissions": { + "__id__": "/jobs/123", + "state": { + "object_id": "/jobs/[NUMID]", + "__embed__": [ + { + "level": "CAN_VIEW", + "group_name": "viewers" + }, + { + "level": "IS_OWNER", + "user_name": "[USERNAME]" + } + ] + }, + "depends_on": [ + { + "node": "resources.jobs.my_job", + "label": "${resources.jobs.my_job.id}" + } + ] } - } - }, - "resources.jobs.my_job.permissions": { - "__id__": "/jobs/123", - "state": { - "object_id": "/jobs/[NUMID]", - "__embed__": [ - { - "level": "CAN_VIEW", - "group_name": "viewers" - }, - { - "level": "IS_OWNER", - "user_name": "[USERNAME]" - } - ] - }, - "depends_on": [ - { - "node": "resources.jobs.my_job", - "label": "${resources.jobs.my_job.id}" - } - ] } - } } diff --git a/acceptance/bundle/state/permission_level_migration/script b/acceptance/bundle/state/permission_level_migration/script index c317985fc88..78d1a8bd0d4 100644 --- a/acceptance/bundle/state/permission_level_migration/script +++ b/acceptance/bundle/state/permission_level_migration/script @@ -15,4 +15,4 @@ title "Deploy (migrates state)" trace $CLI bundle deploy title "Print state after deploy" -trace print_state.py +trace print_state.py | nostamp diff --git a/acceptance/bundle/state/test.toml b/acceptance/bundle/state/test.toml deleted file mode 100644 index abf7cc15484..00000000000 --- a/acceptance/bundle/state/test.toml +++ /dev/null @@ -1,55 +0,0 @@ -# These normalize the deployment stamp recording adds to every job and pipeline, so a test -# asserts the same golden files whether or not recording is on - that is the point of the -# DMS run, rather than keeping a second copy of 600-odd output files. They are repeated per -# subtree rather than living in the parent because bundle/dms asserts the stamp itself and -# would inherit them. -# -# The stamp as a change the plan reports on its own. It shows up at whatever depth the -# enclosing object sits at, so the indent is matched loosely; the body lines are matched as -# `"key": value` pairs rather than `.*` so the match stops at the entry's own closing brace -# instead of running into its siblings. (Go's regexp is RE2, so the indent cannot be -# captured and back-referenced.) -[[Repls]] -Old = '(?m)^ *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\},\n' -New = '' - -# Same entry when it is the last one in the object, so the comma is on the line before. -[[Repls]] -Old = '(?m),\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n' -New = "\n" - -# When the stamp is the only entry, the whole "changes" object exists because of recording. -# The trailing-comma form comes first: the rule after it would match the same text and -# leave the comma on the previous line dangling. -[[Repls]] -Old = '(?m),\n *"changes": \{\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n *\}\n' -New = "\n" - -[[Repls]] -Old = '(?m)^ *"changes": \{\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n *\},?\n' -New = '' - -# The stamp inside a serialized deployment block, as printed by `jobs get` / `pipelines get` -# and in state dumps. Both keys always sit alongside "kind" and "metadata_file_path", so -# each pattern anchors on one of those - that keeps it from matching an unrelated field -# named deployment_id elsewhere in the output. -# -# Order puts these after the root's numeric rules (Order = 10), which have by then turned -# the id into [NUMID]. -[[Repls]] -Old = '(?m)^ *"deployment_id": "\[NUMID\]",\n( *"kind": "BUNDLE")' -New = '${1}' -Order = 20 - -[[Repls]] -# The value is required to be non-empty: a terraform state dump carries -# "version_id": "" for a job it never stamped, and that line is not ours to drop. -Old = '(?m)(^ *"metadata_file_path": [^\n]*),\n *"version_id": "[^"\n]+"\n' -New = "$1\n" -Order = 20 - -# Same two keys in gron.py's flattened form, where each is its own line. -[[Repls]] -Old = '(?m)^json.*\.deployment\.(deployment_id|version_id) = [^\n]*\n' -New = '' -Order = 20 diff --git a/acceptance/bundle/summary/modified_status/script b/acceptance/bundle/summary/modified_status/script index 4da723c08eb..22149b2c76b 100644 --- a/acceptance/bundle/summary/modified_status/script +++ b/acceptance/bundle/summary/modified_status/script @@ -1,14 +1,14 @@ title "Initial view of resources without id and modified_status=created" -trace $CLI bundle summary -o json | jq .resources +trace $CLI bundle summary -o json | nostamp | jq .resources trace $CLI bundle deploy title "Post-deployment view of resources with id and without modified_status" -trace $CLI bundle summary -o json | jq .resources +trace $CLI bundle summary -o json | nostamp | jq .resources mv $VARIANT databricks.yml title "Expecting all resources to have modified_status=deleted" -trace $CLI bundle summary -o json | jq .resources +trace $CLI bundle summary -o json | nostamp | jq .resources trace $CLI bundle destroy --auto-approve -trace $CLI bundle summary -o json | jq .resources +trace $CLI bundle summary -o json | nostamp | jq .resources diff --git a/acceptance/bundle/summary/test.toml b/acceptance/bundle/summary/test.toml deleted file mode 100644 index abf7cc15484..00000000000 --- a/acceptance/bundle/summary/test.toml +++ /dev/null @@ -1,55 +0,0 @@ -# These normalize the deployment stamp recording adds to every job and pipeline, so a test -# asserts the same golden files whether or not recording is on - that is the point of the -# DMS run, rather than keeping a second copy of 600-odd output files. They are repeated per -# subtree rather than living in the parent because bundle/dms asserts the stamp itself and -# would inherit them. -# -# The stamp as a change the plan reports on its own. It shows up at whatever depth the -# enclosing object sits at, so the indent is matched loosely; the body lines are matched as -# `"key": value` pairs rather than `.*` so the match stops at the entry's own closing brace -# instead of running into its siblings. (Go's regexp is RE2, so the indent cannot be -# captured and back-referenced.) -[[Repls]] -Old = '(?m)^ *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\},\n' -New = '' - -# Same entry when it is the last one in the object, so the comma is on the line before. -[[Repls]] -Old = '(?m),\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n' -New = "\n" - -# When the stamp is the only entry, the whole "changes" object exists because of recording. -# The trailing-comma form comes first: the rule after it would match the same text and -# leave the comma on the previous line dangling. -[[Repls]] -Old = '(?m),\n *"changes": \{\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n *\}\n' -New = "\n" - -[[Repls]] -Old = '(?m)^ *"changes": \{\n *"deployment\.(deployment_id|version_id)": \{\n(?: *"[a-z_]+": .*\n)+ *\}\n *\},?\n' -New = '' - -# The stamp inside a serialized deployment block, as printed by `jobs get` / `pipelines get` -# and in state dumps. Both keys always sit alongside "kind" and "metadata_file_path", so -# each pattern anchors on one of those - that keeps it from matching an unrelated field -# named deployment_id elsewhere in the output. -# -# Order puts these after the root's numeric rules (Order = 10), which have by then turned -# the id into [NUMID]. -[[Repls]] -Old = '(?m)^ *"deployment_id": "\[NUMID\]",\n( *"kind": "BUNDLE")' -New = '${1}' -Order = 20 - -[[Repls]] -# The value is required to be non-empty: a terraform state dump carries -# "version_id": "" for a job it never stamped, and that line is not ours to drop. -Old = '(?m)(^ *"metadata_file_path": [^\n]*),\n *"version_id": "[^"\n]+"\n' -New = "$1\n" -Order = 20 - -# Same two keys in gron.py's flattened form, where each is its own line. -[[Repls]] -Old = '(?m)^json.*\.deployment\.(deployment_id|version_id) = [^\n]*\n' -New = '' -Order = 20 From 1eeb57da321d02365a0425f3695961efda4bb57c Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Sun, 9 Aug 2026 19:21:28 +0000 Subject: [PATCH 058/125] acceptance: cover the emptied-resource delete locally Revoking the last grant empties the grants node, which is recorded as a delete rather than the update that emptied it - the service drops a resource from the deployment only on a delete, and an update left it listed with an id but no state, failing the next plan with "unexpected end of JSON input". Grants are the only resource that can empty out and bundle/resources/grants is Cloud = true, so that fix had no local coverage. Reverting it to an Update makes this test fail. Co-authored-by: Isaac --- .../dms/emptied-resource/databricks.yml | 12 ++ .../bundle/dms/emptied-resource/out.test.toml | 4 + .../bundle/dms/emptied-resource/output.txt | 131 ++++++++++++++++++ acceptance/bundle/dms/emptied-resource/script | 16 +++ 4 files changed, 163 insertions(+) create mode 100644 acceptance/bundle/dms/emptied-resource/databricks.yml create mode 100644 acceptance/bundle/dms/emptied-resource/out.test.toml create mode 100644 acceptance/bundle/dms/emptied-resource/output.txt create mode 100644 acceptance/bundle/dms/emptied-resource/script diff --git a/acceptance/bundle/dms/emptied-resource/databricks.yml b/acceptance/bundle/dms/emptied-resource/databricks.yml new file mode 100644 index 00000000000..9bd148025cb --- /dev/null +++ b/acceptance/bundle/dms/emptied-resource/databricks.yml @@ -0,0 +1,12 @@ +bundle: + name: dms-emptied-resource + +experimental: + record_deployment_history: true + +resources: + schemas: + foo: + name: dms_emptied_resource + catalog_name: main + grants: [{ principal: someone@example.com, privileges: [USE_SCHEMA] }] diff --git a/acceptance/bundle/dms/emptied-resource/out.test.toml b/acceptance/bundle/dms/emptied-resource/out.test.toml new file mode 100644 index 00000000000..2a52887146a --- /dev/null +++ b/acceptance/bundle/dms/emptied-resource/out.test.toml @@ -0,0 +1,4 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/dms/emptied-resource/output.txt b/acceptance/bundle/dms/emptied-resource/output.txt new file mode 100644 index 00000000000..d65afaabd78 --- /dev/null +++ b/acceptance/bundle/dms/emptied-resource/output.txt @@ -0,0 +1,131 @@ + +=== Deploy a schema with one grant, then revoke it so the grants node empties out +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-emptied-resource/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> update_file.py databricks.yml grants: [{ principal: someone@example.com, privileges: [USE_SCHEMA] }] grants: [] + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-emptied-resource/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> print_requests.py //api/2.0/bundle --sort +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments", + "body": { + "initial_parent_path": "/Workspace/Users/[USERNAME]/.bundle/dms-emptied-resource/default/state", + "target_name": "default" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions", + "q": { + "version_id": "1" + }, + "body": { + "cli_version": "[CLI_VERSION]", + "version_type": "VERSION_TYPE_DEPLOY", + "target_name": "default", + "display_name": "dms-emptied-resource", + "workspace_info": { + "file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-emptied-resource/default/files", + "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-emptied-resource/default" + } + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions", + "q": { + "version_id": "2" + }, + "body": { + "cli_version": "[CLI_VERSION]", + "version_type": "VERSION_TYPE_DEPLOY", + "target_name": "default", + "display_name": "dms-emptied-resource", + "previous_version_id": "1", + "workspace_info": { + "file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-emptied-resource/default/files", + "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-emptied-resource/default" + } + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/complete", + "body": { + "completion_reason": "VERSION_COMPLETE_SUCCESS" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations", + "q": { + "resource_key": "schemas.foo" + }, + "body": { + "action_type": "OPERATION_ACTION_TYPE_CREATE", + "resource_id": "main.dms_emptied_resource", + "resource_key": "schemas.foo", + "state": "{\"state\":{\"catalog_name\":\"main\",\"name\":\"dms_emptied_resource\"}}", + "status": "OPERATION_STATUS_SUCCEEDED" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations", + "q": { + "resource_key": "schemas.foo.grants" + }, + "body": { + "action_type": "OPERATION_ACTION_TYPE_CREATE", + "resource_id": "schema/main.dms_emptied_resource", + "resource_key": "schemas.foo.grants", + "state": "{\"state\":{\"securable_type\":\"schema\",\"full_name\":\"main.dms_emptied_resource\",\"__embed__\":[{\"principal\":\"someone@example.com\",\"privileges\":[\"USE_SCHEMA\"]}]},\"depends_on\":[{\"node\":\"resources.schemas.foo\",\"label\":\"${resources.schemas.foo.id}\"}]}", + "status": "OPERATION_STATUS_SUCCEEDED" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/complete", + "body": { + "completion_reason": "VERSION_COMPLETE_SUCCESS" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations", + "q": { + "resource_key": "schemas.foo.grants" + }, + "body": { + "action_type": "OPERATION_ACTION_TYPE_DELETE", + "resource_id": "schema/main.dms_emptied_resource", + "resource_key": "schemas.foo.grants", + "status": "OPERATION_STATUS_SUCCEEDED" + } +} + +=== Plan again: reading state back from the service works and reports no work +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.schemas.foo + +This action will result in the deletion of the following UC schemas. Any underlying data may be lost: + delete resources.schemas.foo + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/dms-emptied-resource/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/dms/emptied-resource/script b/acceptance/bundle/dms/emptied-resource/script new file mode 100644 index 00000000000..b5e8ca666d0 --- /dev/null +++ b/acceptance/bundle/dms/emptied-resource/script @@ -0,0 +1,16 @@ +title "Deploy a schema with one grant, then revoke it so the grants node empties out" +trace $CLI bundle deploy +trace update_file.py databricks.yml 'grants: [{ principal: someone@example.com, privileges: [USE_SCHEMA] }]' 'grants: []' +trace $CLI bundle deploy + +# The emptied node is recorded as a delete, not as the update that emptied it: the service +# drops a resource from the deployment only on a delete. Recorded as an update it stayed +# listed with an id but no state, and reading it back failed the next plan with +# "unexpected end of JSON input". +trace print_requests.py //api/2.0/bundle --sort + +title "Plan again: reading state back from the service works and reports no work" +trace $CLI bundle plan | contains.py "Plan: 0 to add, 0 to change, 0 to delete" "!unexpected end of JSON input" + +trace $CLI bundle destroy --auto-approve +rm -f out.requests.txt From 2015a6277bc2ee4f56c765cf7e77ddd56f63afd0 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Sun, 9 Aug 2026 20:00:44 +0000 Subject: [PATCH 059/125] acceptance: document what nostamp actually transforms Name the three shapes the deployment stamp appears in, with an example of each, and the two cases the filter deliberately leaves alone. The jq is three chained passes and which pass handles what was not readable from the code. Co-authored-by: Isaac --- acceptance/bundle/script.prepare | 36 ++++++++++++++++++++++++-------- 1 file changed, 27 insertions(+), 9 deletions(-) diff --git a/acceptance/bundle/script.prepare b/acceptance/bundle/script.prepare index 340d360e2c8..48fbafac777 100644 --- a/acceptance/bundle/script.prepare +++ b/acceptance/bundle/script.prepare @@ -1,15 +1,33 @@ nostamp() { - # Drop the deployment stamp (deployment_id / version_id) that history recording adds to - # every job and pipeline, so a test asserts the same JSON with recording on or off. Pipe - # a plan, a state dump, or a resource payload through this. See DATABRICKS_BUNDLE_DMS in - # bundle/test.toml; bundle/dms asserts the stamp and does not use this. + # Reads JSON on stdin, writes it back with the deployment stamp removed. # - # Only inside a block that also has "kind" and "metadata_file_path", so an unrelated - # deployment_id survives. An empty value survives too: a terraform state dump carries - # "version_id": "" for a job it never stamped. + # Deployment history recording adds deployment_id and version_id to every job and + # pipeline. Acceptance tests compare output byte for byte, so those two extra fields + # would fail every test in the DATABRICKS_BUNDLE_DMS=true run (see bundle/test.toml). + # Pipe a plan, a state dump, or a resource payload through this and the test asserts one + # golden file either way. Tests under bundle/dms assert the stamp itself and must not. # - # A plan reports the stamp as its own change entry, keyed by field path, so those go as - # well - and with them a "changes" object left empty, which recording alone created. + # Three passes, because the stamp shows up in three shapes: + # + # 1. Nested in a deployment block, as printed by `jobs get` / `pipelines get`: + # "deployment": {"deployment_id": "87..", "kind": "BUNDLE", + # "metadata_file_path": "/x", "version_id": "1"} + # -> both keys dropped, "kind" and "metadata_file_path" kept. + # + # 2. Flat in a plan's "changes", keyed by field path: + # "changes": {"deployment.version_id": {"action": "skip", ...}, "name": {...}} + # -> the stamp entries dropped, real changes kept. + # + # 3. A "changes" object that pass 2 emptied to {} - it existed only because of + # recording, so the key goes too. + # + # Two things it deliberately keeps: + # + # - A deployment_id anywhere else. Pass 1 requires "kind" and "metadata_file_path" as + # neighbours, a pair unique to the deployment block, so an unrelated field of the + # same name is untouched. + # - "version_id": "". A terraform state dump carries that for a job it never stamped, + # and it is the test's own expected output, so `.value == ""` keeps it. jq '((.. | objects | select(has("kind") and has("metadata_file_path"))) |= with_entries(select((.key | IN("deployment_id", "version_id")) == false or .value == ""))) | ((.. | objects | .changes? | objects) From 6beb6037ef7c1528f524bc0664b8da1a2bf902e5 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Sun, 9 Aug 2026 20:12:58 +0000 Subject: [PATCH 060/125] acceptance: show which job is deleted first in the depends-on test Both job ids rendered as [NUMID], so the golden could not show the delete order - the one thing the test exists to demonstrate. Name them first, and skip --sort so the order survives into the output. Co-authored-by: Isaac --- acceptance/bundle/dms/depends-on/output.txt | 10 +++++----- acceptance/bundle/dms/depends-on/script | 9 +++++++++ 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/acceptance/bundle/dms/depends-on/output.txt b/acceptance/bundle/dms/depends-on/output.txt index 952dcb00a86..c45a040dab9 100644 --- a/acceptance/bundle/dms/depends-on/output.txt +++ b/acceptance/bundle/dms/depends-on/output.txt @@ -15,9 +15,9 @@ Deployment complete! }, "body": { "action_type": "OPERATION_ACTION_TYPE_CREATE", - "resource_id": "[NUMID]", + "resource_id": "[CHILD_ID]", "resource_key": "jobs.child", - "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-depends-on/default/state/metadata.json\",\"version_id\":\"1\"},\"description\":\"depends on [NUMID]\",\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"child\",\"queue\":{\"enabled\":true}},\"depends_on\":[{\"node\":\"resources.jobs.parent\",\"label\":\"${resources.jobs.parent.id}\"}]}", + "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-depends-on/default/state/metadata.json\",\"version_id\":\"1\"},\"description\":\"depends on [PARENT_ID]\",\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"child\",\"queue\":{\"enabled\":true}},\"depends_on\":[{\"node\":\"resources.jobs.parent\",\"label\":\"${resources.jobs.parent.id}\"}]}", "status": "OPERATION_STATUS_SUCCEEDED" } } @@ -29,7 +29,7 @@ Deployment complete! }, "body": { "action_type": "OPERATION_ACTION_TYPE_CREATE", - "resource_id": "[NUMID]", + "resource_id": "[PARENT_ID]", "resource_key": "jobs.parent", "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-depends-on/default/state/metadata.json\",\"version_id\":\"1\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"parent\",\"queue\":{\"enabled\":true}}}", "status": "OPERATION_STATUS_SUCCEEDED" @@ -48,5 +48,5 @@ Deleting files... Destroy complete! >>> print_requests.py //jobs --oneline -{"method": "POST", "path": "/api/2.2/jobs/delete", "body": {"job_id": [NUMID]}} -{"method": "POST", "path": "/api/2.2/jobs/delete", "body": {"job_id": [NUMID]}} +{"method": "POST", "path": "/api/2.2/jobs/delete", "body": {"job_id": [CHILD_ID]}} +{"method": "POST", "path": "/api/2.2/jobs/delete", "body": {"job_id": [PARENT_ID]}} diff --git a/acceptance/bundle/dms/depends-on/script b/acceptance/bundle/dms/depends-on/script index be1b9d622f8..95e6a8c2772 100644 --- a/acceptance/bundle/dms/depends-on/script +++ b/acceptance/bundle/dms/depends-on/script @@ -3,6 +3,15 @@ trace $CLI bundle deploy trace print_requests.py //versions/1/operations --sort title "Wipe the local state, then destroy: depends_on comes back from DMS, so the child is still deleted before the parent it references" + +# Name both ids while state still exists, so the delete order below is readable. Without +# this both render as [NUMID] and the golden cannot show which job went first. +read_id.py parent > /dev/null +read_id.py child > /dev/null + rm -rf .databricks trace $CLI bundle destroy --auto-approve + +# Not --sort: the order is the assertion. The child references the parent, so it has to be +# deleted first, and the golden shows [CHILD_ID] before [PARENT_ID]. trace print_requests.py //jobs --oneline From 170649b7ee8105553074b9ad1a35865938c5b5f7 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Sun, 9 Aug 2026 20:39:29 +0000 Subject: [PATCH 061/125] bundle: make the fake service enforce that state implies a resource_id The real service rejects an operation that carries state without a resource_id ("state records a resource that exists"), which is how a malformed failed-recreate record reached a user as a raw 400. The fake accepted it, so no local test could catch that class of bug. Enforce the rule in CreateOperation and UpdateOperation. Also correct what AllowExistingResources is for: the guard trips on any non-empty state, not only the five tests that commit a resources.json, and the DMS run fails 154 tests without it. Co-authored-by: Isaac --- bundle/direct/dstate/state.go | 8 +++++--- libs/testserver/bundle.go | 16 ++++++++++++++++ 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/bundle/direct/dstate/state.go b/bundle/direct/dstate/state.go index 91d0aae453b..5eea5fd6a4a 100644 --- a/bundle/direct/dstate/state.go +++ b/bundle/direct/dstate/state.go @@ -276,9 +276,11 @@ type DMSSource struct { // touch are absent from DMS and a later deploy plans them as creates. // // It exists for the CLI's own acceptance tests, which run the whole bundle suite - // with recording on. Most of those tests seed a state fixture, and they assert the - // output of a single deploy rather than reading state back, so the duplication the - // refusal prevents cannot bite them. + // with recording on. The guard trips on any non-empty state, so it catches every + // test that has already deployed once in its script, not only the five that commit + // a resources.json fixture - without this the DMS run fails 154 of them. They + // assert what one deploy does rather than reading state back, so the duplication + // the refusal prevents cannot bite them. AllowExistingResources bool } diff --git a/libs/testserver/bundle.go b/libs/testserver/bundle.go index 574570ce194..95443a28373 100644 --- a/libs/testserver/bundle.go +++ b/libs/testserver/bundle.go @@ -264,6 +264,14 @@ func (s *FakeWorkspace) CreateOperation(req Request, deploymentID, versionID str return dmsInvalidArgument("error_message is only allowed when status is OPERATION_STATUS_FAILED") } + // State describes a resource that exists, so an operation with state must identify + // which resource via resource_id. This applies to both succeeded and failed + // operations: a failed operation reports prior state to document what existed + // before the attempt failed. + if op.State != nil && op.ResourceId == "" { + return dmsInvalidArgument("resource_id is required for an operation that records state") + } + // The service names operations after the resource key, so it keeps one per // resource per version: creating a second one for the same resource conflicts, // and the caller has to use UpdateOperation instead. @@ -374,6 +382,14 @@ func (s *FakeWorkspace) UpdateOperation(req Request, deploymentID, versionID, re return dmsInvalidArgument("error_message is only allowed when status is OPERATION_STATUS_FAILED") } + // State describes a resource that exists, so an operation with state must identify + // which resource via resource_id. This applies to both succeeded and failed + // operations: a failed operation reports prior state to document what existed + // before the attempt failed. + if op.State != nil && op.ResourceId == "" { + return dmsInvalidArgument("resource_id is required for an operation that records state") + } + // Only the mutable fields change; action_type and resource_key stay as created. existing.State = op.State existing.ErrorMessage = op.ErrorMessage From d701cd9580b81cc8f4bbb057a01ea61b2c141966 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Sun, 9 Aug 2026 21:52:57 +0000 Subject: [PATCH 062/125] bundle: record a recreate's intermediate delete as in-progress A recreate is a delete followed by a create, and the service keeps one operation per resource per version with action_type fixed at creation. So the CLI skipped the delete and let the save that followed report the whole recreate - which meant a recreate whose create half failed was recorded as the pre-delete resource with a FAILED status: honest about failing, wrong about what exists, since that resource is already gone. The delete now opens the operation as IN_PROGRESS carrying no state, and the save patches it to SUCCEEDED. A deploy that stops in between leaves the resource described as mid-recreate. The status constant is declared locally: it is generated from the OpenAPI spec, which trails the service proto (databricks-eng/universe#2394529). Also fixes the fake server's UpdateOperation, which unmarshalled the request into the SDK struct and so choked on the string sequence_id it had just sent itself - this only surfaced now because a recreate is the first flow that patches an operation. Four request captures filter on a path that a DMS operation URL also contains ("/jobs", "/pipelines", "/apps"), so they now exclude /api/2.0/bundle. Co-authored-by: Isaac --- .../bundle/dms/partial-update/output.txt | 17 ++++++++++++++--- acceptance/bundle/dms/partial-update/script | 7 ++++--- .../pipelines_recreate/output.txt | 8 ++++---- .../resource_deps/pipelines_recreate/script | 8 ++++---- acceptance/bundle/resources/apps/update/script | 2 +- .../permissions/pipelines/update/output.txt | 10 +++++----- .../permissions/pipelines/update/script | 12 ++++++------ .../resources/pipelines/recreate-keys/_script | 4 +++- bundle/direct/dstate/state.go | 12 +++++------- bundle/direct/dstate/state_test.go | 8 +++++--- bundle/direct/opqueue.go | 8 ++++++++ bundle/direct/oprecorder.go | 18 ++++++++++++++++++ libs/testserver/bundle.go | 17 +++++++++++++---- 13 files changed, 90 insertions(+), 41 deletions(-) diff --git a/acceptance/bundle/dms/partial-update/output.txt b/acceptance/bundle/dms/partial-update/output.txt index f6984c72258..0d65dc2fd06 100644 --- a/acceptance/bundle/dms/partial-update/output.txt +++ b/acceptance/bundle/dms/partial-update/output.txt @@ -92,11 +92,22 @@ Deployment complete! "resource_key": "schemas.foo" }, "body": { - "action_type": "OPERATION_ACTION_TYPE_CREATE", - "resource_id": "other.dms_partial_update_schema", + "action_type": "OPERATION_ACTION_TYPE_RECREATE", "resource_key": "schemas.foo", + "status": "OPERATION_STATUS_IN_PROGRESS" + } +} +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations/schemas.foo", + "q": { + "update_mask": "state,error_message,resource_id,status" + }, + "body": { "state": "{\"state\":{\"catalog_name\":\"other\",\"comment\":\"v1\",\"name\":\"dms_partial_update_schema\"}}", - "status": "OPERATION_STATUS_SUCCEEDED" + "resource_id": "other.dms_partial_update_schema", + "status": "OPERATION_STATUS_SUCCEEDED", + "sequence_id": "1" } } { diff --git a/acceptance/bundle/dms/partial-update/script b/acceptance/bundle/dms/partial-update/script index 9310ded83aa..38c3f66c756 100644 --- a/acceptance/bundle/dms/partial-update/script +++ b/acceptance/bundle/dms/partial-update/script @@ -3,9 +3,10 @@ trace $CLI bundle deploy trace print_requests.py //api/2.0/bundle title "Recreate writes state twice - the entry is dropped, then the new resource is saved" -# Only the save is recorded. The service keeps one operation per resource per version -# and rejects a succeeded recreate that carries no state, so the intermediate drop -# cannot be its own event; the save that follows reports the recreate instead. +# The service keeps one operation per resource per version, so both writes land on the +# same one: the drop opens it as IN_PROGRESS with no state, and the save that follows +# patches it to SUCCEEDED. A deploy that dies in between therefore leaves the resource +# described as mid-recreate rather than as the resource it already deleted. trace update_file.py databricks.yml "catalog_name: main" "catalog_name: other" trace $CLI bundle deploy --auto-approve trace print_requests.py //api/2.0/bundle diff --git a/acceptance/bundle/resource_deps/pipelines_recreate/output.txt b/acceptance/bundle/resource_deps/pipelines_recreate/output.txt index f5d8d159b31..dc088df16bf 100644 --- a/acceptance/bundle/resource_deps/pipelines_recreate/output.txt +++ b/acceptance/bundle/resource_deps/pipelines_recreate/output.txt @@ -11,7 +11,7 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> print_requests.py --nostamp //jobs //pipelines +>>> print_requests.py --nostamp //jobs //pipelines ^//api/2.0/bundle >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged @@ -37,7 +37,7 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> print_requests.py --nostamp //jobs //pipelines +>>> print_requests.py --nostamp //jobs //pipelines ^//api/2.0/bundle === Fetch resource IDs and verify remote state >>> musterr [CLI] pipelines get [FOO_ID] @@ -100,7 +100,7 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> print_requests.py --nostamp //jobs //pipelines +>>> print_requests.py --nostamp //jobs //pipelines ^//api/2.0/bundle >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: @@ -116,7 +116,7 @@ All files and directories at the following location will be deleted: /Workspace/ Deleting files... Destroy complete! ->>> print_requests.py --nostamp //jobs //pipelines +>>> print_requests.py --nostamp //jobs //pipelines ^//api/2.0/bundle { "method": "POST", "path": "/api/2.2/jobs/delete", diff --git a/acceptance/bundle/resource_deps/pipelines_recreate/script b/acceptance/bundle/resource_deps/pipelines_recreate/script index 9d31e809254..691e24279b2 100644 --- a/acceptance/bundle/resource_deps/pipelines_recreate/script +++ b/acceptance/bundle/resource_deps/pipelines_recreate/script @@ -3,7 +3,7 @@ echo "*" > .gitignore trace $CLI bundle plan $CLI bundle plan -o json | nostamp > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy -trace print_requests.py --nostamp //jobs //pipelines > out.create.requests.json +trace print_requests.py --nostamp //jobs //pipelines '^//api/2.0/bundle' > out.create.requests.json foo_id=`read_id.py foo` @@ -16,7 +16,7 @@ trace update_file.py databricks.yml "storage: dbfs:/my-storage" "storage: dbfs:/ trace $CLI bundle plan $CLI bundle plan -o json | nostamp > out.plan_update.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy --auto-approve -trace print_requests.py --nostamp //jobs //pipelines > out.update.requests.$DATABRICKS_BUNDLE_ENGINE.json +trace print_requests.py --nostamp //jobs //pipelines '^//api/2.0/bundle' > out.update.requests.$DATABRICKS_BUNDLE_ENGINE.json foo_id_2=`read_id.py foo` @@ -31,10 +31,10 @@ title "Follow up plan & deploy do nothing" trace $CLI bundle plan $CLI bundle plan -o json | nostamp > out.plan_noop.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy -trace print_requests.py --nostamp //jobs //pipelines +trace print_requests.py --nostamp //jobs //pipelines '^//api/2.0/bundle' trace $CLI bundle destroy --auto-approve -trace print_requests.py --nostamp //jobs //pipelines +trace print_requests.py --nostamp //jobs //pipelines '^//api/2.0/bundle' trace musterr $CLI pipelines get $foo_id trace musterr $CLI pipelines get $foo_id_2 diff --git a/acceptance/bundle/resources/apps/update/script b/acceptance/bundle/resources/apps/update/script index 0d988c0d4a2..1b9faa07547 100644 --- a/acceptance/bundle/resources/apps/update/script +++ b/acceptance/bundle/resources/apps/update/script @@ -1,6 +1,6 @@ print_requests() { # url is output-only field that terraform adds but that is ignored by the backend - print_requests.py //apps --del-body url + print_requests.py //apps '^//api/2.0/bundle' --del-body url } trace $CLI bundle plan diff --git a/acceptance/bundle/resources/permissions/pipelines/update/output.txt b/acceptance/bundle/resources/permissions/pipelines/update/output.txt index 6394c59b40b..5b2916f52a4 100644 --- a/acceptance/bundle/resources/permissions/pipelines/update/output.txt +++ b/acceptance/bundle/resources/permissions/pipelines/update/output.txt @@ -7,7 +7,7 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> print_requests.py --nostamp //pipeline +>>> print_requests.py --nostamp //pipeline ^//api/2.0/bundle >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged @@ -59,7 +59,7 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> print_requests.py --nostamp //pipeline +>>> print_requests.py --nostamp //pipeline ^//api/2.0/bundle >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged @@ -74,7 +74,7 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> print_requests.py --nostamp //pipeline +>>> print_requests.py --nostamp //pipeline ^//api/2.0/bundle >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged @@ -98,7 +98,7 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> print_requests.py --nostamp //pipeline --sort +>>> print_requests.py --nostamp //pipeline ^//api/2.0/bundle --sort >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged @@ -113,7 +113,7 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> print_requests.py --nostamp //pipeline +>>> print_requests.py --nostamp //pipeline ^//api/2.0/bundle >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged diff --git a/acceptance/bundle/resources/permissions/pipelines/update/script b/acceptance/bundle/resources/permissions/pipelines/update/script index b3e9d1ca539..9ace44addb4 100644 --- a/acceptance/bundle/resources/permissions/pipelines/update/script +++ b/acceptance/bundle/resources/permissions/pipelines/update/script @@ -1,7 +1,7 @@ cp databricks.yml databricks.yml.saved trace $CLI bundle plan -o json | nostamp > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy -trace print_requests.py --nostamp //pipeline > out.requests_create.json +trace print_requests.py --nostamp //pipeline '^//api/2.0/bundle' > out.requests_create.json trace $CLI bundle plan pipeline_id="$(read_id.py foo)" @@ -13,14 +13,14 @@ title "Update one permission and deploy again\n" update_file.py databricks.yml CAN_VIEW CAN_MANAGE trace $CLI bundle plan -o json | nostamp > out.plan_update.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy -trace print_requests.py --nostamp //pipeline > out.requests_update.json +trace print_requests.py --nostamp //pipeline '^//api/2.0/bundle' > out.requests_update.json trace $CLI bundle plan title "Delete one permission and deploy again\n" grep -v DELETE_ONE databricks.yml > tmp.yml && mv tmp.yml databricks.yml trace $CLI bundle plan -o json | nostamp > out.plan_delete_one.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy -trace print_requests.py --nostamp //pipeline > out.requests_delete_one.json +trace print_requests.py --nostamp //pipeline '^//api/2.0/bundle' > out.requests_delete_one.json trace $CLI bundle plan title "Delete the whole block and deploy again\n" @@ -28,16 +28,16 @@ grep -v PERMISSIONS databricks.yml > tmp.yml && mv tmp.yml databricks.yml trace cat databricks.yml trace $CLI bundle plan -o json | nostamp > out.plan_delete_all.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy -trace print_requests.py --nostamp //pipeline --sort > out.requests_delete_all.json +trace print_requests.py --nostamp //pipeline '^//api/2.0/bundle' --sort > out.requests_delete_all.json trace $CLI bundle plan title "Restore original config\n" mv databricks.yml.saved databricks.yml trace $CLI bundle plan -o json | nostamp > out.plan_restore.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy -trace print_requests.py --nostamp //pipeline > out.requests_restore_original.json +trace print_requests.py --nostamp //pipeline '^//api/2.0/bundle' > out.requests_restore_original.json trace $CLI bundle plan trace $CLI bundle destroy --auto-approve -print_requests.py --nostamp //pipeline > out.requests_destroy.json +print_requests.py --nostamp //pipeline '^//api/2.0/bundle' > out.requests_destroy.json rm -f out.requests.txt diff --git a/acceptance/bundle/resources/pipelines/recreate-keys/_script b/acceptance/bundle/resources/pipelines/recreate-keys/_script index 2ab649947da..45c53d86743 100644 --- a/acceptance/bundle/resources/pipelines/recreate-keys/_script +++ b/acceptance/bundle/resources/pipelines/recreate-keys/_script @@ -8,7 +8,9 @@ trace $CLI bundle deploy ppid1=`read_id.py my` print_requests() { - jq --sort-keys 'select(.method != "GET" and (.path | contains("/pipelines")))' < out.requests.txt | nostamp + # Excludes /api/2.0/bundle: a DMS operation path embeds the resource key, so it also + # contains "/pipelines". + jq --sort-keys 'select(.method != "GET" and (.path | contains("/pipelines")) and (.path | contains("/api/2.0/bundle") | not))' < out.requests.txt | nostamp rm -f out.requests.txt } diff --git a/bundle/direct/dstate/state.go b/bundle/direct/dstate/state.go index 5eea5fd6a4a..fc11aee7f50 100644 --- a/bundle/direct/dstate/state.go +++ b/bundle/direct/dstate/state.go @@ -195,13 +195,11 @@ func (db *DeploymentState) DeleteState(ctx context.Context, key string, action d // State is nil: the resource no longer exists. // - // A recreate is the exception. It drops the entry and then saves the new - // resource, but the service keeps one operation per resource per version whose - // action_type is fixed at creation, and it rejects a succeeded recreate that - // carries no state ("it leaves a resource that exists"). So the intermediate drop - // cannot be recorded as its own event; the save that follows reports the recreate, - // and if that save never happens the failure path reports it instead. - if db.sink != nil && action != deployplan.Recreate { + // A recreate drops the entry and then saves the replacement. The service keeps one + // operation per resource per version, so the sink opens that one operation as + // in-progress here and the save that follows completes it - leaving an interrupted + // recreate described as mid-recreate rather than as the resource it deleted. + if db.sink != nil { db.sink.RecordOperation(ctx, key, action, deletedID, nil) } diff --git a/bundle/direct/dstate/state_test.go b/bundle/direct/dstate/state_test.go index 9f0374c0ec4..7494f662f96 100644 --- a/bundle/direct/dstate/state_test.go +++ b/bundle/direct/dstate/state_test.go @@ -42,11 +42,13 @@ func TestStateWritesRecordOperations(t *testing.T) { require.NoError(t, db.SaveState(t.Context(), "jobs.my_job", "456", map[string]string{"key": "new"}, nil, deployplan.Recreate)) mustFinalize(t, &db) - // The recreate's intermediate drop is not reported: the service keeps one - // operation per resource per version and rejects a succeeded recreate carrying no - // state, so the save that follows is what reports it. + // Both of the recreate's writes are reported. The service keeps one operation per + // resource per version, so the drop opens it (no state: the old resource is gone and + // the new one does not exist yet) and the save completes it. A deploy that stops in + // between leaves the resource described as mid-recreate. assert.Equal(t, []string{ `create jobs.my_job id=123 state={"state":{"key":"old"}}`, + `recreate jobs.my_job id=123 state=`, `recreate jobs.my_job id=456 state={"state":{"key":"new"}}`, }, sink.ops) } diff --git a/bundle/direct/opqueue.go b/bundle/direct/opqueue.go index ea796abf84d..bb6c028e07f 100644 --- a/bundle/direct/opqueue.go +++ b/bundle/direct/opqueue.go @@ -88,6 +88,14 @@ func (q *operationQueue) RecordOperation(ctx context.Context, resourceKey string return } + // A recreate drops the state entry before creating the replacement. Recorded as + // in-progress, so an interrupted recreate does not leave the resource described as + // the one it just deleted; the create that follows updates it to succeeded. + if action == deployplan.Recreate && state == nil { + q.enqueue(ctx, resourceKey, newInProgressOperation()) + return + } + op, err := newStateOperation(action, resourceID, state) if err != nil { // The deploy already persisted this write locally, so failing it here would diff --git a/bundle/direct/oprecorder.go b/bundle/direct/oprecorder.go index 26afccd8f2f..08caeceea4b 100644 --- a/bundle/direct/oprecorder.go +++ b/bundle/direct/oprecorder.go @@ -24,6 +24,14 @@ const maxOperationStateSize = 64 * 1024 // failing and masking the error we are trying to report. const maxOperationErrorMessageSize = 16 * 1024 +// operationStatusInProgress marks an operation whose writes are not finished. A +// recreate opens one after its delete and updates it to succeeded once the create +// lands, so an interrupted recreate is not left describing the resource it deleted. +// +// Declared here rather than used from the SDK: the enum value is generated from the +// OpenAPI spec, which trails the service proto (databricks-eng/universe#2394529). +const operationStatusInProgress bundledeployments.OperationStatus = "OPERATION_STATUS_IN_PROGRESS" + // recordedOperation is an applied resource operation, serialized and waiting to be // uploaded to the deployment metadata service (DMS). // @@ -66,6 +74,16 @@ func newStateOperation(action deployplan.ActionType, resourceID string, state js }, nil } +// newInProgressOperation opens a recreate before its create half has run. It carries +// no state: the old resource is deleted and the new one does not exist yet, so there +// is nothing that exists to describe. +func newInProgressOperation() recordedOperation { + return recordedOperation{ + action: bundledeployments.OperationActionTypeOperationActionTypeRecreate, + status: operationStatusInProgress, + } +} + // newFailedOperation records an operation that did not apply, so the deployment // history says why a resource failed rather than just omitting it. // diff --git a/libs/testserver/bundle.go b/libs/testserver/bundle.go index 95443a28373..2bc071f8a7f 100644 --- a/libs/testserver/bundle.go +++ b/libs/testserver/bundle.go @@ -343,10 +343,10 @@ func operationBody(op *bundledeployments.Operation) (map[string]any, error) { // version. sequence_id is the concurrency precondition and increments on success. func (s *FakeWorkspace) UpdateOperation(req Request, deploymentID, versionID, resourceKey string) Response { // sequence_id arrives as a string, which the SDK struct cannot hold (it types the - // field int64), so read the body twice: once for the typed fields and once for the - // precondition. - var op bundledeployments.Operation - if err := json.Unmarshal(req.Body, &op); err != nil { + // field int64), so read the body twice: once for the typed fields with that key + // removed, and once for the precondition alone. + var raw map[string]json.RawMessage + if err := json.Unmarshal(req.Body, &raw); err != nil { return Response{StatusCode: 400, Body: map[string]string{"message": err.Error()}} } var precondition struct { @@ -355,6 +355,15 @@ func (s *FakeWorkspace) UpdateOperation(req Request, deploymentID, versionID, re if err := json.Unmarshal(req.Body, &precondition); err != nil { return Response{StatusCode: 400, Body: map[string]string{"message": err.Error()}} } + delete(raw, "sequence_id") + typedBody, err := json.Marshal(raw) + if err != nil { + return Response{StatusCode: 500, Body: map[string]string{"message": err.Error()}} + } + var op bundledeployments.Operation + if err := json.Unmarshal(typedBody, &op); err != nil { + return Response{StatusCode: 400, Body: map[string]string{"message": err.Error()}} + } updateMask := req.URL.Query().Get("update_mask") if updateMask == "" { From ff6b36448d5715871ba12be2a646c8a7e17c8c8e Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Sun, 9 Aug 2026 23:50:20 +0000 Subject: [PATCH 063/125] bundle: cover the delete-order reversal in the dependency graph makeGraph reverses edge direction for a delete so a child is removed before the parent it references. Nothing tested that: graph.go had no test file, and bundle/dms/depends-on only prints the delete order into its golden without asserting it, so dropping depends_on on the read path left that test passing. Removing the reversal now fails all four cases. Co-authored-by: Isaac --- bundle/direct/graph_test.go | 78 +++++++++++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100644 bundle/direct/graph_test.go diff --git a/bundle/direct/graph_test.go b/bundle/direct/graph_test.go new file mode 100644 index 00000000000..a99082407e4 --- /dev/null +++ b/bundle/direct/graph_test.go @@ -0,0 +1,78 @@ +package direct + +import ( + "sync" + "testing" + + "github.com/databricks/cli/bundle/deployplan" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// runOrder returns the order makeGraph's edges let the nodes run in. +func runOrder(t *testing.T, plan *deployplan.Plan) []string { + g, err := makeGraph(plan) + require.NoError(t, err) + require.NoError(t, g.DetectCycle()) + + var mu sync.Mutex + var order []string + g.Run(1, func(node string, failedDependency *string) bool { + mu.Lock() + defer mu.Unlock() + order = append(order, node) + return true + }) + return order +} + +func TestMakeGraphOrdersDependencyBeforeDependent(t *testing.T) { + plan := &deployplan.Plan{Plan: map[string]*deployplan.PlanEntry{ + "resources.jobs.parent": {Action: "create"}, + "resources.jobs.child": { + Action: "create", + DependsOn: []deployplan.DependsOnEntry{{Node: "resources.jobs.parent"}}, + }, + }} + + assert.Equal(t, []string{"resources.jobs.parent", "resources.jobs.child"}, runOrder(t, plan)) +} + +func TestMakeGraphReversesOrderForDelete(t *testing.T) { + // A delete has to run the other way round: the child refers to the parent, so + // removing the parent first would leave the child pointing at nothing. + plan := &deployplan.Plan{Plan: map[string]*deployplan.PlanEntry{ + "resources.jobs.parent": {Action: "delete"}, + "resources.jobs.child": { + Action: "delete", + DependsOn: []deployplan.DependsOnEntry{{Node: "resources.jobs.parent"}}, + }, + }} + + assert.Equal(t, []string{"resources.jobs.child", "resources.jobs.parent"}, runOrder(t, plan)) +} + +func TestMakeGraphRejectsUnknownDependency(t *testing.T) { + plan := &deployplan.Plan{Plan: map[string]*deployplan.PlanEntry{ + "resources.jobs.child": { + Action: "create", + DependsOn: []deployplan.DependsOnEntry{{Node: "resources.jobs.missing", Label: "${resources.jobs.missing.id}"}}, + }, + }} + + _, err := makeGraph(plan) + assert.ErrorContains(t, err, `no such node "resources.jobs.missing"`) +} + +func TestMakeGraphIgnoresUnknownDependencyOnDelete(t *testing.T) { + // A destroy plans only what state tracks, so a dependency on something already + // gone is expected rather than an error. + plan := &deployplan.Plan{Plan: map[string]*deployplan.PlanEntry{ + "resources.jobs.child": { + Action: "delete", + DependsOn: []deployplan.DependsOnEntry{{Node: "resources.jobs.missing"}}, + }, + }} + + assert.Equal(t, []string{"resources.jobs.child"}, runOrder(t, plan)) +} From 125d73b3ad6b9d22ecd272cc6d235691469b82d6 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 10 Aug 2026 00:35:02 +0000 Subject: [PATCH 064/125] bundle: single-source the DMS resource-key prefix, fix two stale comments Review follow-ups from comparing the fake service against the real one. The "resources." prefix was a literal in both the write path (stripped) and the read path (re-added). Now one exported constant, since a change to one side alone would silently file operations under keys nothing reads. deploy.go said the version "is not created until the plan is approved". It is created before the plan is computed, which is why a cancelled deploy leaves a version behind - the comment 60 lines below says so, and dogfood confirms it. The fake does not model the deployment lock that the real service takes when a version is created. That is now written down where someone looking for it would look, along with why refusing on an in-flight version is not a valid stand-in: several tests kill the CLI mid-apply, leaving a version in progress that the real service would release once the lease expired. Co-authored-by: Isaac --- bundle/direct/dstate/dms.go | 8 +++++++- bundle/direct/oprecorder.go | 6 ++---- bundle/phases/deploy.go | 10 +++++----- libs/testserver/bundle.go | 8 ++++++++ 4 files changed, 22 insertions(+), 10 deletions(-) diff --git a/bundle/direct/dstate/dms.go b/bundle/direct/dstate/dms.go index a96e864c761..07e879973f2 100644 --- a/bundle/direct/dstate/dms.go +++ b/bundle/direct/dstate/dms.go @@ -9,6 +9,12 @@ import ( "github.com/databricks/databricks-sdk-go/service/bundledeployments" ) +// ResourceKeyPrefix is what a state key carries and a DMS resource key does not: +// state calls a job "resources.jobs.foo", DMS calls it "jobs.foo". Stripped on the +// way out and re-added on the way back, so both sides must use this one constant or +// operations silently land under keys nothing reads. +const ResourceKeyPrefix = "resources." + // RecordedState is what the CLI serializes into the DMS Operation.State field. It // wraps the config rather than being it, so depends_on survives the round trip: DMS // has no field for dependency edges, and they cannot be recomputed once references @@ -64,7 +70,7 @@ func fetchDeploymentResources(ctx context.Context, client bundledeployments.Bund // DMS reports resource keys without the "resources." prefix (e.g. // "jobs.foo"), but the state DB keys are fully qualified // ("resources.jobs.foo"), so prepend it here. - key := "resources." + res.ResourceKey + key := ResourceKeyPrefix + res.ResourceKey var recorded RecordedState if res.State != nil { diff --git a/bundle/direct/oprecorder.go b/bundle/direct/oprecorder.go index 08caeceea4b..1bcdc1c9aa1 100644 --- a/bundle/direct/oprecorder.go +++ b/bundle/direct/oprecorder.go @@ -181,10 +181,8 @@ func newOperationRecorder(ops operationClient, deploymentID string, version int6 var updatableFields = []string{"state", "error_message", "resource_id", "status"} func (r *operationRecorder) upload(ctx context.Context, resourceKey string, op recordedOperation) error { - // DMS resource keys are unprefixed (e.g. "jobs.foo"), while the CLI's state - // keys carry a leading "resources." (e.g. "resources.jobs.foo"). Strip it on - // the way out; the read path re-adds it (see dstate.fetchDeploymentResources). - dmsKey := strings.TrimPrefix(resourceKey, "resources.") + // The read path re-adds the prefix; see dstate.ResourceKeyPrefix. + dmsKey := strings.TrimPrefix(resourceKey, dstate.ResourceKeyPrefix) operation := bundledeployments.Operation{ ActionType: op.action, diff --git a/bundle/phases/deploy.go b/bundle/phases/deploy.go index d0faca36bca..96f7a0e5251 100644 --- a/bundle/phases/deploy.go +++ b/bundle/phases/deploy.go @@ -167,11 +167,11 @@ func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHand // lock is acquired here // - // Set up DMS recording of this deployment as a version. The version is not - // created until the plan is approved (below), so a cancelled deploy records - // nothing; the deferred CompleteVersion is a no-op until CreateVersion runs. - // CompleteVersion is deferred before lock.Release so it runs while the lock - // is still held (defers run last-in-first-out). + // Set up DMS recording of this deployment as a version. The version itself is + // created further down, before the plan is computed - see the comment there for + // why it cannot wait for approval. CompleteVersion is deferred before + // lock.Release so it runs while the lock is still held (defers run + // last-in-first-out), and is a no-op until CreateVersion has run. recorder, err := newDeploymentRecorder(ctx, b, stateEngine, dms.VersionTypeDeploy) if err != nil { logdiag.LogError(ctx, err) diff --git a/libs/testserver/bundle.go b/libs/testserver/bundle.go index 2bc071f8a7f..e3e8a2428ba 100644 --- a/libs/testserver/bundle.go +++ b/libs/testserver/bundle.go @@ -185,6 +185,14 @@ func (s *FakeWorkspace) CreateVersion(req Request, deploymentID string) Response return dmsAborted("previous_version_id is outdated; the deployment's most recent version is " + d.deployment.LastVersionId) } + // Not modelled: the deployment lock. Creating a version takes one on the real + // service (VersionStorage.LOCK_DURATION_MS), which refuses a second deploy with + // "deployment N is locked by version M (lock expires at ...)" until a two-minute + // lease elapses without a heartbeat. Refusing purely on an in-flight version is + // wrong here: several tests kill the CLI mid-apply, which leaves the version + // in-progress forever, where the real service would let the lease expire. Modelling + // it needs the lease clock, and Version carries no heartbeat field to hang it on. + // bundle_root_path is relative to git_folder_path, so the service rejects a // workspace_info that carries one without the other. if ws := version.WorkspaceInfo; ws != nil && (ws.GitFolderPath == "") != (ws.BundleRootPath == "") { From 5f211eff41a8da35a17a3d9a0705febbe4265a4f Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 10 Aug 2026 09:04:13 +0000 Subject: [PATCH 065/125] bundle: record a declined deploy as aborted, not as a failure Answering "n" to the destructive-change prompt left a version completed as VERSION_COMPLETE_FAILURE, so the history read as if a deploy had run and broken. It now completes as VERSION_COMPLETE_FORCE_ABORT, which says nothing was applied. The version still has to exist by then: it is stamped onto the resources the plan is computed from, and the prompt comes after the plan. Both non-approval outcomes take this path - the user declining, and a console that cannot prompt, which returns an error rather than false. The second is what a non-interactive caller hits, and it was previously recorded as a failure too. Verified on dogfood: the version now reports FORCE_ABORT where it reported FAILURE. Co-authored-by: Isaac --- bundle/phases/deploy.go | 41 ++++++++++++++++++++++++++++------------- libs/dms/recorder.go | 23 +++++++++++++++++------ 2 files changed, 45 insertions(+), 19 deletions(-) diff --git a/bundle/phases/deploy.go b/bundle/phases/deploy.go index 96f7a0e5251..006ed90f12b 100644 --- a/bundle/phases/deploy.go +++ b/bundle/phases/deploy.go @@ -233,8 +233,12 @@ func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHand // Create the version before planning: the plan snapshots the resource config, so // the version has to be stamped on before it is computed or the applied resources - // would not carry it. A cancelled deploy therefore leaves a version behind, - // completed as a failure by the deferred CompleteVersion. + // would not carry it. + // + // Creating it is also what takes the deployment's lock server-side, so a deploy + // that loses a race pays for the upload above before being turned away. Moving it + // earlier does not work: on a first deploy the deployment record is registered + // under the state directory, which the upload is what creates. if err := recorder.CreateVersion(ctx); err != nil { logdiag.LogError(ctx, err) return @@ -288,21 +292,32 @@ func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHand return } - haveApproval, err := approvalForDeploy(ctx, b, plan) - if err != nil { - logdiag.LogError(ctx, err) - return - } - if haveApproval { - // Record operations under the version created before planning, so DMS holds - // the deployed resource state. - setOperationRecorder(ctx, b, recorder) - deployCore(ctx, b, plan, stateEngine, requestedEngine, recorder) - } else { + haveApproval, approvalErr := approvalForDeploy(ctx, b, plan) + if !haveApproval { + // Nothing was applied, so the version records an abort rather than a failure. + // It cannot simply be left uncreated: it is stamped onto the resources the plan + // is computed from, so it has to exist before the prompt. Aborting first also + // makes the deferred CompleteVersion a no-op. + // + // Both outcomes land here - the user declining, and a console that cannot + // prompt at all, which returns an error instead. + if err := recorder.AbortVersion(ctx); err != nil { + logdiag.LogError(ctx, err) + return + } + if approvalErr != nil { + logdiag.LogError(ctx, approvalErr) + return + } cmdio.LogString(ctx, "Deployment cancelled!") return } + // Record operations under the version created before planning, so DMS holds + // the deployed resource state. + setOperationRecorder(ctx, b, recorder) + deployCore(ctx, b, plan, stateEngine, requestedEngine, recorder) + if logdiag.HasError(ctx) { return } diff --git a/libs/dms/recorder.go b/libs/dms/recorder.go index 001126f4fb6..ff195679c95 100644 --- a/libs/dms/recorder.go +++ b/libs/dms/recorder.go @@ -188,6 +188,22 @@ func (r *Recorder) CreateVersion(ctx context.Context) error { // when CreateVersion never ran, which is what lets callers defer it and still not // complete a version a cancelled deploy never created. func (r *Recorder) CompleteVersion(ctx context.Context, success bool) error { + reason := bundledeployments.VersionCompleteVersionCompleteSuccess + if !success { + reason = bundledeployments.VersionCompleteVersionCompleteFailure + } + return r.completeVersion(ctx, reason) +} + +// AbortVersion completes the version as aborted, for a deploy the user declined at +// the approval prompt. The version has to exist by then - it is stamped onto the +// resources the plan is computed from - so this says nothing was applied rather than +// leaving a version that reads like a deploy that failed. +func (r *Recorder) AbortVersion(ctx context.Context) error { + return r.completeVersion(ctx, bundledeployments.VersionCompleteVersionCompleteForceAbort) +} + +func (r *Recorder) completeVersion(ctx context.Context, reason bundledeployments.VersionComplete) error { if r == nil || r.versionNum == 0 || r.completed { return nil } @@ -198,11 +214,6 @@ func (r *Recorder) CompleteVersion(ctx context.Context, success bool) error { versionIDStr := strconv.FormatInt(r.versionNum, 10) versionName := fmt.Sprintf("deployments/%s/versions/%s", r.deploymentID, versionIDStr) - reason := bundledeployments.VersionCompleteVersionCompleteSuccess - if !success { - reason = bundledeployments.VersionCompleteVersionCompleteFailure - } - _, err := r.svc.CompleteVersion(ctx, bundledeployments.CompleteVersionRequest{ Name: versionName, CompletionReason: reason, @@ -214,7 +225,7 @@ func (r *Recorder) CompleteVersion(ctx context.Context, success bool) error { // For destroy operations, delete the deployment record after the version // completes successfully. - if success && r.versionType == VersionTypeDestroy { + if reason == bundledeployments.VersionCompleteVersionCompleteSuccess && r.versionType == VersionTypeDestroy { err = r.svc.DeleteDeployment(ctx, bundledeployments.DeleteDeploymentRequest{ Name: "deployments/" + r.deploymentID, }) From f6c564facef7b4b9ffa9a15f2797c55ce326c48e Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 10 Aug 2026 10:20:59 +0000 Subject: [PATCH 066/125] bundle: drop the empty-resource-id guard, which cannot happen A listed resource always has an id. The service projects one only when the operation carries state (OperationStorage buffers a delete otherwise), and an operation carrying state must have a resource_id (OperationInvariants). So the read path's skip-if-no-id branch was unreachable, and the test asserting it described a response the service does not produce. The fake server had the same wrong model: it upserted a resource for a stateless operation and its comment claimed a failed create is listed with an empty resource_id. It now removes the resource, like the service. bundle/dms/record-failure covered that shape too - its golden showed a resource with no id - and now asserts the resource is absent, with the redeploy still planning a create. Co-authored-by: Isaac --- .../bundle/dms/record-failure/output.txt | 14 ++------------ acceptance/bundle/dms/record-failure/script | 2 +- bundle/direct/dstate/dms.go | 9 --------- bundle/direct/dstate/dms_test.go | 17 ----------------- libs/testserver/bundle.go | 19 ++++++++----------- 5 files changed, 11 insertions(+), 50 deletions(-) diff --git a/acceptance/bundle/dms/record-failure/output.txt b/acceptance/bundle/dms/record-failure/output.txt index cb4073087d7..d1a6eaa1faf 100644 --- a/acceptance/bundle/dms/record-failure/output.txt +++ b/acceptance/bundle/dms/record-failure/output.txt @@ -58,19 +58,9 @@ API message: cluster spec is invalid } } -=== The failed resource is listed without a resource_id or state, so a later deploy plans to create it rather than treating it as already deployed +=== The failed resource is not listed at all: state is what projects a resource, and a failed create records none, so a later deploy plans to create it rather than treating it as already deployed >>> [CLI] api get /api/2.0/bundle/deployments/[NUMID]/resources -{ - "resources": [ - { - "last_action_type": "OPERATION_ACTION_TYPE_CREATE", - "last_version_id": "1", - "name": "deployments/[NUMID]/resources/jobs.doomed", - "resource_key": "jobs.doomed", - "resource_type": "" - } - ] -} +{} === Redeploying from DMS state alone still creates the failed resource: it was never applied, so it must not be skipped as unchanged >>> [CLI] bundle plan diff --git a/acceptance/bundle/dms/record-failure/script b/acceptance/bundle/dms/record-failure/script index 66af9601402..7e9fa3d47c2 100644 --- a/acceptance/bundle/dms/record-failure/script +++ b/acceptance/bundle/dms/record-failure/script @@ -2,7 +2,7 @@ title "A resource that fails to apply is recorded as a failed operation carrying trace musterr $CLI bundle deploy trace print_requests.py //api/2.0/bundle --sort -title "The failed resource is listed without a resource_id or state, so a later deploy plans to create it rather than treating it as already deployed" +title "The failed resource is not listed at all: state is what projects a resource, and a failed create records none, so a later deploy plans to create it rather than treating it as already deployed" # The deployment ID is the workspace node's ID; read it back the way the CLI does. # Extracted with python, not jq: the ID exceeds 2^53 and jq 1.6 rounds it. deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-record-failure/default/state/resources.deployment.json" -o json | python3 -c 'import sys,json; print(json.load(sys.stdin)["object_id"])') diff --git a/bundle/direct/dstate/dms.go b/bundle/direct/dstate/dms.go index 07e879973f2..506a9869ce6 100644 --- a/bundle/direct/dstate/dms.go +++ b/bundle/direct/dstate/dms.go @@ -86,15 +86,6 @@ func fetchDeploymentResources(ctx context.Context, client bundledeployments.Bund } } - // A resource with no id was never created: the deploy that recorded it failed - // before the API assigned one (a failed create is recorded with the error and - // nothing else). Leaving it out keeps it untracked, so the next deploy creates - // it and a destroy skips it - an entry with an empty id would instead look - // tracked and fail the delete with "missing in state". - if res.ResourceId == "" { - continue - } - out[key] = ResourceEntry{ ID: res.ResourceId, State: recorded.State, diff --git a/bundle/direct/dstate/dms_test.go b/bundle/direct/dstate/dms_test.go index c475942f829..8145424ca93 100644 --- a/bundle/direct/dstate/dms_test.go +++ b/bundle/direct/dstate/dms_test.go @@ -61,23 +61,6 @@ func TestFetchDeploymentResourcesUnwrapsEnvelope(t *testing.T) { }, got) } -func TestFetchDeploymentResourcesSkipsResourceWithoutID(t *testing.T) { - // A failed create is recorded with its error and nothing else, so the resource has - // no id. Keeping it would make the resource look tracked while referring to nothing, - // and a later destroy fails with "missing in state" instead of skipping it. - f := &fakeResourceLister{resources: []bundledeployments.Resource{ - {ResourceKey: "jobs.created", ResourceId: "123"}, - {ResourceKey: "jobs.failed"}, - }} - - got, err := fetchDeploymentResources(t.Context(), f, "dep-1") - require.NoError(t, err) - - assert.Equal(t, map[string]ResourceEntry{ - "resources.jobs.created": {ID: "123"}, - }, got) -} - func TestFetchDeploymentResourcesRejectsMalformedState(t *testing.T) { recorded := json.RawMessage(`not json`) f := &fakeResourceLister{resources: []bundledeployments.Resource{ diff --git a/libs/testserver/bundle.go b/libs/testserver/bundle.go index e3e8a2428ba..706fdb8efb5 100644 --- a/libs/testserver/bundle.go +++ b/libs/testserver/bundle.go @@ -304,17 +304,14 @@ func (s *FakeWorkspace) CreateOperation(req Request, deploymentID, versionID str return Response{StatusCode: 500, Body: map[string]string{"message": err.Error()}} } - // Reflect the operation onto the deployment-level resource set the way the - // backend does: a delete removes the resource, anything else upserts it. - // - // A failed operation is upserted too, matching the service, but it carries - // neither a resource_id nor state, so the read path treats the resource as not - // yet created rather than as existing state (verified against the service: a - // failed create is listed with an empty resource_id and no state). - switch { - case op.ActionType == bundledeployments.OperationActionTypeOperationActionTypeDelete && !failed: + // Reflect the operation onto the deployment-level resource set the way the backend + // does: state is what projects a resource, so an operation without it removes the + // resource instead of listing one (OperationStorage.createOperation buffers a delete + // when the entity has no state). Together with the invariant that state requires a + // resource_id, that means a listed resource always has an id. + if op.State == nil { delete(d.resources, resourceKey) - default: + } else { d.resources[resourceKey] = bundledeployments.Resource{ Name: "deployments/" + deploymentID + "/resources/" + resourceKey, ResourceKey: resourceKey, @@ -421,7 +418,7 @@ func (s *FakeWorkspace) UpdateOperation(req Request, deploymentID, versionID, re // Mirror onto the resource set the same way CreateOperation does, so the read // path reflects the newest write. - if existing.ActionType == bundledeployments.OperationActionTypeOperationActionTypeDelete && !failed { + if existing.State == nil { delete(d.resources, resourceKey) } else { d.resources[resourceKey] = bundledeployments.Resource{ From f26c7d609325c9ce4c44d9ac4441be95b2d9c76c Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 10 Aug 2026 10:48:08 +0000 Subject: [PATCH 067/125] bundle: correct what AllowExistingResources is for An earlier revision of this comment claimed the guard fires for any test that has already deployed once. It does not: once a deploy has been recorded a deployment exists, and a non-empty DeploymentID short-circuits the guard. Verified by turning the flag off for resources/jobs/update, which deploys twice and still passes. What actually needs it is the tests that seed a state file before deploying - the WAL and idempotency ones - where state exists before recording ever starts. Co-authored-by: Isaac --- bundle/direct/dstate/state.go | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/bundle/direct/dstate/state.go b/bundle/direct/dstate/state.go index fc11aee7f50..85db3e0399f 100644 --- a/bundle/direct/dstate/state.go +++ b/bundle/direct/dstate/state.go @@ -274,11 +274,10 @@ type DMSSource struct { // touch are absent from DMS and a later deploy plans them as creates. // // It exists for the CLI's own acceptance tests, which run the whole bundle suite - // with recording on. The guard trips on any non-empty state, so it catches every - // test that has already deployed once in its script, not only the five that commit - // a resources.json fixture - without this the DMS run fails 154 of them. They - // assert what one deploy does rather than reading state back, so the duplication - // the refusal prevents cannot bite them. + // with recording on. The ones that need it seed a state file before deploying (the + // WAL and idempotency tests), so the guard fires on their first deploy and they + // never reach what they are actually testing. They assert what a deploy does rather + // than reading state back, so the duplication the refusal prevents cannot bite them. AllowExistingResources bool } From c882a39cfaf821da7faf7b39cd96c77526ae9371 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 10 Aug 2026 11:40:52 +0000 Subject: [PATCH 068/125] bundle: drop AllowExistingResources; opt the tests that need it out instead The option let recording proceed on a bundle whose state already tracks resources, which the guard exists to refuse. It was there only so the acceptance suite could run, and a user could reach it through an env var - a way to get exactly the resource duplication the guard prevents. Only eight test directories need it, so they opt out of the DMS run with the reason in their test.toml. Every other test now runs with the guard live, which is real coverage: a change that starts tripping it will fail rather than pass. Two shapes need the opt-out. Most seed a state file before deploying. The idempotency ones rewind state and wipe the remote state path, which is where the deployment record lives - so the next deploy finds populated state and no deployment, exactly what the guard refuses. Co-authored-by: Isaac --- .../deploy/files/out-of-band-delete/out.test.toml | 2 +- .../deploy/files/out-of-band-delete/test.toml | 5 +++++ .../deploy/wal/corrupted-wal-entry/out.test.toml | 2 +- .../deploy/wal/corrupted-wal-entry/test.toml | 4 ++++ .../bundle/deploy/wal/stale-wal/out.test.toml | 2 +- acceptance/bundle/deploy/wal/stale-wal/test.toml | 5 +++++ acceptance/bundle/dms/test.toml | 5 ----- .../invariant/delete_idempotent/out.test.toml | 2 +- .../bundle/invariant/delete_idempotent/test.toml | 6 ++++++ .../invariant/destroy_idempotent/out.test.toml | 2 +- .../bundle/invariant/destroy_idempotent/test.toml | 6 ++++++ acceptance/bundle/invariant/migrate/out.test.toml | 2 +- acceptance/bundle/invariant/migrate/test.toml | 5 +++++ .../jobs/destroy_without_mgmtperms/test.toml | 4 ++++ .../with_permissions/out.test.toml | 2 +- .../without_permissions/out.test.toml | 2 +- .../state/permission_level_migration/out.test.toml | 2 +- .../state/permission_level_migration/test.toml | 5 +++++ acceptance/bundle/test.toml | 7 ++----- bundle/direct/dstate/state.go | 14 +------------- bundle/env/dms.go | 12 ------------ cmd/bundle/utils/process.go | 5 ++--- 22 files changed, 54 insertions(+), 47 deletions(-) create mode 100644 acceptance/bundle/deploy/wal/corrupted-wal-entry/test.toml create mode 100644 acceptance/bundle/resources/permissions/jobs/destroy_without_mgmtperms/test.toml diff --git a/acceptance/bundle/deploy/files/out-of-band-delete/out.test.toml b/acceptance/bundle/deploy/files/out-of-band-delete/out.test.toml index 92ced1275e4..48cc8d7c1c4 100644 --- a/acceptance/bundle/deploy/files/out-of-band-delete/out.test.toml +++ b/acceptance/bundle/deploy/files/out-of-band-delete/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/deploy/files/out-of-band-delete/test.toml b/acceptance/bundle/deploy/files/out-of-band-delete/test.toml index 6a62c44e94d..c36335a3c48 100644 --- a/acceptance/bundle/deploy/files/out-of-band-delete/test.toml +++ b/acceptance/bundle/deploy/files/out-of-band-delete/test.toml @@ -1,3 +1,8 @@ +# Recording needs a bundle it has seen from the start. This test seeds a state file, +# so recording refuses it. TODO(DMS): drop this once existing state can be +# handed over to the service (see the TODO in dstate.Open). +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] + Badness = "After the remote bundle files are deleted out-of-band, the next deploy does not re-upload them until the local sync snapshot is removed." [Env] diff --git a/acceptance/bundle/deploy/wal/corrupted-wal-entry/out.test.toml b/acceptance/bundle/deploy/wal/corrupted-wal-entry/out.test.toml index 96a25cb4752..2a52887146a 100644 --- a/acceptance/bundle/deploy/wal/corrupted-wal-entry/out.test.toml +++ b/acceptance/bundle/deploy/wal/corrupted-wal-entry/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deploy/wal/corrupted-wal-entry/test.toml b/acceptance/bundle/deploy/wal/corrupted-wal-entry/test.toml new file mode 100644 index 00000000000..a55b7341585 --- /dev/null +++ b/acceptance/bundle/deploy/wal/corrupted-wal-entry/test.toml @@ -0,0 +1,4 @@ +# Recording needs a bundle it has seen from the start. This test seeds a state file, +# so recording refuses it. TODO(DMS): drop this once existing state can be +# handed over to the service (see the TODO in dstate.Open). +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] diff --git a/acceptance/bundle/deploy/wal/stale-wal/out.test.toml b/acceptance/bundle/deploy/wal/stale-wal/out.test.toml index 96a25cb4752..2a52887146a 100644 --- a/acceptance/bundle/deploy/wal/stale-wal/out.test.toml +++ b/acceptance/bundle/deploy/wal/stale-wal/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deploy/wal/stale-wal/test.toml b/acceptance/bundle/deploy/wal/stale-wal/test.toml index 934683ba6d8..09ff4752240 100644 --- a/acceptance/bundle/deploy/wal/stale-wal/test.toml +++ b/acceptance/bundle/deploy/wal/stale-wal/test.toml @@ -1,3 +1,8 @@ +# Recording needs a bundle it has seen from the start. This test seeds a state file, +# so recording refuses it. TODO(DMS): drop this once existing state can be +# handed over to the service (see the TODO in dstate.Open). +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] + # Deploy with a stale WAL (old serial) - WAL should be deleted and ignored. [[Server]] diff --git a/acceptance/bundle/dms/test.toml b/acceptance/bundle/dms/test.toml index e491f968d76..d11bc99a977 100644 --- a/acceptance/bundle/dms/test.toml +++ b/acceptance/bundle/dms/test.toml @@ -20,8 +20,3 @@ Ignore = [ # so they force allow it the same way DMS development does. bundle/dms/not-supported # covers the rejection. Env.DATABRICKS_BUNDLE_FORCE_ALLOW_RECORD_DEPLOYMENT_HISTORY = "1" - -# The parent lets the rest of the suite record a bundle whose state already tracks -# resources, since most of those tests seed a state fixture. bundle/dms/existing-state -# asserts that refusal, so it has to stay on here. -Env.DATABRICKS_BUNDLE_DMS_ALLOW_EXISTING_RESOURCES = "" diff --git a/acceptance/bundle/invariant/delete_idempotent/out.test.toml b/acceptance/bundle/invariant/delete_idempotent/out.test.toml index 3ce69ae1a49..f1d5732abd2 100644 --- a/acceptance/bundle/invariant/delete_idempotent/out.test.toml +++ b/acceptance/bundle/invariant/delete_idempotent/out.test.toml @@ -1,7 +1,7 @@ Local = true Cloud = true RequiresUnityCatalog = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] EnvMatrix.INPUT_CONFIG = [ "alert.yml.tmpl", diff --git a/acceptance/bundle/invariant/delete_idempotent/test.toml b/acceptance/bundle/invariant/delete_idempotent/test.toml index 3f5bb92afad..aab4929d07c 100644 --- a/acceptance/bundle/invariant/delete_idempotent/test.toml +++ b/acceptance/bundle/invariant/delete_idempotent/test.toml @@ -1,3 +1,9 @@ +# Recording needs a bundle it has seen from the start. This test rewinds state and +# wipes the remote path the deployment record lives under, so recording refuses it. +# TODO(DMS): drop this once existing state can be handed over to the service (see +# the TODO in dstate.Open). +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] + EnvMatrix.READPLAN = ["", "1"] # Snapshot of pre-delete state used to re-run the delete on state that still diff --git a/acceptance/bundle/invariant/destroy_idempotent/out.test.toml b/acceptance/bundle/invariant/destroy_idempotent/out.test.toml index 3ce69ae1a49..f1d5732abd2 100644 --- a/acceptance/bundle/invariant/destroy_idempotent/out.test.toml +++ b/acceptance/bundle/invariant/destroy_idempotent/out.test.toml @@ -1,7 +1,7 @@ Local = true Cloud = true RequiresUnityCatalog = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] EnvMatrix.INPUT_CONFIG = [ "alert.yml.tmpl", diff --git a/acceptance/bundle/invariant/destroy_idempotent/test.toml b/acceptance/bundle/invariant/destroy_idempotent/test.toml index 16cf0797a77..ddd6d204f5d 100644 --- a/acceptance/bundle/invariant/destroy_idempotent/test.toml +++ b/acceptance/bundle/invariant/destroy_idempotent/test.toml @@ -1,3 +1,9 @@ +# Recording needs a bundle it has seen from the start. This test rewinds state and +# wipes the remote path the deployment record lives under, so recording refuses it. +# TODO(DMS): drop this once existing state can be handed over to the service (see +# the TODO in dstate.Open). +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] + EnvMatrix.READPLAN = ["", "1"] # Snapshot of pre-destroy state used to re-run destroy on state that still diff --git a/acceptance/bundle/invariant/migrate/out.test.toml b/acceptance/bundle/invariant/migrate/out.test.toml index cf188cbc54b..0971af823da 100644 --- a/acceptance/bundle/invariant/migrate/out.test.toml +++ b/acceptance/bundle/invariant/migrate/out.test.toml @@ -1,7 +1,7 @@ Local = true Cloud = true RequiresUnityCatalog = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] EnvMatrix.INPUT_CONFIG = [ "alert.yml.tmpl", diff --git a/acceptance/bundle/invariant/migrate/test.toml b/acceptance/bundle/invariant/migrate/test.toml index bb2337b32aa..517416132a2 100644 --- a/acceptance/bundle/invariant/migrate/test.toml +++ b/acceptance/bundle/invariant/migrate/test.toml @@ -1,3 +1,8 @@ +# Recording needs a bundle it has seen from the start. This test seeds a state file, +# so recording refuses it. TODO(DMS): drop this once existing state can be +# handed over to the service (see the TODO in dstate.Open). +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] + # vector_search_endpoints and vector_search_indexes have no terraform converter EnvMatrixExclude.no_vector_search_endpoint = ["INPUT_CONFIG=vector_search_endpoint.yml.tmpl"] EnvMatrixExclude.no_vector_search_index = ["INPUT_CONFIG=vector_search_index.yml.tmpl"] diff --git a/acceptance/bundle/resources/permissions/jobs/destroy_without_mgmtperms/test.toml b/acceptance/bundle/resources/permissions/jobs/destroy_without_mgmtperms/test.toml new file mode 100644 index 00000000000..a55b7341585 --- /dev/null +++ b/acceptance/bundle/resources/permissions/jobs/destroy_without_mgmtperms/test.toml @@ -0,0 +1,4 @@ +# Recording needs a bundle it has seen from the start. This test seeds a state file, +# so recording refuses it. TODO(DMS): drop this once existing state can be +# handed over to the service (see the TODO in dstate.Open). +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] diff --git a/acceptance/bundle/resources/permissions/jobs/destroy_without_mgmtperms/with_permissions/out.test.toml b/acceptance/bundle/resources/permissions/jobs/destroy_without_mgmtperms/with_permissions/out.test.toml index aa99ae397ac..19458fe3f0c 100644 --- a/acceptance/bundle/resources/permissions/jobs/destroy_without_mgmtperms/with_permissions/out.test.toml +++ b/acceptance/bundle/resources/permissions/jobs/destroy_without_mgmtperms/with_permissions/out.test.toml @@ -3,5 +3,5 @@ Cloud = true RunsOnDbr = false CloudEnvs.azure = false CloudEnvs.gcp = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/permissions/jobs/destroy_without_mgmtperms/without_permissions/out.test.toml b/acceptance/bundle/resources/permissions/jobs/destroy_without_mgmtperms/without_permissions/out.test.toml index aa99ae397ac..19458fe3f0c 100644 --- a/acceptance/bundle/resources/permissions/jobs/destroy_without_mgmtperms/without_permissions/out.test.toml +++ b/acceptance/bundle/resources/permissions/jobs/destroy_without_mgmtperms/without_permissions/out.test.toml @@ -3,5 +3,5 @@ Cloud = true RunsOnDbr = false CloudEnvs.azure = false CloudEnvs.gcp = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/state/permission_level_migration/out.test.toml b/acceptance/bundle/state/permission_level_migration/out.test.toml index 96a25cb4752..2a52887146a 100644 --- a/acceptance/bundle/state/permission_level_migration/out.test.toml +++ b/acceptance/bundle/state/permission_level_migration/out.test.toml @@ -1,4 +1,4 @@ Local = true Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/state/permission_level_migration/test.toml b/acceptance/bundle/state/permission_level_migration/test.toml index 7fc493d51a4..4adec0e8bb5 100644 --- a/acceptance/bundle/state/permission_level_migration/test.toml +++ b/acceptance/bundle/state/permission_level_migration/test.toml @@ -1,3 +1,8 @@ +# Recording needs a bundle it has seen from the start. This test seeds a state file, +# so recording refuses it. TODO(DMS): drop this once existing state can be +# handed over to the service (see the TODO in dstate.Open). +EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] + Ignore = [".databricks"] [EnvMatrix] diff --git a/acceptance/bundle/test.toml b/acceptance/bundle/test.toml index 32ccd18e452..e7355915277 100644 --- a/acceptance/bundle/test.toml +++ b/acceptance/bundle/test.toml @@ -18,12 +18,9 @@ EnvMatrixExclude.dms_local_only = ["DATABRICKS_BUNDLE_DMS=true", "CONFIG_Cloud=t # which is a design decision, so the saved-plan path is left out of the DMS run for now. EnvMatrixExclude.dms_no_readplan = ["DATABRICKS_BUNDLE_DMS=true", "READPLAN=1"] -# Recording is gated off for users (see validate.ValidateRecordDeploymentHistory) and -# refuses a bundle whose state already tracks resources - which most tests here seed. -# Both are forced on: these tests assert what a deploy does, so the resource duplication -# the refusal guards against cannot bite them. +# Recording is gated off for users (see validate.ValidateRecordDeploymentHistory), so +# force it on: the point of this run is to exercise it. Env.DATABRICKS_BUNDLE_FORCE_ALLOW_RECORD_DEPLOYMENT_HISTORY = "1" -Env.DATABRICKS_BUNDLE_DMS_ALLOW_EXISTING_RESOURCES = "1" # The DMS run asserts the same golden files as the engine runs. EnvRepl.DATABRICKS_BUNDLE_DMS = false diff --git a/bundle/direct/dstate/state.go b/bundle/direct/dstate/state.go index 85db3e0399f..aaf5cda7400 100644 --- a/bundle/direct/dstate/state.go +++ b/bundle/direct/dstate/state.go @@ -267,18 +267,6 @@ type DMSSource struct { // DeploymentID is resolved from the deployment's workspace node (see // dms.ResolveDeploymentID), and empty before the first recorded deploy. DeploymentID string - - // AllowExistingResources records a bundle whose state file already tracks - // resources, instead of refusing it. Those resources are not handed over to DMS: - // the first recorded deploy reports only what it touches, so the ones it does not - // touch are absent from DMS and a later deploy plans them as creates. - // - // It exists for the CLI's own acceptance tests, which run the whole bundle suite - // with recording on. The ones that need it seed a state file before deploying (the - // WAL and idempotency tests), so the guard fires on their first deploy and they - // never reach what they are actually testing. They assert what a deploy does rather - // than reading state back, so the duplication the refusal prevents cannot bite them. - AllowExistingResources bool } // Open reads the deployment state from disk (and recovers the WAL when @@ -343,7 +331,7 @@ func (db *DeploymentState) Open(ctx context.Context, path string, withRecovery W // featureStateVersion with a feature flag plus a tombstone per resource so an // older CLI refuses the state instead of deploying against resources it // cannot see. - if dmsSource.DeploymentID == "" && len(db.Data.State) > 0 && !dmsSource.AllowExistingResources { + if dmsSource.DeploymentID == "" && len(db.Data.State) > 0 { // The remedy is ordered deliberately: this error also blocks destroy, so the // setting has to come out first or there is no way to tear the bundle down. return fmt.Errorf(`cannot record deployment history for a bundle that already has deployed resources tracked in %s: only new deployments can be recorded diff --git a/bundle/env/dms.go b/bundle/env/dms.go index 812d492d4bc..53aeaccc011 100644 --- a/bundle/env/dms.go +++ b/bundle/env/dms.go @@ -23,15 +23,3 @@ func DMS(ctx context.Context) bool { func RecordsDeploymentHistory(ctx context.Context, configured bool) bool { return configured || DMS(ctx) } - -// DMSAllowExistingResourcesVariable names the environment variable that lets a bundle -// with resources already in its state file be recorded, which is otherwise refused -// (see dstate.DMSSource.AllowExistingResources for what that costs). -const DMSAllowExistingResourcesVariable = "DATABRICKS_BUNDLE_DMS_ALLOW_EXISTING_RESOURCES" - -// DMSAllowExistingResources reports whether the environment allows recording a bundle -// that already tracks resources. -func DMSAllowExistingResources(ctx context.Context) bool { - value, ok := get(ctx, []string{DMSAllowExistingResourcesVariable}) - return ok && value != "" && value != "0" && value != "false" -} diff --git a/cmd/bundle/utils/process.go b/cmd/bundle/utils/process.go index f2b1750205e..36205181115 100644 --- a/cmd/bundle/utils/process.go +++ b/cmd/bundle/utils/process.go @@ -234,9 +234,8 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle return b, stateDesc, root.ErrAlreadyPrinted } dmsSource = &dstate.DMSSource{ - Client: w.BundleDeployments, - DeploymentID: deploymentID, - AllowExistingResources: env.DMSAllowExistingResources(ctx), + Client: w.BundleDeployments, + DeploymentID: deploymentID, } // Stamp the deployment onto the resources before anything diffs them. From 403e6597ed22358f074e24f74d35d9f23dc37017 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 10 Aug 2026 12:01:11 +0000 Subject: [PATCH 069/125] bundle: say "unset" in the recording refusal, and table the sink tests The refusal's last line said to "leave experimental.record_deployment_history out", which reads like advice for a config you have not written yet. The user seeing this already has it set, so "unset" is the instruction that matches their situation. The two sink tests differed only in which writes they made and what they expected, so they are now cases of one table-driven test. Co-authored-by: Isaac --- .../bundle/dms/existing-state/output.txt | 4 +- bundle/direct/dstate/state.go | 2 +- bundle/direct/dstate/state_test.go | 83 ++++++++++--------- 3 files changed, 49 insertions(+), 40 deletions(-) diff --git a/acceptance/bundle/dms/existing-state/output.txt b/acceptance/bundle/dms/existing-state/output.txt index 6233f158974..b1b18d281c6 100644 --- a/acceptance/bundle/dms/existing-state/output.txt +++ b/acceptance/bundle/dms/existing-state/output.txt @@ -19,7 +19,7 @@ To record this bundle's history, start it over as a new deployment: 2. run "databricks bundle destroy" to delete the existing resources 3. add experimental.record_deployment_history back and deploy again -To keep the existing resources instead, leave experimental.record_deployment_history out +To keep the existing resources instead, unset experimental.record_deployment_history === No deployment was created in DMS @@ -34,7 +34,7 @@ To record this bundle's history, start it over as a new deployment: 2. run "databricks bundle destroy" to delete the existing resources 3. add experimental.record_deployment_history back and deploy again -To keep the existing resources instead, leave experimental.record_deployment_history out +To keep the existing resources instead, unset experimental.record_deployment_history >>> print_requests.py //api/2.0/bundle --oneline diff --git a/bundle/direct/dstate/state.go b/bundle/direct/dstate/state.go index aaf5cda7400..861686a87a7 100644 --- a/bundle/direct/dstate/state.go +++ b/bundle/direct/dstate/state.go @@ -341,7 +341,7 @@ To record this bundle's history, start it over as a new deployment: 2. run "databricks bundle destroy" to delete the existing resources 3. add experimental.record_deployment_history back and deploy again -To keep the existing resources instead, leave experimental.record_deployment_history out`, path) +To keep the existing resources instead, unset experimental.record_deployment_history`, path) } if dmsSource.DeploymentID != "" { if err := db.readDMSState(ctx, dmsSource); err != nil { diff --git a/bundle/direct/dstate/state_test.go b/bundle/direct/dstate/state_test.go index 7494f662f96..b0dafb2933a 100644 --- a/bundle/direct/dstate/state_test.go +++ b/bundle/direct/dstate/state_test.go @@ -29,47 +29,56 @@ func (f *fakeSink) RecordOperation(ctx context.Context, resourceKey string, acti } func TestStateWritesRecordOperations(t *testing.T) { - path := filepath.Join(t.TempDir(), "state.json") - sink := &fakeSink{} - - var db DeploymentState - require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil)) - db.SetOperationSink(sink) - - // A recreate: the old entry is dropped, then the new resource is saved. - require.NoError(t, db.SaveState(t.Context(), "jobs.my_job", "123", map[string]string{"key": "old"}, nil, deployplan.Create)) - require.NoError(t, db.DeleteState(t.Context(), "jobs.my_job", deployplan.Recreate)) - require.NoError(t, db.SaveState(t.Context(), "jobs.my_job", "456", map[string]string{"key": "new"}, nil, deployplan.Recreate)) - mustFinalize(t, &db) - - // Both of the recreate's writes are reported. The service keeps one operation per - // resource per version, so the drop opens it (no state: the old resource is gone and - // the new one does not exist yet) and the save completes it. A deploy that stops in - // between leaves the resource described as mid-recreate. - assert.Equal(t, []string{ - `create jobs.my_job id=123 state={"state":{"key":"old"}}`, - `recreate jobs.my_job id=123 state=`, - `recreate jobs.my_job id=456 state={"state":{"key":"new"}}`, - }, sink.ops) -} + tests := []struct { + name string + write func(t *testing.T, db *DeploymentState) + want []string + }{ + { + // The service keeps one operation per resource per version, so the drop + // opens it (no state: the old resource is gone and the new one does not + // exist yet) and the save completes it. A deploy that stops in between + // leaves the resource described as mid-recreate. + name: "recreate reports both of its writes", + write: func(t *testing.T, db *DeploymentState) { + require.NoError(t, db.SaveState(t.Context(), "jobs.my_job", "123", map[string]string{"key": "old"}, nil, deployplan.Create)) + require.NoError(t, db.DeleteState(t.Context(), "jobs.my_job", deployplan.Recreate)) + require.NoError(t, db.SaveState(t.Context(), "jobs.my_job", "456", map[string]string{"key": "new"}, nil, deployplan.Recreate)) + }, + want: []string{ + `create jobs.my_job id=123 state={"state":{"key":"old"}}`, + `recreate jobs.my_job id=123 state=`, + `recreate jobs.my_job id=456 state={"state":{"key":"new"}}`, + }, + }, + { + name: "real delete reports the id it had and no state", + write: func(t *testing.T, db *DeploymentState) { + require.NoError(t, db.SaveState(t.Context(), "jobs.my_job", "123", map[string]string{}, nil, deployplan.Create)) + require.NoError(t, db.DeleteState(t.Context(), "jobs.my_job", deployplan.Delete)) + }, + want: []string{ + `create jobs.my_job id=123 state={"state":{}}`, + `delete jobs.my_job id=123 state=`, + }, + }, + } -func TestDeleteStateRecordsRealDelete(t *testing.T) { - path := filepath.Join(t.TempDir(), "state.json") - sink := &fakeSink{} + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + path := filepath.Join(t.TempDir(), "state.json") + sink := &fakeSink{} - var db DeploymentState - require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil)) - db.SetOperationSink(sink) + var db DeploymentState + require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil)) + db.SetOperationSink(sink) - require.NoError(t, db.SaveState(t.Context(), "jobs.my_job", "123", map[string]string{}, nil, deployplan.Create)) - require.NoError(t, db.DeleteState(t.Context(), "jobs.my_job", deployplan.Delete)) - mustFinalize(t, &db) + tt.write(t, &db) + mustFinalize(t, &db) - // A real delete reports the id it had and no state: the resource is gone. - assert.Equal(t, []string{ - `create jobs.my_job id=123 state={"state":{}}`, - `delete jobs.my_job id=123 state=`, - }, sink.ops) + assert.Equal(t, tt.want, sink.ops) + }) + } } func TestStateWritesRecordNothingWithoutSink(t *testing.T) { From eca068d260f772bd454b6822f2f87aa001ac520c Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 10 Aug 2026 16:20:01 +0000 Subject: [PATCH 070/125] bundle: note where the SDK bypass ends The comment explained why the operations calls skip the generated client but not how to get rid of it. The fix belongs in the OpenAPI spec the SDK is generated from; once sequence_id is typed as a string this file is deletable. Co-authored-by: Isaac --- bundle/direct/opclient.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/bundle/direct/opclient.go b/bundle/direct/opclient.go index 009089cdb85..8df7cd6a862 100644 --- a/bundle/direct/opclient.go +++ b/bundle/direct/opclient.go @@ -18,6 +18,10 @@ import ( // that way), so unmarshalling a CreateOperation response fails with // "invalid character '1' after top-level value". The write itself succeeds - the // status is 200 - so only the response parse is affected. +// +// TODO(DMS): this whole file goes away once the SDK types sequence_id as a string. +// The fix belongs in the OpenAPI spec the SDK is generated from, not here; until then +// every other DMS call still goes through the SDK, so keep the bypass to operations. // operationResponse is the part of an operation response the CLI reads back. type operationResponse struct { From 5d1beb50c05583ae270ae06a170df7ac03c51dde Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 10 Aug 2026 16:26:04 +0000 Subject: [PATCH 071/125] bundle: drop graph_test.go It covers makeGraph, not deployment-history recording, so it does not belong in this PR. The delete-order reversal it pinned is worth a test; that goes in a separate change against bundle/direct. Co-authored-by: Isaac --- bundle/direct/graph_test.go | 78 ------------------------------------- 1 file changed, 78 deletions(-) delete mode 100644 bundle/direct/graph_test.go diff --git a/bundle/direct/graph_test.go b/bundle/direct/graph_test.go deleted file mode 100644 index a99082407e4..00000000000 --- a/bundle/direct/graph_test.go +++ /dev/null @@ -1,78 +0,0 @@ -package direct - -import ( - "sync" - "testing" - - "github.com/databricks/cli/bundle/deployplan" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -// runOrder returns the order makeGraph's edges let the nodes run in. -func runOrder(t *testing.T, plan *deployplan.Plan) []string { - g, err := makeGraph(plan) - require.NoError(t, err) - require.NoError(t, g.DetectCycle()) - - var mu sync.Mutex - var order []string - g.Run(1, func(node string, failedDependency *string) bool { - mu.Lock() - defer mu.Unlock() - order = append(order, node) - return true - }) - return order -} - -func TestMakeGraphOrdersDependencyBeforeDependent(t *testing.T) { - plan := &deployplan.Plan{Plan: map[string]*deployplan.PlanEntry{ - "resources.jobs.parent": {Action: "create"}, - "resources.jobs.child": { - Action: "create", - DependsOn: []deployplan.DependsOnEntry{{Node: "resources.jobs.parent"}}, - }, - }} - - assert.Equal(t, []string{"resources.jobs.parent", "resources.jobs.child"}, runOrder(t, plan)) -} - -func TestMakeGraphReversesOrderForDelete(t *testing.T) { - // A delete has to run the other way round: the child refers to the parent, so - // removing the parent first would leave the child pointing at nothing. - plan := &deployplan.Plan{Plan: map[string]*deployplan.PlanEntry{ - "resources.jobs.parent": {Action: "delete"}, - "resources.jobs.child": { - Action: "delete", - DependsOn: []deployplan.DependsOnEntry{{Node: "resources.jobs.parent"}}, - }, - }} - - assert.Equal(t, []string{"resources.jobs.child", "resources.jobs.parent"}, runOrder(t, plan)) -} - -func TestMakeGraphRejectsUnknownDependency(t *testing.T) { - plan := &deployplan.Plan{Plan: map[string]*deployplan.PlanEntry{ - "resources.jobs.child": { - Action: "create", - DependsOn: []deployplan.DependsOnEntry{{Node: "resources.jobs.missing", Label: "${resources.jobs.missing.id}"}}, - }, - }} - - _, err := makeGraph(plan) - assert.ErrorContains(t, err, `no such node "resources.jobs.missing"`) -} - -func TestMakeGraphIgnoresUnknownDependencyOnDelete(t *testing.T) { - // A destroy plans only what state tracks, so a dependency on something already - // gone is expected rather than an error. - plan := &deployplan.Plan{Plan: map[string]*deployplan.PlanEntry{ - "resources.jobs.child": { - Action: "delete", - DependsOn: []deployplan.DependsOnEntry{{Node: "resources.jobs.missing"}}, - }, - }} - - assert.Equal(t, []string{"resources.jobs.child"}, runOrder(t, plan)) -} From 9ef3061597f85ea03a6863b345a67f8d4a9f1171 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 10 Aug 2026 16:57:59 +0000 Subject: [PATCH 072/125] bundle: decide the in-progress status at the state write The queue inferred it from the write's shape - action == Recreate && state == nil - which coupled the recorder to how a recreate happens to be implemented and gave the call site no way to say what it meant. Pass a dstate.OperationInfo instead, so the recreate's delete asks for in-progress explicitly and the queue just reports it. Doing so also stops discarding the resource id. newInProgressOperation() built its own struct and ignored the id the sink was already given, so an interrupted recreate recorded an empty resource_id; it now names the resource that was mid-flight. The service allows this - state requires an id, not the reverse - and projection still skips the resource while state is nil. Co-authored-by: Isaac --- .../bundle/dms/partial-update/output.txt | 1 + acceptance/bundle/dms/partial-update/script | 7 ++-- bundle/direct/apply.go | 17 +++++----- bundle/direct/bind.go | 8 ++--- bundle/direct/bundle_apply.go | 3 +- bundle/direct/dstate/dms.go | 15 +++++++- bundle/direct/dstate/state.go | 21 +++++------- bundle/direct/dstate/state_test.go | 34 +++++++++++-------- bundle/direct/opqueue.go | 13 ++----- bundle/direct/opqueue_test.go | 26 +++++++------- bundle/direct/oprecorder.go | 26 ++++++-------- bundle/direct/oprecorder_test.go | 9 ++--- bundle/migrate/build_state.go | 2 +- 13 files changed, 93 insertions(+), 89 deletions(-) diff --git a/acceptance/bundle/dms/partial-update/output.txt b/acceptance/bundle/dms/partial-update/output.txt index 0d65dc2fd06..ec3cca1095b 100644 --- a/acceptance/bundle/dms/partial-update/output.txt +++ b/acceptance/bundle/dms/partial-update/output.txt @@ -93,6 +93,7 @@ Deployment complete! }, "body": { "action_type": "OPERATION_ACTION_TYPE_RECREATE", + "resource_id": "main.dms_partial_update_schema", "resource_key": "schemas.foo", "status": "OPERATION_STATUS_IN_PROGRESS" } diff --git a/acceptance/bundle/dms/partial-update/script b/acceptance/bundle/dms/partial-update/script index 38c3f66c756..52340c21037 100644 --- a/acceptance/bundle/dms/partial-update/script +++ b/acceptance/bundle/dms/partial-update/script @@ -4,9 +4,10 @@ trace print_requests.py //api/2.0/bundle title "Recreate writes state twice - the entry is dropped, then the new resource is saved" # The service keeps one operation per resource per version, so both writes land on the -# same one: the drop opens it as IN_PROGRESS with no state, and the save that follows -# patches it to SUCCEEDED. A deploy that dies in between therefore leaves the resource -# described as mid-recreate rather than as the resource it already deleted. +# same one: the drop opens it as IN_PROGRESS carrying the deleted id but no state, and +# the save that follows patches it to SUCCEEDED with the new id. A deploy that dies in +# between therefore leaves the resource described as mid-recreate rather than as the +# resource it already deleted. trace update_file.py databricks.yml "catalog_name: main" "catalog_name: other" trace $CLI bundle deploy --auto-approve trace print_requests.py //api/2.0/bundle diff --git a/bundle/direct/apply.go b/bundle/direct/apply.go index d615b5c3566..ec5761c4a76 100644 --- a/bundle/direct/apply.go +++ b/bundle/direct/apply.go @@ -75,7 +75,7 @@ func (d *DeploymentUnit) Create(ctx context.Context, db *dstate.DeploymentState, return err } - err = db.SaveState(ctx, d.ResourceKey, newID, newState, d.DependsOn, deployplan.Create) + err = db.SaveState(ctx, d.ResourceKey, newID, newState, d.DependsOn, dstate.OperationInfo{Action: deployplan.Create}) if err != nil { return fmt.Errorf("saving state after creating id=%s: %w", newID, err) } @@ -119,8 +119,9 @@ func (d *DeploymentUnit) Recreate(ctx context.Context, db *dstate.DeploymentStat // // Recorded as a recreate, not a delete: if the create below fails, this is the // operation DMS is left with, and it says the resource is mid-recreate rather - // than deliberately removed. - err = db.DeleteState(ctx, d.ResourceKey, deployplan.Recreate) + // than deliberately removed. In-progress for the same reason - the create that + // follows updates the same operation to succeeded. + err = db.DeleteState(ctx, d.ResourceKey, dstate.OperationInfo{Action: deployplan.Recreate, InProgress: true}) if err != nil { return fmt.Errorf("deleting state: %w", err) } @@ -166,12 +167,12 @@ func (d *DeploymentUnit) Update(ctx context.Context, db *dstate.DeploymentState, // Recorded as a delete, not the update that caused it: the resource is no longer // tracked, and DMS drops it from the deployment only for a delete. Recording an // update would leave it listed with no state, which the next plan cannot read. - err = db.DeleteState(ctx, d.ResourceKey, deployplan.Delete) + err = db.DeleteState(ctx, d.ResourceKey, dstate.OperationInfo{Action: deployplan.Delete}) if err != nil { return fmt.Errorf("deleting state id=%s: %w", id, err) } } else { - err = db.SaveState(ctx, d.ResourceKey, id, newState, d.DependsOn, deployplan.Update) + err = db.SaveState(ctx, d.ResourceKey, id, newState, d.DependsOn, dstate.OperationInfo{Action: deployplan.Update}) if err != nil { return fmt.Errorf("saving state id=%s: %w", id, err) } @@ -216,7 +217,7 @@ func (d *DeploymentUnit) UpdateWithID(ctx context.Context, db *dstate.Deployment return err } - err = db.SaveState(ctx, d.ResourceKey, newID, newState, d.DependsOn, deployplan.UpdateWithID) + err = db.SaveState(ctx, d.ResourceKey, newID, newState, d.DependsOn, dstate.OperationInfo{Action: deployplan.UpdateWithID}) if err != nil { return fmt.Errorf("saving state id=%s: %w", oldID, err) } @@ -258,7 +259,7 @@ func (d *DeploymentUnit) Delete(ctx context.Context, db *dstate.DeploymentState, } } - err = db.DeleteState(ctx, d.ResourceKey, deployplan.Delete) + err = db.DeleteState(ctx, d.ResourceKey, dstate.OperationInfo{Action: deployplan.Delete}) if err != nil { return fmt.Errorf("deleting state id=%s: %w", oldID, err) } @@ -299,7 +300,7 @@ func (d *DeploymentUnit) Resize(ctx context.Context, db *dstate.DeploymentState, return fmt.Errorf("resizing id=%s: %w", id, err) } - err = db.SaveState(ctx, d.ResourceKey, id, newState, d.DependsOn, deployplan.Resize) + err = db.SaveState(ctx, d.ResourceKey, id, newState, d.DependsOn, dstate.OperationInfo{Action: deployplan.Resize}) if err != nil { return fmt.Errorf("saving state id=%s: %w", id, err) } diff --git a/bundle/direct/bind.go b/bundle/direct/bind.go index 4de9c8d736a..8e298e7a315 100644 --- a/bundle/direct/bind.go +++ b/bundle/direct/bind.go @@ -93,7 +93,7 @@ func (b *DeploymentBundle) Bind(ctx context.Context, client *databricks.Workspac } // Save state with ID and empty state (like migrate does) - err = b.StateDB.SaveState(ctx, resourceKey, resourceID, struct{}{}, nil, deployplan.Create) + err = b.StateDB.SaveState(ctx, resourceKey, resourceID, struct{}{}, nil, dstate.OperationInfo{Action: deployplan.Create}) if err != nil { os.Remove(tmpStatePath) return nil, err @@ -151,7 +151,7 @@ func (b *DeploymentBundle) Bind(ctx context.Context, client *databricks.Workspac return nil, err } - err = b.StateDB.SaveState(ctx, resourceKey, resourceID, sv.Value, dependsOn, deployplan.Create) + err = b.StateDB.SaveState(ctx, resourceKey, resourceID, sv.Value, dependsOn, dstate.OperationInfo{Action: deployplan.Create}) if err != nil { os.Remove(tmpStatePath) return nil, err @@ -221,7 +221,7 @@ func (b *DeploymentBundle) Unbind(ctx context.Context, statePath, resourceKey st } // Delete the main resource - err = b.StateDB.DeleteState(ctx, resourceKey, deployplan.Delete) + err = b.StateDB.DeleteState(ctx, resourceKey, dstate.OperationInfo{Action: deployplan.Delete}) if err != nil { return err } @@ -235,7 +235,7 @@ func (b *DeploymentBundle) Unbind(ctx context.Context, statePath, resourceKey st for key := range b.StateDB.Data.State { if key == permissionsKey || key == grantsKey || strings.HasPrefix(key, resourceKey+".") { - err = b.StateDB.DeleteState(ctx, key, deployplan.Delete) + err = b.StateDB.DeleteState(ctx, key, dstate.OperationInfo{Action: deployplan.Delete}) if err != nil { return err } diff --git a/bundle/direct/bundle_apply.go b/bundle/direct/bundle_apply.go index 76710211c05..68e009cb413 100644 --- a/bundle/direct/bundle_apply.go +++ b/bundle/direct/bundle_apply.go @@ -8,6 +8,7 @@ import ( "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/cli/bundle/direct/dstate" "github.com/databricks/cli/bundle/terraform_dabs_map" "github.com/databricks/cli/libs/logdiag" "github.com/databricks/cli/libs/structs/structaccess" @@ -102,7 +103,7 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa if entry.Gone { // Planning confirmed the resource is already deleted remotely; only // remove it from the state, without calling the delete API. - err = b.StateDB.DeleteState(ctx, resourceKey, action) + err = b.StateDB.DeleteState(ctx, resourceKey, dstate.OperationInfo{Action: action}) } else { err = d.Destroy(ctx, &b.StateDB) } diff --git a/bundle/direct/dstate/dms.go b/bundle/direct/dstate/dms.go index 506a9869ce6..fc765aeb66a 100644 --- a/bundle/direct/dstate/dms.go +++ b/bundle/direct/dstate/dms.go @@ -25,6 +25,19 @@ type RecordedState struct { DependsOn []deployplan.DependsOnEntry `json:"depends_on,omitempty"` } +// OperationInfo is what a state write reports to the deployment metadata service. +// The caller describes the write; the sink does not infer it from the state being nil. +type OperationInfo struct { + // Action is the operation DMS records for this write. + Action deployplan.ActionType + + // InProgress marks a write that is half of a larger change, so an interrupted + // deploy does not leave the resource described as finished. The service keeps one + // operation per resource per version, so the second write updates this same + // operation to succeeded. Only a recreate's delete sets it. + InProgress bool +} + // OperationSink records one resource operation with the deployment metadata service. // SaveState and DeleteState call it for every state write, so what DMS holds mirrors // the WAL - including the intermediate writes of a recreate. @@ -32,7 +45,7 @@ type RecordedState struct { // It does not return an error: the upload happens on a background worker, and the // deploy learns about a failure when the queue is drained. type OperationSink interface { - RecordOperation(ctx context.Context, resourceKey string, action deployplan.ActionType, resourceID string, state json.RawMessage) + RecordOperation(ctx context.Context, resourceKey string, info OperationInfo, resourceID string, state json.RawMessage) } // readDMSState replaces the file-derived resource state with the state recorded in diff --git a/bundle/direct/dstate/state.go b/bundle/direct/dstate/state.go index 861686a87a7..77983f81b20 100644 --- a/bundle/direct/dstate/state.go +++ b/bundle/direct/dstate/state.go @@ -131,10 +131,10 @@ func NewDatabase(lineage string, serial int) Database { } } -// SaveState records the resource's state after action was applied to it. action is -// what the deployment metadata service reports for the write; it is ignored when the -// bundle does not record deployment history. -func (db *DeploymentState) SaveState(ctx context.Context, key, newID string, state any, dependsOn []deployplan.DependsOnEntry, action deployplan.ActionType) error { +// SaveState records the resource's state after an operation was applied to it. info +// is what the deployment metadata service reports for the write; it is ignored when +// the bundle does not record deployment history. +func (db *DeploymentState) SaveState(ctx context.Context, key, newID string, state any, dependsOn []deployplan.DependsOnEntry, info OperationInfo) error { db.AssertOpenedForWrite() db.mu.Lock() defer db.mu.Unlock() @@ -167,15 +167,15 @@ func (db *DeploymentState) SaveState(ctx context.Context, key, newID string, sta if err != nil { return err } - db.sink.RecordOperation(ctx, key, action, newID, recorded) + db.sink.RecordOperation(ctx, key, info, newID, recorded) } return nil } -// DeleteState drops the resource's state entry. action distinguishes a real delete +// DeleteState drops the resource's state entry. info distinguishes a real delete // from the intermediate drop a recreate performs, both of which are recorded. -func (db *DeploymentState) DeleteState(ctx context.Context, key string, action deployplan.ActionType) error { +func (db *DeploymentState) DeleteState(ctx context.Context, key string, info OperationInfo) error { db.AssertOpenedForWrite() db.mu.Lock() defer db.mu.Unlock() @@ -194,13 +194,8 @@ func (db *DeploymentState) DeleteState(ctx context.Context, key string, action d delete(db.stateIDs, key) // State is nil: the resource no longer exists. - // - // A recreate drops the entry and then saves the replacement. The service keeps one - // operation per resource per version, so the sink opens that one operation as - // in-progress here and the save that follows completes it - leaving an interrupted - // recreate described as mid-recreate rather than as the resource it deleted. if db.sink != nil { - db.sink.RecordOperation(ctx, key, action, deletedID, nil) + db.sink.RecordOperation(ctx, key, info, deletedID, nil) } return nil diff --git a/bundle/direct/dstate/state_test.go b/bundle/direct/dstate/state_test.go index b0dafb2933a..36256f7b26b 100644 --- a/bundle/direct/dstate/state_test.go +++ b/bundle/direct/dstate/state_test.go @@ -24,8 +24,12 @@ type fakeSink struct { ops []string } -func (f *fakeSink) RecordOperation(ctx context.Context, resourceKey string, action deployplan.ActionType, resourceID string, state json.RawMessage) { - f.ops = append(f.ops, fmt.Sprintf("%s %s id=%s state=%s", action, resourceKey, resourceID, string(state))) +func (f *fakeSink) RecordOperation(ctx context.Context, resourceKey string, info OperationInfo, resourceID string, state json.RawMessage) { + entry := fmt.Sprintf("%s %s id=%s state=%s", info.Action, resourceKey, resourceID, string(state)) + if info.InProgress { + entry += " in_progress" + } + f.ops = append(f.ops, entry) } func TestStateWritesRecordOperations(t *testing.T) { @@ -41,21 +45,21 @@ func TestStateWritesRecordOperations(t *testing.T) { // leaves the resource described as mid-recreate. name: "recreate reports both of its writes", write: func(t *testing.T, db *DeploymentState) { - require.NoError(t, db.SaveState(t.Context(), "jobs.my_job", "123", map[string]string{"key": "old"}, nil, deployplan.Create)) - require.NoError(t, db.DeleteState(t.Context(), "jobs.my_job", deployplan.Recreate)) - require.NoError(t, db.SaveState(t.Context(), "jobs.my_job", "456", map[string]string{"key": "new"}, nil, deployplan.Recreate)) + require.NoError(t, db.SaveState(t.Context(), "jobs.my_job", "123", map[string]string{"key": "old"}, nil, OperationInfo{Action: deployplan.Create})) + require.NoError(t, db.DeleteState(t.Context(), "jobs.my_job", OperationInfo{Action: deployplan.Recreate, InProgress: true})) + require.NoError(t, db.SaveState(t.Context(), "jobs.my_job", "456", map[string]string{"key": "new"}, nil, OperationInfo{Action: deployplan.Recreate})) }, want: []string{ `create jobs.my_job id=123 state={"state":{"key":"old"}}`, - `recreate jobs.my_job id=123 state=`, + `recreate jobs.my_job id=123 state= in_progress`, `recreate jobs.my_job id=456 state={"state":{"key":"new"}}`, }, }, { name: "real delete reports the id it had and no state", write: func(t *testing.T, db *DeploymentState) { - require.NoError(t, db.SaveState(t.Context(), "jobs.my_job", "123", map[string]string{}, nil, deployplan.Create)) - require.NoError(t, db.DeleteState(t.Context(), "jobs.my_job", deployplan.Delete)) + require.NoError(t, db.SaveState(t.Context(), "jobs.my_job", "123", map[string]string{}, nil, OperationInfo{Action: deployplan.Create})) + require.NoError(t, db.DeleteState(t.Context(), "jobs.my_job", OperationInfo{Action: deployplan.Delete})) }, want: []string{ `create jobs.my_job id=123 state={"state":{}}`, @@ -87,8 +91,8 @@ func TestStateWritesRecordNothingWithoutSink(t *testing.T) { // No sink: recording is off, and the writes still succeed. var db DeploymentState require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil)) - require.NoError(t, db.SaveState(t.Context(), "jobs.my_job", "123", map[string]string{}, nil, deployplan.Create)) - require.NoError(t, db.DeleteState(t.Context(), "jobs.my_job", deployplan.Delete)) + require.NoError(t, db.SaveState(t.Context(), "jobs.my_job", "123", map[string]string{}, nil, OperationInfo{Action: deployplan.Create})) + require.NoError(t, db.DeleteState(t.Context(), "jobs.my_job", OperationInfo{Action: deployplan.Delete})) mustFinalize(t, &db) } @@ -98,7 +102,7 @@ func TestOpenSaveFinalizeRoundTrip(t *testing.T) { var db DeploymentState require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil)) - require.NoError(t, db.SaveState(t.Context(), "jobs.my_job", "123", map[string]string{"key": "val"}, nil, deployplan.Create)) + require.NoError(t, db.SaveState(t.Context(), "jobs.my_job", "123", map[string]string{"key": "val"}, nil, OperationInfo{Action: deployplan.Create})) mustFinalize(t, &db) // Re-open and verify persisted data. @@ -184,7 +188,7 @@ func TestHeaderOnlyWALRecoveryDoesNotAdvanceSerial(t *testing.T) { // Commit serial 1 with one resource. var db DeploymentState require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil)) - require.NoError(t, db.SaveState(t.Context(), "jobs.my_job", "123", map[string]string{}, nil, deployplan.Create)) + require.NoError(t, db.SaveState(t.Context(), "jobs.my_job", "123", map[string]string{}, nil, OperationInfo{Action: deployplan.Create})) mustFinalize(t, &db) var committed DeploymentState @@ -248,12 +252,12 @@ func TestDeleteState(t *testing.T) { var db DeploymentState require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil)) - require.NoError(t, db.SaveState(t.Context(), "jobs.my_job", "123", map[string]string{}, nil, deployplan.Create)) + require.NoError(t, db.SaveState(t.Context(), "jobs.my_job", "123", map[string]string{}, nil, OperationInfo{Action: deployplan.Create})) mustFinalize(t, &db) var db2 DeploymentState require.NoError(t, db2.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil)) - require.NoError(t, db2.DeleteState(t.Context(), "jobs.my_job", deployplan.Delete)) + require.NoError(t, db2.DeleteState(t.Context(), "jobs.my_job", OperationInfo{Action: deployplan.Delete})) mustFinalize(t, &db2) var db3 DeploymentState @@ -281,7 +285,7 @@ func TestGetOrInitLineageReadableBeforeWriteAndPersisted(t *testing.T) { // Upgrading to write reuses the same lineage (it goes into the WAL header), // and a write makes it durable. require.NoError(t, db.UpgradeToWrite()) - require.NoError(t, db.SaveState(t.Context(), "jobs.my_job", "123", map[string]string{}, nil, deployplan.Create)) + require.NoError(t, db.SaveState(t.Context(), "jobs.my_job", "123", map[string]string{}, nil, OperationInfo{Action: deployplan.Create})) mustFinalize(t, &db) // Re-open: the persisted lineage matches the one read before the write. diff --git a/bundle/direct/opqueue.go b/bundle/direct/opqueue.go index bb6c028e07f..ba9213a7322 100644 --- a/bundle/direct/opqueue.go +++ b/bundle/direct/opqueue.go @@ -7,6 +7,7 @@ import ( "sync" "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/cli/bundle/direct/dstate" "github.com/databricks/cli/libs/log" ) @@ -83,20 +84,12 @@ func newOperationQueue(ctx context.Context, uploader operationUploader) *operati // // An earlier upload failure does not stop this: every write is still recorded, best // effort, so DMS ends up as close to reality as it can get. -func (q *operationQueue) RecordOperation(ctx context.Context, resourceKey string, action deployplan.ActionType, resourceID string, state json.RawMessage) { +func (q *operationQueue) RecordOperation(ctx context.Context, resourceKey string, info dstate.OperationInfo, resourceID string, state json.RawMessage) { if q == nil { return } - // A recreate drops the state entry before creating the replacement. Recorded as - // in-progress, so an interrupted recreate does not leave the resource described as - // the one it just deleted; the create that follows updates it to succeeded. - if action == deployplan.Recreate && state == nil { - q.enqueue(ctx, resourceKey, newInProgressOperation()) - return - } - - op, err := newStateOperation(action, resourceID, state) + op, err := newStateOperation(info, resourceID, state) if err != nil { // The deploy already persisted this write locally, so failing it here would // report an error about history for a resource that deployed fine. diff --git a/bundle/direct/opqueue_test.go b/bundle/direct/opqueue_test.go index 1048db35d9c..8ad98b31017 100644 --- a/bundle/direct/opqueue_test.go +++ b/bundle/direct/opqueue_test.go @@ -87,7 +87,7 @@ func envelope(t *testing.T, name string) json.RawMessage { func recordState(t *testing.T, q *operationQueue, resourceKey, name string) { t.Helper() - q.RecordOperation(t.Context(), resourceKey, deployplan.Update, "id-1", envelope(t, name)) + q.RecordOperation(t.Context(), resourceKey, dstate.OperationInfo{Action: deployplan.Update}, "id-1", envelope(t, name)) } func TestOperationQueueUploadsEachOperation(t *testing.T) { @@ -150,8 +150,8 @@ func TestOperationQueueUploadsQueuedWritesWhileWorkersAreBusy(t *testing.T) { // A resource whose ID is only known after it was created: the first write has no // ID, the second fills it in. - q.RecordOperation(t.Context(), "resources.jobs.foo", deployplan.Create, "", envelope(t, "created")) - q.RecordOperation(t.Context(), "resources.jobs.foo", deployplan.Create, "id-1", envelope(t, "updated")) + q.RecordOperation(t.Context(), "resources.jobs.foo", dstate.OperationInfo{Action: deployplan.Create}, "", envelope(t, "created")) + q.RecordOperation(t.Context(), "resources.jobs.foo", dstate.OperationInfo{Action: deployplan.Create}, "id-1", envelope(t, "updated")) close(f.block) require.NoError(t, q.close()) @@ -181,11 +181,11 @@ func TestOperationQueueRecordDuringUploadIsStillUploaded(t *testing.T) { f := &fakeUploader{block: make(chan struct{}), started: make(chan string, 1)} q := newOperationQueue(t.Context(), f) - q.RecordOperation(t.Context(), "resources.jobs.foo", deployplan.Create, "", envelope(t, "v1")) + q.RecordOperation(t.Context(), "resources.jobs.foo", dstate.OperationInfo{Action: deployplan.Create}, "", envelope(t, "v1")) assert.Equal(t, "resources.jobs.foo", <-f.started) // The worker has taken the key off the queue and is uploading v1 right now. - q.RecordOperation(t.Context(), "resources.jobs.foo", deployplan.Create, "id-1", envelope(t, "v2")) + q.RecordOperation(t.Context(), "resources.jobs.foo", dstate.OperationInfo{Action: deployplan.Create}, "id-1", envelope(t, "v2")) close(f.block) require.NoError(t, q.close()) @@ -223,11 +223,11 @@ func TestOperationQueueKeepsRecordingAfterUploadError(t *testing.T) { // Wait for the failing upload to finish, so the error is stored before the next // record rather than racing it. - q.RecordOperation(t.Context(), "resources.jobs.foo", deployplan.Create, "id-1", envelope(t, "v1")) + q.RecordOperation(t.Context(), "resources.jobs.foo", dstate.OperationInfo{Action: deployplan.Create}, "id-1", envelope(t, "v1")) assert.Equal(t, "resources.jobs.foo", <-f.done) // The next resource is still accepted, even though the first upload failed. - q.RecordOperation(t.Context(), "resources.jobs.bar", deployplan.Create, "id-2", envelope(t, "v1")) + q.RecordOperation(t.Context(), "resources.jobs.bar", dstate.OperationInfo{Action: deployplan.Create}, "id-2", envelope(t, "v1")) // Both were attempted, and close still reports the failure so the deploy fails. require.ErrorIs(t, q.close(), uploadErr) @@ -248,10 +248,10 @@ func TestOperationQueueDrainsQueuedOperationsAfterUploadError(t *testing.T) { // Every worker is parked mid-upload, so these stay queued. for i := range operationUploadWorkers { - q.RecordOperation(t.Context(), "resources.jobs.hold"+strconv.Itoa(i), deployplan.Create, "id-1", envelope(t, "v1")) + q.RecordOperation(t.Context(), "resources.jobs.hold"+strconv.Itoa(i), dstate.OperationInfo{Action: deployplan.Create}, "id-1", envelope(t, "v1")) assert.Equal(t, "resources.jobs.hold"+strconv.Itoa(i), <-f.started) } - q.RecordOperation(t.Context(), "resources.jobs.queued", deployplan.Create, "id-2", envelope(t, "v1")) + q.RecordOperation(t.Context(), "resources.jobs.queued", dstate.OperationInfo{Action: deployplan.Create}, "id-2", envelope(t, "v1")) close(f.block) require.ErrorIs(t, q.close(), uploadErr) @@ -267,7 +267,7 @@ func TestOperationQueueRecordDropsUnsupportedAction(t *testing.T) { // The state write already succeeded, so an operation that cannot be described is // dropped with a warning rather than failing the deploy. - q.RecordOperation(t.Context(), "resources.jobs.foo", deployplan.Skip, "id-1", nil) + q.RecordOperation(t.Context(), "resources.jobs.foo", dstate.OperationInfo{Action: deployplan.Skip}, "id-1", nil) require.NoError(t, q.close()) assert.Empty(t, f.recorded()) @@ -277,7 +277,7 @@ func TestOperationQueueRecordDropsOversizedState(t *testing.T) { f := &fakeUploader{} q := newOperationQueue(t.Context(), f) - q.RecordOperation(t.Context(), "resources.jobs.foo", deployplan.Create, "id-1", envelope(t, strings.Repeat("x", maxOperationStateSize))) + q.RecordOperation(t.Context(), "resources.jobs.foo", dstate.OperationInfo{Action: deployplan.Create}, "id-1", envelope(t, strings.Repeat("x", maxOperationStateSize))) require.NoError(t, q.close()) assert.Empty(t, f.recorded()) @@ -353,7 +353,7 @@ func TestOperationQueueUploadsOneResourceAtATime(t *testing.T) { wg.Go(func() { for i := range perWorker { key := "resources.jobs.job" + strconv.Itoa((w*perWorker+i)%distinctKeyMod) - q.RecordOperation(ctx, key, deployplan.Update, "id-1", states[w]) + q.RecordOperation(ctx, key, dstate.OperationInfo{Action: deployplan.Update}, "id-1", states[w]) } }) } @@ -373,6 +373,6 @@ func TestNilOperationQueueIsNoOp(t *testing.T) { // no-op, so Apply does not have to branch. q := newOperationQueue(t.Context(), nil) require.Nil(t, q) - q.RecordOperation(t.Context(), "resources.jobs.foo", deployplan.Create, "id-1", nil) + q.RecordOperation(t.Context(), "resources.jobs.foo", dstate.OperationInfo{Action: deployplan.Create}, "id-1", nil) require.NoError(t, q.close()) } diff --git a/bundle/direct/oprecorder.go b/bundle/direct/oprecorder.go index 1bcdc1c9aa1..775ae930998 100644 --- a/bundle/direct/oprecorder.go +++ b/bundle/direct/oprecorder.go @@ -24,9 +24,8 @@ const maxOperationStateSize = 64 * 1024 // failing and masking the error we are trying to report. const maxOperationErrorMessageSize = 16 * 1024 -// operationStatusInProgress marks an operation whose writes are not finished. A -// recreate opens one after its delete and updates it to succeeded once the create -// lands, so an interrupted recreate is not left describing the resource it deleted. +// operationStatusInProgress marks an operation whose writes are not finished; see +// dstate.OperationInfo.InProgress for when a write asks for it. // // Declared here rather than used from the SDK: the enum value is generated from the // OpenAPI spec, which trails the service proto (databricks-eng/universe#2394529). @@ -56,8 +55,8 @@ type recordedOperation struct { // newStateOperation describes a state write for upload. state is the serialized // RecordedState envelope the state DB just persisted, and nil for a delete, where // the resource is gone. It errors when the state exceeds maxOperationStateSize. -func newStateOperation(action deployplan.ActionType, resourceID string, state json.RawMessage) (recordedOperation, error) { - actionType, err := deployActionToSDK(action) +func newStateOperation(info dstate.OperationInfo, resourceID string, state json.RawMessage) (recordedOperation, error) { + actionType, err := deployActionToSDK(info.Action) if err != nil { return recordedOperation{}, err } @@ -66,24 +65,19 @@ func newStateOperation(action deployplan.ActionType, resourceID string, state js return recordedOperation{}, fmt.Errorf("serialized state is %d bytes, which exceeds the %d byte limit for recording deployment history", len(state), maxOperationStateSize) } + status := bundledeployments.OperationStatusOperationStatusSucceeded + if info.InProgress { + status = operationStatusInProgress + } + return recordedOperation{ action: actionType, resourceID: resourceID, - status: bundledeployments.OperationStatusOperationStatusSucceeded, + status: status, state: state, }, nil } -// newInProgressOperation opens a recreate before its create half has run. It carries -// no state: the old resource is deleted and the new one does not exist yet, so there -// is nothing that exists to describe. -func newInProgressOperation() recordedOperation { - return recordedOperation{ - action: bundledeployments.OperationActionTypeOperationActionTypeRecreate, - status: operationStatusInProgress, - } -} - // newFailedOperation records an operation that did not apply, so the deployment // history says why a resource failed rather than just omitting it. // diff --git a/bundle/direct/oprecorder_test.go b/bundle/direct/oprecorder_test.go index d80caa09341..d368fe47c9d 100644 --- a/bundle/direct/oprecorder_test.go +++ b/bundle/direct/oprecorder_test.go @@ -9,6 +9,7 @@ import ( "testing" "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/cli/bundle/direct/dstate" "github.com/databricks/databricks-sdk-go/service/bundledeployments" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -48,7 +49,7 @@ func (f *fakeOpClient) UpdateOperation(ctx context.Context, parent, resourceKey // an operationQueue worker does. func uploadOne(t *testing.T, u operationUploader, resourceKey string, action deployplan.ActionType, resourceID string, state json.RawMessage) { t.Helper() - op, err := newStateOperation(action, resourceID, state) + op, err := newStateOperation(dstate.OperationInfo{Action: action}, resourceID, state) require.NoError(t, err) require.NoError(t, u.upload(t.Context(), resourceKey, op)) } @@ -109,7 +110,7 @@ func TestNewStateOperationRecordsEnvelopeAsIs(t *testing.T) { // carries it through untouched, sensitive fields and all. state := json.RawMessage(`{"state":{"name":"foo","token":"super-secret"}}`) - op, err := newStateOperation(deployplan.Create, "job-123", state) + op, err := newStateOperation(dstate.OperationInfo{Action: deployplan.Create}, "job-123", state) require.NoError(t, err) assert.JSONEq(t, string(state), string(op.state)) @@ -117,14 +118,14 @@ func TestNewStateOperationRecordsEnvelopeAsIs(t *testing.T) { } func TestNewStateOperationRejectsUnsupportedAction(t *testing.T) { - _, err := newStateOperation(deployplan.Skip, "job-123", nil) + _, err := newStateOperation(dstate.OperationInfo{Action: deployplan.Skip}, "job-123", nil) assert.Error(t, err) } func TestNewStateOperationRejectsOversizedState(t *testing.T) { big := json.RawMessage(strings.Repeat("x", maxOperationStateSize+1)) - _, err := newStateOperation(deployplan.Create, "job-123", big) + _, err := newStateOperation(dstate.OperationInfo{Action: deployplan.Create}, "job-123", big) assert.ErrorContains(t, err, "exceeds the 65536 byte limit") } diff --git a/bundle/migrate/build_state.go b/bundle/migrate/build_state.go index 459c14a0b6d..93dc65e3363 100644 --- a/bundle/migrate/build_state.go +++ b/bundle/migrate/build_state.go @@ -235,7 +235,7 @@ func BuildStateFromTF( // Migration rebuilds local state from terraform's; nothing is deployed, and // the DMS sink is never set on this state, so the action is not reported. - if err := stateDB.SaveState(ctx, node, id, sv.Value, dependsOn, deployplan.Create); err != nil { + if err := stateDB.SaveState(ctx, node, id, sv.Value, dependsOn, dstate.OperationInfo{Action: deployplan.Create}); err != nil { return warningsSeen, fmt.Errorf("%s: SaveState: %w", node, err) } } From 5edff67517bd0596fba8b7ca985dff82c9d1e84a Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 10 Aug 2026 22:11:44 +0000 Subject: [PATCH 073/125] bundle: merge a resource's queued operations into one upload Writes that piled up behind an in-flight upload were each uploaded, so a resource cost one request per state write. Only the newest write describes the resource as it now stands, so merge them and send one. The merged upload keeps the OLDER write's action_type. The service fixes that field when the operation is created and rejects it in an update mask - only state, error_message, resource_id and status are updatable - so taking the newer action would report a recreate as a plain create. Everything else comes from the newer write, which keeps status and error_message paired the way the service requires. A failed create still leaves the operation unrecorded, so the next upload for the resource creates rather than updates; nothing has to fall back to a PATCH. Co-authored-by: Isaac --- bundle/direct/opqueue.go | 49 ++++++------- bundle/direct/opqueue_test.go | 127 ++++++++++++++++++++++++++++------ bundle/direct/oprecorder.go | 14 ++++ 3 files changed, 143 insertions(+), 47 deletions(-) diff --git a/bundle/direct/opqueue.go b/bundle/direct/opqueue.go index ba9213a7322..4ee3712008c 100644 --- a/bundle/direct/opqueue.go +++ b/bundle/direct/opqueue.go @@ -25,8 +25,9 @@ const ( // // - One resource, one upload at a time. DMS keeps a single state per resource, so // overlapping uploads could land out of order and leave the older state. -// - Newest operation wins. Each carries the resource's full state, so a queued -// operation superseded by a newer one is dropped ("coalesced"). +// - Newest operation wins. Each write carries the resource's full state, so writes +// that pile up behind an in-flight upload are merged into one ("coalesced") and +// the resource costs a single request instead of one per write. // // close reports the first upload failure, which fails the deploy: DMS becomes the // source of truth (see dstate.readDMSState), so a missing record would make the @@ -42,11 +43,11 @@ type operationQueue struct { // mu guards the fields below. mu sync.Mutex - // pending holds the operations waiting per resource key, oldest first. Every one - // is uploaded: a resource can write state more than once in a deploy (a recreate - // drops it, then saves the new resource), and each write is its own event, so - // dropping the older one would hide a step. No key means nothing is waiting. - pending map[string][]recordedOperation + // pending holds the one operation waiting per resource key: a resource can write + // state more than once in a deploy (a recreate drops the entry, then saves the new + // resource), and writes that arrive before the previous one is uploaded are merged + // by mergeOperation. No key means nothing is waiting. + pending map[string]recordedOperation // queuedOrUploading means "some worker will get to this key". Recording such a // key writes to pending only, so two workers never upload one resource at once. @@ -66,7 +67,7 @@ func newOperationQueue(ctx context.Context, uploader operationUploader) *operati q := &operationQueue{ uploader: uploader, queue: make(chan string, operationQueueSize), - pending: make(map[string][]recordedOperation), + pending: make(map[string]recordedOperation), queuedOrUploading: make(map[string]bool), } @@ -117,17 +118,20 @@ func (q *operationQueue) recordFailure(ctx context.Context, resourceKey string, q.enqueue(ctx, resourceKey, op) } -// enqueue appends op to the operations waiting for resourceKey and makes sure a -// worker will pick it up. +// enqueue makes op the operation waiting for resourceKey, merged onto whatever was +// already waiting, and makes sure a worker will pick it up. func (q *operationQueue) enqueue(ctx context.Context, resourceKey string, op recordedOperation) { q.mu.Lock() - q.pending[resourceKey] = append(q.pending[resourceKey], op) + if waiting, ok := q.pending[resourceKey]; ok { + op = mergeOperation(waiting, op) + } + q.pending[resourceKey] = op alreadyHandled := q.queuedOrUploading[resourceKey] q.queuedOrUploading[resourceKey] = true q.mu.Unlock() // A worker will re-read pending before it finishes, so it picks up the operation - // appended above. Queueing again would let a second worker upload the same key. + // stored above. Queueing again would let a second worker upload the same key. if alreadyHandled { return } @@ -176,26 +180,23 @@ func (q *operationQueue) work(ctx context.Context) { } } -// take claims the oldest operation waiting for resourceKey, reporting false and -// clearing the queuedOrUploading mark when nothing is left, which lets record queue -// it again. Both happen under one lock, so a key can never be left for no worker to -// pick up. +// take claims the operation waiting for resourceKey, reporting false and clearing the +// queuedOrUploading mark when nothing is left, which lets record queue it again. Both +// happen under one lock, so a key can never be left for no worker to pick up. func (q *operationQueue) take(resourceKey string) (recordedOperation, bool) { q.mu.Lock() defer q.mu.Unlock() - ops := q.pending[resourceKey] - if len(ops) == 0 { - delete(q.pending, resourceKey) + op, ok := q.pending[resourceKey] + if !ok { delete(q.queuedOrUploading, resourceKey) return recordedOperation{}, false } - // Oldest first, so the service sees the writes in the order they happened. The - // mark stays until the branch above clears it, so anything recorded during this - // upload is still picked up and no second worker takes the key meanwhile. - q.pending[resourceKey] = ops[1:] - return ops[0], true + // The mark stays until the branch above clears it, so anything recorded during + // this upload is still picked up and no second worker takes the key meanwhile. + delete(q.pending, resourceKey) + return op, true } // setErr keeps the first upload error; later ones are dropped because one failure diff --git a/bundle/direct/opqueue_test.go b/bundle/direct/opqueue_test.go index 8ad98b31017..9e14fd11a1a 100644 --- a/bundle/direct/opqueue_test.go +++ b/bundle/direct/opqueue_test.go @@ -27,10 +27,12 @@ type fakeUploader struct { done chan string err error - mu sync.Mutex - uploads []string - actions map[string]bundledeployments.OperationActionType - resourceIDs map[string]string + mu sync.Mutex + uploads []string + actions map[string]bundledeployments.OperationActionType + resourceIDs map[string]string + statuses map[string]bundledeployments.OperationStatus + errorMessages map[string]string } func (f *fakeUploader) upload(ctx context.Context, resourceKey string, op recordedOperation) error { @@ -46,9 +48,13 @@ func (f *fakeUploader) upload(ctx context.Context, resourceKey string, op record if f.actions == nil { f.actions = map[string]bundledeployments.OperationActionType{} f.resourceIDs = map[string]string{} + f.statuses = map[string]bundledeployments.OperationStatus{} + f.errorMessages = map[string]string{} } f.actions[resourceKey] = op.action f.resourceIDs[resourceKey] = op.resourceID + f.statuses[resourceKey] = op.status + f.errorMessages[resourceKey] = op.errorMessage f.mu.Unlock() // Sent outside the lock: a test that stops reading this channel would otherwise @@ -77,6 +83,30 @@ func (f *fakeUploader) resourceIDFor(resourceKey string) string { return f.resourceIDs[resourceKey] } +func (f *fakeUploader) statusFor(resourceKey string) bundledeployments.OperationStatus { + f.mu.Lock() + defer f.mu.Unlock() + return f.statuses[resourceKey] +} + +func (f *fakeUploader) errorMessageFor(resourceKey string) string { + f.mu.Lock() + defer f.mu.Unlock() + return f.errorMessages[resourceKey] +} + +// uploadsFor returns the uploads recorded for one resource key, for tests where +// other resources are uploaded alongside it. +func uploadsFor(f *fakeUploader, resourceKey string) []string { + var out []string + for _, u := range f.recorded() { + if strings.HasPrefix(u, resourceKey+"=") { + out = append(out, u) + } + } + return out +} + // envelope builds the serialized RecordedState the state DB hands the queue. func envelope(t *testing.T, name string) json.RawMessage { t.Helper() @@ -102,14 +132,14 @@ func TestOperationQueueUploadsEachOperation(t *testing.T) { assert.Len(t, f.recorded(), 20) } -func TestOperationQueueUploadsEveryWriteForSameResource(t *testing.T) { - // Hold the first upload so the writes behind it queue up. Each one is its own - // event, so all three are uploaded, oldest first - a resource can legitimately - // write state several times in one deploy (see Recreate). +func TestOperationQueueMergesWritesQueuedBehindAnUpload(t *testing.T) { + // Hold the first upload so the writes behind it queue up. The two that pile up + // merge into one carrying the newest state, so the resource costs two requests + // rather than three - the in-flight one, then everything after it. // - // started is buffered for all three: every write now uploads, and a worker - // blocking on an unread send would deadlock the drain below. - f := &fakeUploader{block: make(chan struct{}), started: make(chan string, 3)} + // started is buffered for both uploads: a worker blocking on an unread send + // would deadlock the drain below. + f := &fakeUploader{block: make(chan struct{}), started: make(chan string, 2)} q := newOperationQueue(t.Context(), f) recordState(t, q, "resources.jobs.foo", "v1") @@ -125,16 +155,50 @@ func TestOperationQueueUploadsEveryWriteForSameResource(t *testing.T) { assert.Equal(t, []string{ `resources.jobs.foo={"state":{"name":"v1"}}`, - `resources.jobs.foo={"state":{"name":"v2"}}`, `resources.jobs.foo={"state":{"name":"v3"}}`, }, f.recorded()) } -func TestOperationQueueUploadsQueuedWritesWhileWorkersAreBusy(t *testing.T) { +func TestOperationQueueMergedRecreateKeepsItsActionType(t *testing.T) { + // A recreate records its intermediate delete, then the create that replaces the + // resource. Merging them must upload the recreate's action: the service fixes + // action_type when the operation is created and rejects it in an update mask, so + // taking the newer create's action would report the resource as merely created. + // + // Every worker is parked so both writes land in pending and merge before upload. + f := &fakeUploader{block: make(chan struct{}), started: make(chan string, operationUploadWorkers+2)} + q := newOperationQueue(t.Context(), f) + + for i := range operationUploadWorkers { + recordState(t, q, "resources.jobs.hold"+strconv.Itoa(i), "v1") + assert.Equal(t, "resources.jobs.hold"+strconv.Itoa(i), <-f.started) + } + + q.RecordOperation(t.Context(), "resources.jobs.foo", dstate.OperationInfo{Action: deployplan.Recreate, InProgress: true}, "old-id", nil) + q.RecordOperation(t.Context(), "resources.jobs.foo", dstate.OperationInfo{Action: deployplan.Create}, "new-id", envelope(t, "replacement")) + + close(f.block) + require.NoError(t, q.close()) + + assert.Equal(t, + bundledeployments.OperationActionTypeOperationActionTypeRecreate, + f.actionFor("resources.jobs.foo")) + // The newer write's own fields still win: the replacement's id, state and its + // succeeded status, not the in-progress the delete asked for. + assert.Equal(t, "new-id", f.resourceIDFor("resources.jobs.foo")) + assert.Equal(t, []string{ + `resources.jobs.foo={"state":{"name":"replacement"}}`, + }, uploadsFor(f, "resources.jobs.foo")) + assert.Equal(t, + bundledeployments.OperationStatusOperationStatusSucceeded, + f.statusFor("resources.jobs.foo")) +} + +func TestOperationQueueMergesQueuedWritesWhileWorkersAreBusy(t *testing.T) { // Every worker is parked mid-upload, so the writes below sit in pending rather - // than being picked up. Both still go out, in order, once a worker frees up. + // than being picked up. They merge into one upload carrying the newest state. // - // started is buffered for the two foo writes as well: nothing reads it after the + // started is buffered for the merged foo write as well: nothing reads it after the // loop below, and a worker blocking on the send would deadlock the drain. f := &fakeUploader{block: make(chan struct{}), started: make(chan string, operationUploadWorkers+2)} q := newOperationQueue(t.Context(), f) @@ -156,16 +220,9 @@ func TestOperationQueueUploadsQueuedWritesWhileWorkersAreBusy(t *testing.T) { close(f.block) require.NoError(t, q.close()) - var uploadsForFoo []string - for _, u := range f.recorded() { - if strings.HasPrefix(u, "resources.jobs.foo=") { - uploadsForFoo = append(uploadsForFoo, u) - } - } assert.Equal(t, []string{ - `resources.jobs.foo={"state":{"name":"created"}}`, `resources.jobs.foo={"state":{"name":"updated"}}`, - }, uploadsForFoo) + }, uploadsFor(f, "resources.jobs.foo")) // The ID recorded last is the one the create learned. assert.Equal(t, "id-1", f.resourceIDFor("resources.jobs.foo")) @@ -201,6 +258,30 @@ func TestOperationQueueRecordDuringUploadIsStillUploaded(t *testing.T) { assert.Empty(t, q.queuedOrUploading) } +func TestOperationQueueMergedFailureKeepsStatusAndMessageTogether(t *testing.T) { + // The service rejects error_message unless the status is failed, so a merge must + // not mix the newer status with the older message or vice versa. A resource that + // writes state and then fails is the sequence that would expose it. + f := &fakeUploader{block: make(chan struct{}), started: make(chan string, operationUploadWorkers+2)} + q := newOperationQueue(t.Context(), f) + + for i := range operationUploadWorkers { + recordState(t, q, "resources.jobs.hold"+strconv.Itoa(i), "v1") + assert.Equal(t, "resources.jobs.hold"+strconv.Itoa(i), <-f.started) + } + + recordState(t, q, "resources.jobs.foo", "v1") + q.recordFailure(t.Context(), "resources.jobs.foo", deployplan.Update, "id-1", envelope(t, "prior"), errors.New("boom")) + + close(f.block) + require.NoError(t, q.close()) + + assert.Equal(t, + bundledeployments.OperationStatusOperationStatusFailed, + f.statusFor("resources.jobs.foo")) + assert.Equal(t, "boom", f.errorMessageFor("resources.jobs.foo")) +} + func TestOperationQueueReturnsUploadError(t *testing.T) { uploadErr := errors.New("boom") f := &fakeUploader{err: uploadErr} diff --git a/bundle/direct/oprecorder.go b/bundle/direct/oprecorder.go index 775ae930998..87c6e48bb9c 100644 --- a/bundle/direct/oprecorder.go +++ b/bundle/direct/oprecorder.go @@ -52,6 +52,20 @@ type recordedOperation struct { state json.RawMessage } +// mergeOperation folds a newer write onto one still waiting to be uploaded, so a +// resource costs one request no matter how many times it writes state. The newer +// write describes the resource as it now stands, so its fields win. +// +// The action is the exception: it comes from the older write, because the service +// fixes action_type when the operation is created and rejects it in an update mask +// (only state, error_message, resource_id and status are updatable). Keeping the +// older one is what makes the merged upload a single create that still reports how +// the resource got here - a recreate whose second write is a create stays a recreate. +func mergeOperation(older, newer recordedOperation) recordedOperation { + newer.action = older.action + return newer +} + // newStateOperation describes a state write for upload. state is the serialized // RecordedState envelope the state DB just persisted, and nil for a delete, where // the resource is gone. It errors when the state exceeds maxOperationStateSize. From ac37a34360f1be538a5a8d52663d97141f0017a0 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 10 Aug 2026 22:55:38 +0000 Subject: [PATCH 074/125] acceptance: refresh generated files after the merge Four tests main added need the DATABRICKS_BUNDLE_DMS matrix line this branch adds to the bundle suite, and yamlfmt wants no inner spaces in a flow mapping. Co-authored-by: Isaac --- .../destroy/lineage-mismatch-after-redeploy/out.test.toml | 1 + acceptance/bundle/dms/emptied-resource/databricks.yml | 2 +- .../drift/recreated_same_name/out.test.toml | 1 + .../drift/telemetry_config_unmanaged/out.test.toml | 1 + .../drift/telemetry_config_with_config_update/out.test.toml | 1 + 5 files changed, 5 insertions(+), 1 deletion(-) diff --git a/acceptance/bundle/destroy/lineage-mismatch-after-redeploy/out.test.toml b/acceptance/bundle/destroy/lineage-mismatch-after-redeploy/out.test.toml index 0938e678987..51a5602947c 100644 --- a/acceptance/bundle/destroy/lineage-mismatch-after-redeploy/out.test.toml +++ b/acceptance/bundle/destroy/lineage-mismatch-after-redeploy/out.test.toml @@ -1,2 +1,3 @@ Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/dms/emptied-resource/databricks.yml b/acceptance/bundle/dms/emptied-resource/databricks.yml index 9bd148025cb..157b10f9050 100644 --- a/acceptance/bundle/dms/emptied-resource/databricks.yml +++ b/acceptance/bundle/dms/emptied-resource/databricks.yml @@ -9,4 +9,4 @@ resources: foo: name: dms_emptied_resource catalog_name: main - grants: [{ principal: someone@example.com, privileges: [USE_SCHEMA] }] + grants: [{principal: someone@example.com, privileges: [USE_SCHEMA]}] diff --git a/acceptance/bundle/resources/model_serving_endpoints/drift/recreated_same_name/out.test.toml b/acceptance/bundle/resources/model_serving_endpoints/drift/recreated_same_name/out.test.toml index 8c52d40aa2d..8197bd50f28 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/drift/recreated_same_name/out.test.toml +++ b/acceptance/bundle/resources/model_serving_endpoints/drift/recreated_same_name/out.test.toml @@ -1,3 +1,4 @@ Cloud = true RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_unmanaged/out.test.toml b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_unmanaged/out.test.toml index 0938e678987..51a5602947c 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_unmanaged/out.test.toml +++ b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_unmanaged/out.test.toml @@ -1,2 +1,3 @@ Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_with_config_update/out.test.toml b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_with_config_update/out.test.toml index 0938e678987..51a5602947c 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_with_config_update/out.test.toml +++ b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_with_config_update/out.test.toml @@ -1,2 +1,3 @@ Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] From c0e796ec40f713ca88b99bfb877ce33b3df46cc9 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 10 Aug 2026 23:05:50 +0000 Subject: [PATCH 075/125] bundle: rename the recording env var, and only honour "true" DATABRICKS_BUNDLE_DMS said which service does the work rather than what the flag turns on; DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY matches the config field it stands in for. env.DMS becomes unexported: RecordsDeploymentHistory is the single predicate callers should use. The value check is now an allow-list. A deny-list read "TRUE" and "yes" as on, which is the wrong default for a feature that is still gated off. Co-authored-by: Isaac --- .../empty_code_source/out.test.toml | 2 +- .../local_code_source/out.test.toml | 2 +- acceptance/bundle/apps/app_yaml/out.test.toml | 2 +- .../artifact_and_app_same_path/out.test.toml | 2 +- .../bundle/apps/compute_size/out.test.toml | 2 +- .../bundle/apps/delete_deleting/out.test.toml | 2 +- .../bundle/apps/git_source/out.test.toml | 2 +- .../bundle/apps/job_permissions/out.test.toml | 2 +- .../job_permissions_warning/out.test.toml | 2 +- .../apps/value_from_warning/out.test.toml | 2 +- .../ai_runtime_code_source/out.test.toml | 2 +- .../volume_doesnot_exist/out.test.toml | 2 +- .../volume_not_deployed/out.test.toml | 2 +- .../artifact_upload_for_volumes/out.test.toml | 2 +- .../out.test.toml | 2 +- .../out.test.toml | 2 +- .../artifacts_dynamic_version/out.test.toml | 2 +- .../artifacts/build_and_files/out.test.toml | 2 +- .../build_and_files_whl/out.test.toml | 2 +- .../artifacts/glob_exact_whl/out.test.toml | 2 +- .../artifacts/globs_in_files/out.test.toml | 2 +- .../globs_in_files_in_include/out.test.toml | 2 +- .../artifacts/globs_invalid/out.test.toml | 2 +- .../bundle/artifacts/issue_3109/out.test.toml | 2 +- .../artifacts/nil_artifacts/out.test.toml | 2 +- .../same_name_libraries/out.test.toml | 2 +- .../bundle/artifacts/shell/bash/out.test.toml | 2 +- .../artifacts/shell/basic/out.test.toml | 2 +- .../bundle/artifacts/shell/cmd/out.test.toml | 2 +- .../artifacts/shell/default/out.test.toml | 2 +- .../artifacts/shell/err-bash/out.test.toml | 2 +- .../artifacts/shell/err-sh/out.test.toml | 2 +- .../artifacts/shell/invalid/out.test.toml | 2 +- .../bundle/artifacts/shell/sh/out.test.toml | 2 +- .../unique_name_libraries/out.test.toml | 2 +- .../upload_multiple_libraries/out.test.toml | 2 +- .../whl_change_version/out.test.toml | 2 +- .../bundle/artifacts/whl_dbfs/out.test.toml | 2 +- .../artifacts/whl_dynamic/out.test.toml | 2 +- .../artifacts/whl_explicit/out.test.toml | 2 +- .../artifacts/whl_implicit/out.test.toml | 2 +- .../whl_implicit_custom_path/out.test.toml | 2 +- .../whl_implicit_notebook/out.test.toml | 2 +- .../artifacts/whl_multiple/out.test.toml | 2 +- .../artifacts/whl_no_cleanup/out.test.toml | 2 +- .../whl_prebuilt_multiple/out.test.toml | 2 +- .../whl_prebuilt_outside/out.test.toml | 2 +- .../out.test.toml | 2 +- .../whl_via_environment_key/out.test.toml | 2 +- .../bundle/benchmarks/deploy/out.test.toml | 2 +- .../bundle/benchmarks/plan/out.test.toml | 2 +- .../bundle/benchmarks/validate/out.test.toml | 2 +- acceptance/bundle/bundle_tag/id/out.test.toml | 2 +- .../bundle/bundle_tag/url/out.test.toml | 2 +- .../bundle/bundle_tag/url_ref/out.test.toml | 2 +- .../cli_defaults/out.test.toml | 2 +- .../config_edits/out.test.toml | 2 +- .../dashboard_etag/out.test.toml | 2 +- .../flushed_cache/out.test.toml | 2 +- .../formatting_preserved/out.test.toml | 2 +- .../job_fields/out.test.toml | 2 +- .../job_multiple_tasks/out.test.toml | 2 +- .../job_params_variables/out.test.toml | 2 +- .../job_pipeline_task/out.test.toml | 2 +- .../multiple_files/out.test.toml | 2 +- .../multiple_resources/out.test.toml | 2 +- .../output_json/out.test.toml | 2 +- .../output_no_changes/out.test.toml | 2 +- .../pipeline_fields/out.test.toml | 2 +- .../out.test.toml | 2 +- .../resolve_variables/out.test.toml | 2 +- .../select_basic/out.test.toml | 2 +- .../select_multiple/out.test.toml | 2 +- .../skip_permissions/out.test.toml | 2 +- .../cli_default_split_element/out.test.toml | 2 +- .../split/dotted_target/out.test.toml | 2 +- .../split/isolation/out.test.toml | 2 +- .../split/keyed_edit/out.test.toml | 2 +- .../split/keyed_remove/out.test.toml | 2 +- .../split/keyed_rename/out.test.toml | 2 +- .../split/keyed_twoblock/out.test.toml | 2 +- .../split/multifile/out.test.toml | 2 +- .../nested_add_split_parent/out.test.toml | 2 +- .../split/nested_sequence/out.test.toml | 2 +- .../split/positional/out.test.toml | 2 +- .../remove_field_both_blocks/out.test.toml | 2 +- .../remove_with_unrelated_add/out.test.toml | 2 +- .../rename_ambiguous_pairing/out.test.toml | 2 +- .../out.test.toml | 2 +- .../rename_two_removes_one_add/out.test.toml | 2 +- .../split/target_variable/out.test.toml | 2 +- .../split/variable_file_order/out.test.toml | 2 +- .../target_override/out.test.toml | 2 +- .../task_rename_revert/out.test.toml | 2 +- .../validation_errors/out.test.toml | 2 +- .../bundle/debug/list-targets/out.test.toml | 2 +- acceptance/bundle/debug/out.test.toml | 2 +- .../bundle/deploy/empty-bundle/out.test.toml | 2 +- .../deploy/experimental-python/out.test.toml | 2 +- .../deploy/fail-on-active-runs/out.test.toml | 2 +- .../files/no-snapshot-sync/out.test.toml | 2 +- .../files/out-of-band-delete/out.test.toml | 2 +- .../deploy/files/out-of-band-delete/test.toml | 2 +- .../deploy/force-lock-config/out.test.toml | 2 +- .../immutable-no-artifacts/out.test.toml | 2 +- .../out.test.toml | 2 +- .../bundle/deploy/immutable/out.test.toml | 2 +- .../bundle/deploy/mlops-stacks/out.test.toml | 2 +- .../deploy/pipeline-config-dots/out.test.toml | 2 +- .../deploy/python-notebook/out.test.toml | 2 +- .../deploy/readplan/basic/out.test.toml | 2 +- .../cli-version-mismatch/out.test.toml | 2 +- .../grants-remove-principal/out.test.toml | 2 +- .../readplan/invalid-plan/out.test.toml | 2 +- .../readplan/lineage-mismatch/out.test.toml | 2 +- .../readplan/plan-not-found/out.test.toml | 2 +- .../plan-version-mismatch/out.test.toml | 2 +- .../readplan/postgres_role/out.test.toml | 2 +- .../readplan/serial-mismatch/out.test.toml | 2 +- .../readplan/terraform-error/out.test.toml | 2 +- acceptance/bundle/deploy/readplan/test.toml | 2 +- .../readplan/unknown-field/out.test.toml | 2 +- .../deploy/snapshot-comparison/out.test.toml | 2 +- .../deploy/spark-jar-task/out.test.toml | 2 +- .../deploy/wal/chain-3-jobs/out.test.toml | 2 +- .../wal/corrupted-wal-entry/out.test.toml | 2 +- .../deploy/wal/corrupted-wal-entry/test.toml | 2 +- .../wal/crash-after-create/out.test.toml | 2 +- .../bundle/deploy/wal/empty-wal/out.test.toml | 2 +- .../wal/failed-plan-no-wal/out.test.toml | 2 +- .../wal/future-serial-wal/out.test.toml | 2 +- .../deploy/wal/header-only-wal/out.test.toml | 2 +- .../deploy/wal/lineage-mismatch/out.test.toml | 2 +- .../bundle/deploy/wal/stale-wal/out.test.toml | 2 +- .../bundle/deploy/wal/stale-wal/test.toml | 2 +- .../deploy/wal/wal-with-delete/out.test.toml | 2 +- .../yaml-sync-empty-grants/out.test.toml | 2 +- .../deployment/bind/alert/out.test.toml | 2 +- .../deployment/bind/catalog/out.test.toml | 2 +- .../deployment/bind/cluster/out.test.toml | 2 +- .../deployment/bind/dashboard/out.test.toml | 2 +- .../bind/dashboard/recreation/out.test.toml | 2 +- .../bind/database_instance/out.test.toml | 2 +- .../deployment/bind/experiment/out.test.toml | 2 +- .../bind/external_location/out.test.toml | 2 +- .../deployment/bind/genie_space/out.test.toml | 2 +- .../already-managed-different/out.test.toml | 2 +- .../job/already-managed-same/out.test.toml | 2 +- .../bind/job/engine-from-config/out.test.toml | 2 +- .../bind/job/generate-and-bind/out.test.toml | 2 +- .../bind/job/job-abort-bind/out.test.toml | 2 +- .../job/job-spark-python-task/out.test.toml | 2 +- .../bind/job/noop-job/out.test.toml | 2 +- .../bind/job/python-job/out.test.toml | 2 +- .../bind/job/stale-state/out.test.toml | 2 +- .../bind/model-serving-endpoint/out.test.toml | 2 +- .../bind/pipelines/recreate/out.test.toml | 2 +- .../bind/pipelines/update/out.test.toml | 2 +- .../bind/postgres_database/out.test.toml | 2 +- .../bind/postgres_role/out.test.toml | 2 +- .../bind/quality-monitor/out.test.toml | 2 +- .../bind/registered-model/out.test.toml | 2 +- .../deployment/bind/schema/out.test.toml | 2 +- .../bind/secret-scope/out.test.toml | 2 +- .../bind/sql_warehouse/out.test.toml | 2 +- acceptance/bundle/deployment/bind/test.toml | 2 +- .../bind/vector_search_endpoint/out.test.toml | 2 +- .../bind/vector_search_index/out.test.toml | 2 +- .../deployment/bind/volume/out.test.toml | 2 +- .../unbind/engine-from-config/out.test.toml | 2 +- .../deployment/unbind/grants/out.test.toml | 2 +- .../deployment/unbind/job/out.test.toml | 2 +- .../unbind/permissions/out.test.toml | 2 +- .../unbind/python-job/out.test.toml | 2 +- acceptance/bundle/deployment/unbind/test.toml | 2 +- .../destroy/all-resources/out.test.toml | 2 +- .../force-lock-node-limit/out.test.toml | 2 +- .../destroy/jobs-and-pipeline/out.test.toml | 2 +- .../out.test.toml | 2 +- .../bundle/dms/depends-on/out.test.toml | 2 +- .../bundle/dms/emptied-resource/out.test.toml | 2 +- .../bundle/dms/emptied-resource/output.txt | 2 +- acceptance/bundle/dms/emptied-resource/script | 2 +- .../bundle/dms/existing-state/out.test.toml | 2 +- .../dms/multiple-resources/out.test.toml | 2 +- acceptance/bundle/dms/no-drift/out.test.toml | 2 +- .../bundle/dms/no-resources/out.test.toml | 2 +- .../bundle/dms/not-supported/out.test.toml | 2 +- .../dms/operation-upload-fails/out.test.toml | 2 +- .../bundle/dms/partial-update/out.test.toml | 2 +- .../bundle/dms/provenance/out.test.toml | 2 +- .../bundle/dms/record-failure/out.test.toml | 2 +- acceptance/bundle/dms/record/out.test.toml | 2 +- .../dms/redeploy-after-destroy/out.test.toml | 2 +- acceptance/bundle/dms/summary/out.test.toml | 2 +- acceptance/bundle/dms/test.toml | 5 ++-- .../dms/version-never-created/out.test.toml | 2 +- .../bundle/empty_string_dropped/out.test.toml | 2 +- .../empty_string_variable/out.test.toml | 2 +- .../environments/dependencies/out.test.toml | 2 +- .../skip_name_prefix_for_schema/out.test.toml | 2 +- .../bundle/generate/alert/out.test.toml | 2 +- .../alert_existing_id_not_found/out.test.toml | 2 +- .../app_not_yet_deployed/out.test.toml | 2 +- .../generate/app_subfolders/out.test.toml | 2 +- .../bundle/generate/auto-bind/out.test.toml | 2 +- .../generate/dashboard-inplace/out.test.toml | 2 +- .../bundle/generate/dashboard/out.test.toml | 2 +- .../out.test.toml | 2 +- .../out.test.toml | 2 +- .../out.test.toml | 2 +- .../generate/designer_job/out.test.toml | 2 +- .../bundle/generate/genie_space/out.test.toml | 2 +- .../out.test.toml | 2 +- .../genie_space_inplace/out.test.toml | 2 +- .../bundle/generate/git_job/out.test.toml | 2 +- .../generate/include_warning/out.test.toml | 2 +- .../bundle/generate/ipynb_job/out.test.toml | 2 +- .../job_nested_notebooks/out.test.toml | 2 +- .../generate/lakeflow_pipelines/out.test.toml | 2 +- .../bundle/generate/pipeline/out.test.toml | 2 +- .../pipeline_and_deploy/out.test.toml | 2 +- .../generate/pipeline_with_glob/out.test.toml | 2 +- .../generate/pipeline_with_sql/out.test.toml | 2 +- .../bundle/generate/python_job/out.test.toml | 2 +- .../python_job_and_deploy/out.test.toml | 2 +- .../spark_python_task_job/out.test.toml | 2 +- acceptance/bundle/git-permerror/out.test.toml | 2 +- .../bundle/help/bundle-deploy/out.test.toml | 2 +- .../bundle-deployment-migrate/out.test.toml | 2 +- .../help/bundle-deployment/out.test.toml | 2 +- .../bundle/help/bundle-destroy/out.test.toml | 2 +- .../bundle-generate-dashboard/out.test.toml | 2 +- .../help/bundle-generate-job/out.test.toml | 2 +- .../bundle-generate-pipeline/out.test.toml | 2 +- .../bundle/help/bundle-generate/out.test.toml | 2 +- .../bundle/help/bundle-init/out.test.toml | 2 +- .../bundle/help/bundle-open/out.test.toml | 2 +- .../bundle/help/bundle-run/out.test.toml | 2 +- .../bundle/help/bundle-schema/out.test.toml | 2 +- .../bundle/help/bundle-summary/out.test.toml | 2 +- .../bundle/help/bundle-sync/out.test.toml | 2 +- .../bundle/help/bundle-validate/out.test.toml | 2 +- acceptance/bundle/help/bundle/out.test.toml | 2 +- .../includes/glob_in_root_path/out.test.toml | 2 +- .../include_outside_root/out.test.toml | 2 +- .../non_yaml_in_include/out.test.toml | 2 +- .../includes/yml_outside_root/out.test.toml | 2 +- .../bundle/integration_whl/base/out.test.toml | 2 +- .../custom_params/out.test.toml | 2 +- .../interactive_cluster/out.test.toml | 2 +- .../out.test.toml | 2 +- .../interactive_single_user/out.test.toml | 2 +- .../integration_whl/serverless/out.test.toml | 2 +- .../serverless_custom_params/out.test.toml | 2 +- .../serverless_dynamic_version/out.test.toml | 2 +- .../integration_whl/wrapper/out.test.toml | 2 +- .../wrapper_custom_params/out.test.toml | 2 +- .../invariant/continue_293/out.test.toml | 2 +- .../bundle/invariant/continue_293/test.toml | 2 +- .../invariant/delete_idempotent/out.test.toml | 2 +- .../invariant/delete_idempotent/test.toml | 2 +- .../destroy_idempotent/out.test.toml | 2 +- .../invariant/destroy_idempotent/test.toml | 2 +- .../bundle/invariant/migrate/out.test.toml | 2 +- acceptance/bundle/invariant/migrate/test.toml | 2 +- .../bundle/invariant/no_drift/out.test.toml | 2 +- .../bundle/invariant/no_drift/test.toml | 2 +- .../bundle/libraries/maven/out.test.toml | 2 +- .../outside_of_bundle_root/out.test.toml | 2 +- .../bundle/libraries/pypi/out.test.toml | 2 +- .../lifecycle/prevent-destroy/out.test.toml | 2 +- .../started-validation/out.test.toml | 2 +- .../bundle/lifecycle/started/out.test.toml | 2 +- .../local_state_staleness/out.test.toml | 2 +- acceptance/bundle/migrate/added/out.test.toml | 2 +- .../migrate/auto-migrate-clean/out.test.toml | 2 +- .../auto-migrate-empty-tfstate/out.test.toml | 2 +- .../migrate/auto-migrate-envvar/out.test.toml | 2 +- .../auto-migrate-push-failure/out.test.toml | 2 +- .../out.test.toml | 2 +- acceptance/bundle/migrate/basic/out.test.toml | 2 +- .../bundle/migrate/dashboards/out.test.toml | 2 +- .../migrate/default-python/out.test.toml | 2 +- .../engine-config-direct/out.test.toml | 2 +- .../engine-config-terraform/out.test.toml | 2 +- .../bundle/migrate/grants/out.test.toml | 2 +- .../bundle/migrate/permissions/out.test.toml | 2 +- .../bundle/migrate/profile_arg/out.test.toml | 2 +- .../bundle/migrate/removed/out.test.toml | 2 +- acceptance/bundle/migrate/runas/out.test.toml | 2 +- acceptance/bundle/migrate/test.toml | 2 +- .../bundle/migrate/var_arg/out.test.toml | 2 +- .../multi_profile/auto_select/out.test.toml | 2 +- .../multi_profile/env_auth_skip/out.test.toml | 2 +- .../no_workspace_profiles/out.test.toml | 2 +- .../non_interactive_error/out.test.toml | 2 +- acceptance/bundle/open/out.test.toml | 2 +- .../bundle/override/clusters/out.test.toml | 2 +- .../bundle/override/job_cluster/out.test.toml | 2 +- .../override/job_cluster_var/out.test.toml | 2 +- .../bundle/override/job_tasks/out.test.toml | 2 +- .../override/merge-string-map/out.test.toml | 2 +- .../override/pipeline_cluster/out.test.toml | 2 +- .../paths/designer_notebook/out.test.toml | 2 +- .../bundle/paths/fallback/out.test.toml | 2 +- .../paths/git_source_jobs/out.test.toml | 2 +- .../invalid_pipeline_globs/out.test.toml | 2 +- acceptance/bundle/paths/nominal/out.test.toml | 2 +- .../paths/outside_root_no_sync/out.test.toml | 2 +- .../out.test.toml | 2 +- .../bundle/paths/pipeline_globs/out.test.toml | 2 +- .../out.test.toml | 2 +- .../out.test.toml | 2 +- .../out.test.toml | 2 +- .../relative_path_outside_root/out.test.toml | 2 +- .../relative_path_translation/out.test.toml | 2 +- .../bundle/plan/no_upload/out.test.toml | 2 +- .../presets/preset_vs_dev_mode/out.test.toml | 2 +- .../out.test.toml | 2 +- .../out.test.toml | 2 +- .../experimental-compatibility/out.test.toml | 2 +- .../python/grants-aliases/out.test.toml | 2 +- .../python/mutator-ordering/out.test.toml | 2 +- .../out.test.toml | 2 +- .../python/pipelines-support/out.test.toml | 2 +- .../python/propagates-auth-env/out.test.toml | 2 +- .../python/resolve-variable/out.test.toml | 2 +- .../python/resource-loading/out.test.toml | 2 +- .../python/restricted-execution/out.test.toml | 2 +- .../python/schemas-support/out.test.toml | 2 +- .../python/unicode-support/out.test.toml | 2 +- .../python/volumes-support/out.test.toml | 2 +- .../bundle/quality_monitor/out.test.toml | 2 +- acceptance/bundle/refschema/out.test.toml | 2 +- .../bad_ref_string_to_int/out.test.toml | 2 +- .../resource_deps/bad_syntax/out.test.toml | 2 +- .../computed_volume_path/out.test.toml | 2 +- .../resource_deps/create_error/out.test.toml | 2 +- .../resource_deps/duplicate_ref/out.test.toml | 2 +- .../resource_deps/grant_ref/out.test.toml | 2 +- .../resource_deps/id_chain/out.test.toml | 2 +- .../resource_deps/id_star/out.test.toml | 2 +- .../immutable_field_ref/out.test.toml | 2 +- .../out.test.toml | 2 +- .../out.test.toml | 2 +- .../out.test.toml | 2 +- .../implicit_deps_volume/out.test.toml | 2 +- .../bundle/resource_deps/job_id/out.test.toml | 2 +- .../job_id_big_graph/delete_all/out.test.toml | 2 +- .../job_id_big_graph/destroy/out.test.toml | 2 +- .../job_id_delete_bar/out.test.toml | 2 +- .../job_id_delete_foo/out.test.toml | 2 +- .../resource_deps/job_tasks/out.test.toml | 2 +- .../resource_deps/jobs_update/out.test.toml | 2 +- .../jobs_update_remote/out.test.toml | 2 +- .../resource_deps/loop_jobs/out.test.toml | 2 +- .../resource_deps/loop_self/out.test.toml | 2 +- .../out.test.toml | 2 +- .../missing_map_key/out.test.toml | 2 +- .../missing_string_field/out.test.toml | 2 +- .../resource_deps/model_id_ref/out.test.toml | 2 +- .../non_existent_field/out.test.toml | 2 +- .../permission_ref/out.test.toml | 2 +- .../pipelines_recreate/out.test.toml | 2 +- .../out.test.toml | 2 +- .../remote_app_url/out.test.toml | 2 +- .../out.test.toml | 2 +- .../remote_pipeline/out.test.toml | 2 +- .../resource_deps/resources_var/out.test.toml | 2 +- .../resources_var_presets/out.test.toml | 2 +- .../out.test.toml | 2 +- .../tf_path_only_error/out.test.toml | 2 +- .../tf_path_renames/out.test.toml | 2 +- .../unicode_reference/out.test.toml | 2 +- .../volume_path_contains_id/out.test.toml | 2 +- .../volume_path_job_ref/out.test.toml | 2 +- .../resources/alerts/basic/out.test.toml | 2 +- .../resources/alerts/with_file/out.test.toml | 2 +- .../out.test.toml | 2 +- .../with_file_run_from_subdir/out.test.toml | 2 +- .../out.test.toml | 2 +- .../apps/config-drift-stopped/out.test.toml | 2 +- .../resources/apps/config-drift/out.test.toml | 2 +- .../apps/config-no-deployment/out.test.toml | 2 +- .../apps/create_already_exists/out.test.toml | 2 +- .../apps/default_description/out.test.toml | 2 +- .../git-source-no-deployment/out.test.toml | 2 +- .../resources/apps/immutable/out.test.toml | 2 +- .../apps/inline_config/out.test.toml | 2 +- .../lifecycle-started-omitted/out.test.toml | 2 +- .../out.test.toml | 2 +- .../lifecycle-started-toggle/out.test.toml | 2 +- .../apps/lifecycle-started/out.test.toml | 2 +- .../apps/readplan-lifecycle/out.test.toml | 2 +- .../apps/resource-refs/out.test.toml | 2 +- .../resources/apps/update/out.test.toml | 2 +- .../catalogs/auto-approve/out.test.toml | 2 +- .../resources/catalogs/basic/out.test.toml | 2 +- .../drift/managed_properties/out.test.toml | 2 +- .../catalogs/empty-name/out.test.toml | 2 +- .../catalogs/with-schemas/out.test.toml | 2 +- .../deploy/data_security_mode/out.test.toml | 2 +- .../deploy/instance_pool/out.test.toml | 2 +- .../instance_pool_and_node_type/out.test.toml | 2 +- .../deploy/local_ssd_count/out.test.toml | 2 +- .../deploy/num_workers_absent/out.test.toml | 2 +- .../clusters/deploy/simple/out.test.toml | 2 +- .../deploy/update-after-create/out.test.toml | 2 +- .../update-and-resize-autoscale/out.test.toml | 2 +- .../deploy/update-and-resize/out.test.toml | 2 +- .../deploy/workload_type/out.test.toml | 2 +- .../out.test.toml | 2 +- .../lifecycle-started-toggle/out.test.toml | 2 +- .../clusters/lifecycle-started/out.test.toml | 2 +- .../clusters/readplan-lifecycle/out.test.toml | 2 +- .../resize-terminated-fallback/out.test.toml | 2 +- .../run/spark_python_task/out.test.toml | 2 +- .../change-embed-credentials/out.test.toml | 2 +- .../dashboards/change-name/out.test.toml | 2 +- .../change-parent-path/out.test.toml | 2 +- .../change-serialized-dashboard/out.test.toml | 2 +- .../dataset-catalog-schema/out.test.toml | 2 +- .../delete-trashed-out-of-band/out.test.toml | 2 +- .../dashboards/destroy/out.test.toml | 2 +- .../dashboards/detect-change/out.test.toml | 2 +- .../dashboards/generate_inplace/out.test.toml | 2 +- .../dashboards/nested-folders/out.test.toml | 2 +- .../out.test.toml | 2 +- .../out.test.toml | 2 +- .../out.test.toml | 2 +- .../resources/dashboards/simple/out.test.toml | 2 +- .../simple_outside_bundle_root/out.test.toml | 2 +- .../dashboards/simple_syncroot/out.test.toml | 2 +- .../unpublish-out-of-band/out.test.toml | 2 +- .../database_catalogs/basic/out.test.toml | 2 +- .../database_catalogs/recreate/out.test.toml | 2 +- .../database_instances/recreate/out.test.toml | 2 +- .../single-instance/out.test.toml | 2 +- .../resources/experiments/basic/out.test.toml | 2 +- .../external_locations/out.test.toml | 2 +- .../genie_spaces/delete_warning/out.test.toml | 2 +- .../genie_spaces/inline/out.test.toml | 2 +- .../parent_path_update/out.test.toml | 2 +- .../recreate_when_gone/out.test.toml | 2 +- .../serialized_space/out.test.toml | 2 +- .../genie_spaces/simple/out.test.toml | 2 +- .../version_migration/out.test.toml | 2 +- .../resources/grants/catalogs/out.test.toml | 2 +- .../grants/registered_models/out.test.toml | 2 +- .../schemas/all_privileges/out.test.toml | 2 +- .../all_privileges_coexist/out.test.toml | 2 +- .../schemas/change_privilege/out.test.toml | 2 +- .../duplicate_principals/out.test.toml | 2 +- .../duplicate_privileges/out.test.toml | 2 +- .../grants/schemas/empty_array/out.test.toml | 2 +- .../out_of_band_principal/out.test.toml | 2 +- .../grants/schemas/remove_all/out.test.toml | 2 +- .../schemas/remove_principal/out.test.toml | 2 +- .../resources/grants/volumes/out.test.toml | 2 +- .../resources/independent/out.test.toml | 2 +- .../resources/instance_pools/out.test.toml | 2 +- .../resources/job_runs/basic/out.test.toml | 2 +- .../job_runs/job_parameters/out.test.toml | 2 +- .../resources/job_runs/redeploy/out.test.toml | 2 +- .../resources/jobs/alert-task/out.test.toml | 2 +- .../resources/jobs/big_id/out.test.toml | 2 +- .../bundle/resources/jobs/big_id/test.toml | 2 +- .../jobs/check-metadata/out.test.toml | 2 +- .../resources/jobs/create-error/out.test.toml | 2 +- .../resources/jobs/delete_job/out.test.toml | 2 +- .../resources/jobs/delete_task/out.test.toml | 2 +- .../resources/jobs/delete_task/test.toml | 2 +- .../jobs/double-underscore-keys/out.test.toml | 2 +- .../jobs/fail-on-active-runs/out.test.toml | 2 +- .../instance_pool_and_node_type/out.test.toml | 2 +- .../jobs/no-git-provider/out.test.toml | 2 +- .../resources/jobs/num_workers/out.test.toml | 2 +- .../jobs/on_failure_empty_slice/out.test.toml | 2 +- .../jobs/remote_add_tag/out.test.toml | 2 +- .../jobs/remote_delete/deploy/out.test.toml | 2 +- .../jobs/remote_delete/deploy/test.toml | 2 +- .../jobs/remote_delete/destroy/out.test.toml | 2 +- .../removed_from_config/out.test.toml | 2 +- .../jobs/remote_matches_config/out.test.toml | 2 +- .../jobs/shared-root-path/out.test.toml | 2 +- .../jobs/tags_empty_map/out.test.toml | 2 +- .../resources/jobs/task-source/out.test.toml | 2 +- .../jobs/tasks-reorder-locally/out.test.toml | 2 +- .../unknown-terraform-field/out.test.toml | 2 +- .../resources/jobs/update/out.test.toml | 2 +- .../bundle/resources/jobs/update/test.toml | 2 +- .../jobs/update_single_node/out.test.toml | 2 +- .../jobs/webhook-reorder-remote/out.test.toml | 2 +- .../basic/out.test.toml | 2 +- .../drift/recreated_same_name/out.test.toml | 2 +- .../telemetry_config_unmanaged/out.test.toml | 2 +- .../out.test.toml | 2 +- .../drift/write_only/out.test.toml | 2 +- .../recreate/catalog-name/out.test.toml | 2 +- .../recreate/name-change/out.test.toml | 2 +- .../recreate/route-optimized/out.test.toml | 2 +- .../recreate/schema-name/out.test.toml | 2 +- .../recreate/table-prefix/out.test.toml | 2 +- .../running-endpoint/out.test.toml | 2 +- .../update/ai-gateway/out.test.toml | 2 +- .../both_gateway_and_tags/out.test.toml | 2 +- .../update/config/out.test.toml | 2 +- .../update/email-notifications/out.test.toml | 2 +- .../update/tags/out.test.toml | 2 +- .../resources/models/basic/out.test.toml | 2 +- .../resources/models/empty-name/out.test.toml | 2 +- .../models/readplan-permissions/out.test.toml | 2 +- .../apps/current_can_manage/out.test.toml | 2 +- .../apps/other_can_manage/out.test.toml | 2 +- .../clusters/current_can_manage/out.test.toml | 2 +- .../permissions/clusters/target/out.test.toml | 2 +- .../dashboards/create/out.test.toml | 2 +- .../current_can_manage/out.test.toml | 2 +- .../current_can_manage/out.test.toml | 2 +- .../permissions/factcheck/out.test.toml | 2 +- .../current_can_manage/out.test.toml | 2 +- .../out_of_band_deletion/out.test.toml | 2 +- .../jobs/added_remotely/out.test.toml | 2 +- .../jobs/current_can_manage/out.test.toml | 2 +- .../jobs/current_can_manage_run/out.test.toml | 2 +- .../jobs/current_is_owner/out.test.toml | 2 +- .../permissions/jobs/delete_one/out.test.toml | 2 +- .../jobs/deleted_remotely/out.test.toml | 2 +- .../jobs/destroy_without_mgmtperms/test.toml | 2 +- .../with_permissions/out.test.toml | 2 +- .../without_permissions/out.test.toml | 2 +- .../permissions/jobs/empty_list/out.test.toml | 2 +- .../jobs/other_can_manage/out.test.toml | 2 +- .../jobs/other_can_manage_run/out.test.toml | 2 +- .../jobs/other_is_owner/out.test.toml | 2 +- .../jobs/reorder_locally/out.test.toml | 2 +- .../jobs/reorder_remotely/out.test.toml | 2 +- .../permissions/jobs/update/out.test.toml | 2 +- .../permissions/jobs/viewers/out.test.toml | 2 +- .../models/current_can_manage/out.test.toml | 2 +- .../resources/permissions/out.test.toml | 2 +- .../pipelines/504/create/out.test.toml | 2 +- .../pipelines/504/plan/out.test.toml | 2 +- .../pipelines/504/update/out.test.toml | 2 +- .../current_can_manage/out.test.toml | 2 +- .../pipelines/current_is_owner/out.test.toml | 2 +- .../pipelines/empty_list/out.test.toml | 2 +- .../pipelines/other_can_manage/out.test.toml | 2 +- .../pipelines/other_is_owner/out.test.toml | 2 +- .../pipelines/update/out.test.toml | 2 +- .../current_can_manage/out.test.toml | 2 +- .../current_can_manage/out.test.toml | 2 +- .../target_permissions/out.test.toml | 2 +- .../current_can_manage/out.test.toml | 2 +- .../allow-duplicate-names/out.test.toml | 2 +- .../pipelines/auto-approve/out.test.toml | 2 +- .../pipelines/drift/parameters/out.test.toml | 2 +- .../pipelines/lakeflow-pipeline/out.test.toml | 2 +- .../pipelines/num-workers-zero/out.test.toml | 2 +- .../pipelines/photon-true/out.test.toml | 2 +- .../change-ingestion-definition/out.test.toml | 2 +- .../change-storage/out.test.toml | 2 +- .../pipelines/recreate/out.test.toml | 2 +- .../remote_matches_config/out.test.toml | 2 +- .../resources/pipelines/update/out.test.toml | 2 +- .../pipelines/zero-value-fields/out.test.toml | 2 +- .../postgres_branches/basic/out.test.toml | 2 +- .../purge_on_delete/out.test.toml | 2 +- .../purge_on_delete_transitions/out.test.toml | 2 +- .../postgres_branches/recreate/out.test.toml | 2 +- .../replace_existing/out.test.toml | 2 +- .../update_protected/out.test.toml | 2 +- .../without_branch_id/out.test.toml | 2 +- .../postgres_catalogs/basic/out.test.toml | 2 +- .../postgres_catalogs/recreate/out.test.toml | 2 +- .../postgres_databases/basic/out.test.toml | 2 +- .../live_errors/bad_database_id/out.test.toml | 2 +- .../live_errors/bad_role_ref/out.test.toml | 2 +- .../postgres_databases/recreate/out.test.toml | 2 +- .../replace_existing/out.test.toml | 2 +- .../replace_existing/test.toml | 2 +- .../postgres_databases/update/out.test.toml | 2 +- .../postgres_endpoints/basic/out.test.toml | 2 +- .../postgres_endpoints/recreate/out.test.toml | 2 +- .../replace_existing/out.test.toml | 2 +- .../update_autoscaling/out.test.toml | 2 +- .../without_endpoint_id/out.test.toml | 2 +- .../postgres_projects/basic/out.test.toml | 2 +- .../purge_on_delete/out.test.toml | 2 +- .../purge_on_delete_transitions/out.test.toml | 2 +- .../postgres_projects/recreate/out.test.toml | 2 +- .../update_display_name/out.test.toml | 2 +- .../without_project_id/out.test.toml | 2 +- .../postgres_roles/basic/out.test.toml | 2 +- .../inherited-role-bind/out.test.toml | 2 +- .../inherited-role-conflict/out.test.toml | 2 +- .../inherited-role-conflict/test.toml | 2 +- .../recreate-postgres-role/out.test.toml | 2 +- .../postgres_roles/recreate/out.test.toml | 2 +- .../replace_existing/out.test.toml | 2 +- .../postgres_roles/replace_existing/test.toml | 2 +- .../postgres_roles/update/out.test.toml | 2 +- .../basic/out.test.toml | 2 +- .../recreate/out.test.toml | 2 +- .../change_assets_dir/out.test.toml | 2 +- .../change_output_schema_name/out.test.toml | 2 +- .../change_table_name/out.test.toml | 2 +- .../quality_monitors/create/out.test.toml | 2 +- .../aliases_converge/out.test.toml | 2 +- .../registered_models/basic/out.test.toml | 2 +- .../drift/browse_only/out.test.toml | 2 +- .../schemas/auto-approve/out.test.toml | 2 +- .../drift/managed_properties/out.test.toml | 2 +- .../resources/schemas/recreate/out.test.toml | 2 +- .../resources/schemas/update/out.test.toml | 2 +- .../secret_scopes/backend-type/out.test.toml | 2 +- .../secret_scopes/basic/out.test.toml | 2 +- .../secret_scopes/delete_scope/out.test.toml | 2 +- .../permissions-collapse/out.test.toml | 2 +- .../secret_scopes/permissions/out.test.toml | 2 +- .../resources/secrets/basic/out.test.toml | 2 +- .../secrets/direct-only/out.test.toml | 2 +- .../secrets/update-value/out.test.toml | 2 +- .../out.test.toml | 2 +- .../validate-no-plain-text/out.test.toml | 2 +- .../lifecycle-started-edit/out.test.toml | 2 +- .../out.test.toml | 2 +- .../lifecycle-started-toggle/out.test.toml | 2 +- .../lifecycle-started/out.test.toml | 2 +- .../resources/sql_warehouses/out.test.toml | 2 +- .../basic/out.test.toml | 2 +- .../recreate/out.test.toml | 2 +- .../basic/out.test.toml | 2 +- .../drift/budget_policy/out.test.toml | 2 +- .../drift/recreated_same_name/out.test.toml | 2 +- .../drift/target_qps/out.test.toml | 2 +- .../recreate/create-fails/out.test.toml | 2 +- .../recreate/endpoint_type/out.test.toml | 2 +- .../update/budget_policy/out.test.toml | 2 +- .../update/target_qps/out.test.toml | 2 +- .../vector_search_indexes/basic/out.test.toml | 2 +- .../drift/deleted_remotely/out.test.toml | 2 +- .../drift/orphaned_endpoint/out.test.toml | 2 +- .../grants/select/out.test.toml | 2 +- .../embedding_dimension/out.test.toml | 2 +- .../recreate/pending_deletion/out.test.toml | 2 +- .../recreate/with_endpoint/out.test.toml | 2 +- .../schema_normalization/out.test.toml | 2 +- .../volumes/catalog-var-ref/out.test.toml | 2 +- .../volumes/change-comment/out.test.toml | 2 +- .../volumes/change-name/out.test.toml | 2 +- .../volumes/change-schema-name/out.test.toml | 2 +- .../resources/volumes/recreate/out.test.toml | 2 +- .../volumes/remote-change-name/out.test.toml | 2 +- .../volumes/remote-delete/out.test.toml | 2 +- .../set-storage-location/out.test.toml | 2 +- .../volumes/set-volume-path/out.test.toml | 2 +- .../volumes/uppercase-name/out.test.toml | 2 +- .../root/env-not-a-directory/out.test.toml | 2 +- .../bundle/root/env-not-found/out.test.toml | 2 +- .../bundle/root/not-found/out.test.toml | 2 +- .../bundle/root/real-empty-dir/out.test.toml | 2 +- .../bundle/run/app-with-job/out.test.toml | 2 +- acceptance/bundle/run/basic/out.test.toml | 2 +- .../bundle/run/diagnostics/out.test.toml | 2 +- .../run/inline-script/basic/out.test.toml | 2 +- .../run/inline-script/cwd/out.test.toml | 2 +- .../profile-is-passed/from_flag/out.test.toml | 2 +- .../target-is-passed/default/out.test.toml | 2 +- .../target-is-passed/from_flag/out.test.toml | 2 +- .../run/inline-script/no-auth/out.test.toml | 2 +- .../run/inline-script/no-bundle/out.test.toml | 2 +- .../inline-script/no-separator/out.test.toml | 2 +- .../bundle/run/jobs/partial_run/out.test.toml | 2 +- acceptance/bundle/run/no-state/out.test.toml | 2 +- .../bundle/run/refresh-flags/out.test.toml | 2 +- .../bundle/run/scripts/basic/out.test.toml | 2 +- .../bundle/run/scripts/cwd/out.test.toml | 2 +- .../profile-is-passed/from_flag/out.test.toml | 2 +- .../target-is-passed/default/out.test.toml | 2 +- .../target-is-passed/from_flag/out.test.toml | 2 +- .../run/scripts/env-bad-prefix/out.test.toml | 2 +- .../run/scripts/env-precedence/out.test.toml | 2 +- .../run/scripts/env-section/out.test.toml | 2 +- .../run/scripts/exit_code/out.test.toml | 2 +- .../bundle/run/scripts/io/out.test.toml | 2 +- .../bundle/run/scripts/no-auth/out.test.toml | 2 +- .../scripts/no-interpolation/out.test.toml | 2 +- .../run/scripts/no_content/out.test.toml | 2 +- .../run/scripts/shell/envvar/out.test.toml | 2 +- .../run/scripts/shell/math/out.test.toml | 2 +- .../out.test.toml | 2 +- .../out.test.toml | 2 +- .../out.test.toml | 2 +- .../bundle/run/state-wiped/out.test.toml | 2 +- .../run_as/allowed/regular_user/out.test.toml | 2 +- .../allowed/service_principal/out.test.toml | 2 +- .../run_as/dashboard_embed/out.test.toml | 2 +- .../run_as/empty_override/out.test.toml | 2 +- .../bundle/run_as/empty_run_as/out.test.toml | 2 +- .../run_as/empty_run_as_dict/out.test.toml | 2 +- .../bundle/run_as/empty_sp/out.test.toml | 2 +- .../bundle/run_as/empty_user/out.test.toml | 2 +- .../run_as/empty_user_and_sp/out.test.toml | 2 +- .../invalid_both_sp_and_user/out.test.toml | 2 +- .../bundle/run_as/job_default/out.test.toml | 2 +- .../model_serving_different/out.test.toml | 2 +- .../model_serving_matching/out.test.toml | 2 +- acceptance/bundle/run_as/out.test.toml | 2 +- .../pipelines/regular_user/out.test.toml | 2 +- .../pipelines/service_principal/out.test.toml | 2 +- .../run_as/pipelines_legacy/out.test.toml | 2 +- acceptance/bundle/script.prepare | 3 +- .../scripts/no-trailing-newline/out.test.toml | 2 +- acceptance/bundle/scripts/out.test.toml | 2 +- .../restricted-execution/out.test.toml | 2 +- .../bundle/select/ambiguous/out.test.toml | 2 +- acceptance/bundle/select/basic/out.test.toml | 2 +- .../select/grants_permissions/out.test.toml | 2 +- .../bundle/select/missing/out.test.toml | 2 +- .../bundle/select/rejected/out.test.toml | 2 +- acceptance/bundle/state/bad_env/out.test.toml | 2 +- .../bundle/state/bad_json_local/out.test.toml | 2 +- acceptance/bundle/state/basic/out.test.toml | 2 +- .../bundle/state/engine_default/out.test.toml | 2 +- .../state/engine_mismatch/out.test.toml | 2 +- .../bundle/state/feature_flags/out.test.toml | 2 +- .../state/force_pull_commands/out.test.toml | 2 +- .../bundle/state/future_version/out.test.toml | 2 +- .../state/lineage_different/out.test.toml | 2 +- .../permission_level_migration/out.test.toml | 2 +- .../permission_level_migration/test.toml | 2 +- .../bundle/state/same_serial/out.test.toml | 2 +- .../bundle/state/state_present/out.test.toml | 2 +- .../missing-libraries-file-path/out.test.toml | 2 +- .../summary/modified_status/out.test.toml | 2 +- acceptance/bundle/sync/dryrun/out.test.toml | 2 +- acceptance/bundle/sync/out.test.toml | 2 +- .../bundle/syncroot/dotdot-git/out.test.toml | 2 +- .../syncroot/dotdot-nogit/out.test.toml | 2 +- .../config-remote-sync-error/out.test.toml | 2 +- .../config-remote-sync-recreate/out.test.toml | 2 +- .../config-remote-sync-save/out.test.toml | 2 +- .../config-remote-sync/out.test.toml | 2 +- .../out.test.toml | 2 +- .../deploy-artifact-path-type/out.test.toml | 2 +- .../deploy-artifacts-variables/out.test.toml | 2 +- .../deploy-compute-type/out.test.toml | 2 +- .../deploy-config-file-count/out.test.toml | 2 +- .../deploy-error-message/out.test.toml | 2 +- .../telemetry/deploy-error/out.test.toml | 2 +- .../deploy-experimental/out.test.toml | 2 +- .../telemetry/deploy-mode/out.test.toml | 2 +- .../deploy-name-prefix/custom/out.test.toml | 2 +- .../mode-development/out.test.toml | 2 +- .../telemetry/deploy-no-uuid/out.test.toml | 2 +- .../telemetry/deploy-run-as/out.test.toml | 2 +- .../deploy-target-count/out.test.toml | 2 +- .../deploy-variable-count/out.test.toml | 2 +- .../deploy-whl-artifacts/out.test.toml | 2 +- .../out.test.toml | 2 +- .../bundle/telemetry/deploy/out.test.toml | 2 +- acceptance/bundle/telemetry/test.toml | 2 +- .../helper_upper_lower/out.test.toml | 2 +- .../helper_username/out.test.toml | 2 +- .../helpers-error/out.test.toml | 2 +- .../number-precision/out.test.toml | 2 +- .../supported-url/out.test.toml | 2 +- .../unsupported-url/out.test.toml | 2 +- .../wrong-path/out.test.toml | 2 +- .../wrong-url/out.test.toml | 2 +- .../bundle/templates/dbt-sql/out.test.toml | 2 +- .../default-minimal/python/out.test.toml | 2 +- .../default-minimal/skip/out.test.toml | 2 +- .../default-minimal/sql/out.test.toml | 2 +- .../azure-government/out.test.toml | 2 +- .../default-python/classic/out.test.toml | 2 +- .../combinations/classic/out.test.toml | 2 +- .../combinations/serverless/out.test.toml | 2 +- .../fail-missing-uv/out.test.toml | 2 +- .../integration_classic/out.test.toml | 2 +- .../default-python/no-uc/out.test.toml | 2 +- .../serverless-customcatalog/out.test.toml | 2 +- .../default-python/serverless/out.test.toml | 2 +- .../templates/default-scala/out.test.toml | 2 +- .../templates/default-sql/out.test.toml | 2 +- .../lakeflow-integrations/out.test.toml | 2 +- .../lakeflow-pipelines/python/out.test.toml | 2 +- .../lakeflow-pipelines/sql/out.test.toml | 2 +- .../templates/nested-output/out.test.toml | 2 +- .../pydabs/check-consistency/out.test.toml | 2 +- .../pydabs/check-formatting/out.test.toml | 2 +- .../pydabs/deploy-classic/out.test.toml | 2 +- .../pydabs/init-classic/out.test.toml | 2 +- .../telemetry/custom-template/out.test.toml | 2 +- .../templates/telemetry/dbt-sql/out.test.toml | 2 +- .../telemetry/default-python/out.test.toml | 2 +- .../telemetry/default-sql/out.test.toml | 2 +- acceptance/bundle/templates/test.toml | 2 +- acceptance/bundle/test.toml | 10 +++---- .../trampoline/warning_message/out.test.toml | 2 +- .../out.test.toml | 2 +- .../out.test.toml | 2 +- .../bundle/undefined_resources/out.test.toml | 2 +- .../internal_server_error/out.test.toml | 2 +- .../bundle/upload/timeout/out.test.toml | 2 +- acceptance/bundle/user_agent/out.test.toml | 2 +- .../bundle/user_agent/simple/out.test.toml | 2 +- acceptance/bundle/user_agent/test.toml | 2 +- .../validate/anchor_containers/out.test.toml | 2 +- .../out.test.toml | 2 +- .../validate/dashboard_defaults/out.test.toml | 2 +- .../dashboard_required_name/out.test.toml | 2 +- .../out.test.toml | 2 +- .../definitions_yaml_anchors/out.test.toml | 2 +- .../duplicate_yaml_merge_key/out.test.toml | 2 +- .../empty_resources/empty_def/out.test.toml | 2 +- .../empty_resources/empty_dict/out.test.toml | 2 +- .../empty_resources/null/out.test.toml | 2 +- .../empty_resources/with_grants/out.test.toml | 2 +- .../with_permissions/out.test.toml | 2 +- .../bundle/validate/empty_tasks/out.test.toml | 2 +- .../engine-config-valid/out.test.toml | 2 +- acceptance/bundle/validate/enum/out.test.toml | 2 +- .../validate/enum_resource_refs/out.test.toml | 2 +- .../genie_space_complex/out.test.toml | 2 +- .../genie_space_defaults/out.test.toml | 2 +- .../out.test.toml | 2 +- .../grants_required_principal/out.test.toml | 2 +- .../immutable_workspace_paths/out.test.toml | 2 +- .../validate/include_locations/out.test.toml | 2 +- .../invalid-engine-bundle/out.test.toml | 2 +- .../invalid-engine-target/out.test.toml | 2 +- .../validate/job-references/out.test.toml | 2 +- .../out.test.toml | 2 +- .../model_serving_conversion/out.test.toml | 2 +- .../models/missing_name/out.test.toml | 2 +- .../validate/models/user_id/out.test.toml | 2 +- .../validate/no_dashboard_etag/out.test.toml | 2 +- .../no_genie_space_etag/out.test.toml | 2 +- .../bundle/validate/permissions/out.test.toml | 2 +- .../permissions_overlap/out.test.toml | 2 +- .../presets_max_concurrent_runs/out.test.toml | 2 +- .../presets_name_prefix/out.test.toml | 2 +- .../presets_name_prefix_dev/out.test.toml | 2 +- .../validate/presets_tags/out.test.toml | 2 +- .../bundle/validate/required/out.test.toml | 2 +- .../reserved_deployment_fields/out.test.toml | 2 +- .../sql_warehouse_required_name/out.test.toml | 2 +- .../bundle/validate/strict/out.test.toml | 2 +- .../validate/sync_patterns/out.test.toml | 2 +- .../validate/var_in_bundle_name/out.test.toml | 2 +- .../validate/volume_defaults/out.test.toml | 2 +- .../bundle/variables/arg-repeat/out.test.toml | 2 +- .../variables/complex-cross-ref/out.test.toml | 2 +- .../complex-cycle-self/out.test.toml | 2 +- .../variables/complex-cycle/out.test.toml | 2 +- .../variables/complex-simple/out.test.toml | 2 +- .../complex-transitive-deep/out.test.toml | 2 +- .../complex-transitive-deeper/out.test.toml | 2 +- .../complex-transitive/out.test.toml | 2 +- .../complex-with-var-reference/out.test.toml | 2 +- .../complex-within-complex/out.test.toml | 2 +- .../bundle/variables/complex/out.test.toml | 2 +- .../complex_multiple_files/out.test.toml | 2 +- .../bundle/variables/cycle/out.test.toml | 2 +- .../variables/double_underscore/out.test.toml | 2 +- .../bundle/variables/empty/out.test.toml | 2 +- .../variables/env_overrides/out.test.toml | 2 +- .../variables/file-defaults/out.test.toml | 2 +- .../bundle/variables/git-branch/out.test.toml | 2 +- .../bundle/variables/host/out.test.toml | 2 +- acceptance/bundle/variables/int/out.test.toml | 2 +- .../bundle/variables/issue_2436/out.test.toml | 2 +- .../issue_3039_lookup_with_ref/out.test.toml | 2 +- .../bundle/variables/lookup/out.test.toml | 2 +- .../prepend-workspace-var/out.test.toml | 2 +- .../variables/resolve-builtin/out.test.toml | 2 +- .../variables/resolve-empty/out.test.toml | 2 +- .../out.test.toml | 2 +- .../resolve-nonstrings/out.test.toml | 2 +- .../resolve-resources-fields/out.test.toml | 2 +- .../resolve-vars-in-root-path/out.test.toml | 2 +- .../variables/unicode_reference/out.test.toml | 2 +- .../bundle/variables/vanilla/out.test.toml | 2 +- .../bundle/variables/var_in_var/out.test.toml | 2 +- .../variable_in_resource_key/out.test.toml | 2 +- .../out.test.toml | 2 +- .../without_definition/out.test.toml | 2 +- .../volume_path/invalid_file/out.test.toml | 2 +- .../invalid_resource/out.test.toml | 2 +- .../volume_path/invalid_root/out.test.toml | 2 +- .../volume_path/invalid_state/out.test.toml | 2 +- .../bundle/volume_path/valid/out.test.toml | 2 +- build/pybin/python | 1 + build/pybin/python3 | 1 + build/pybin/uv | 9 ++++++ bundle/env/dms.go | 28 +++++++++++-------- bundle/env/dms_test.go | 17 ++++++----- bundle/phases/dms.go | 3 +- 901 files changed, 941 insertions(+), 920 deletions(-) create mode 120000 build/pybin/python create mode 120000 build/pybin/python3 create mode 100755 build/pybin/uv diff --git a/acceptance/bundle/ai_runtime_task/empty_code_source/out.test.toml b/acceptance/bundle/ai_runtime_task/empty_code_source/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/ai_runtime_task/empty_code_source/out.test.toml +++ b/acceptance/bundle/ai_runtime_task/empty_code_source/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/ai_runtime_task/local_code_source/out.test.toml b/acceptance/bundle/ai_runtime_task/local_code_source/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/ai_runtime_task/local_code_source/out.test.toml +++ b/acceptance/bundle/ai_runtime_task/local_code_source/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/apps/app_yaml/out.test.toml b/acceptance/bundle/apps/app_yaml/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/apps/app_yaml/out.test.toml +++ b/acceptance/bundle/apps/app_yaml/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/apps/artifact_and_app_same_path/out.test.toml b/acceptance/bundle/apps/artifact_and_app_same_path/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/apps/artifact_and_app_same_path/out.test.toml +++ b/acceptance/bundle/apps/artifact_and_app_same_path/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/apps/compute_size/out.test.toml b/acceptance/bundle/apps/compute_size/out.test.toml index 0fc51ee009b..264de50aa96 100644 --- a/acceptance/bundle/apps/compute_size/out.test.toml +++ b/acceptance/bundle/apps/compute_size/out.test.toml @@ -1,3 +1,3 @@ Cloud = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/apps/delete_deleting/out.test.toml b/acceptance/bundle/apps/delete_deleting/out.test.toml index 51a5602947c..2c6699da193 100644 --- a/acceptance/bundle/apps/delete_deleting/out.test.toml +++ b/acceptance/bundle/apps/delete_deleting/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/apps/git_source/out.test.toml b/acceptance/bundle/apps/git_source/out.test.toml index ac167149094..d9a6e3e56d8 100644 --- a/acceptance/bundle/apps/git_source/out.test.toml +++ b/acceptance/bundle/apps/git_source/out.test.toml @@ -1,4 +1,4 @@ Cloud = false RequiresWarehouse = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/apps/job_permissions/out.test.toml b/acceptance/bundle/apps/job_permissions/out.test.toml index 026bc5f1a2d..3587a9c012d 100644 --- a/acceptance/bundle/apps/job_permissions/out.test.toml +++ b/acceptance/bundle/apps/job_permissions/out.test.toml @@ -1,4 +1,4 @@ Cloud = true RequiresUnityCatalog = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/apps/job_permissions_warning/out.test.toml b/acceptance/bundle/apps/job_permissions_warning/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/apps/job_permissions_warning/out.test.toml +++ b/acceptance/bundle/apps/job_permissions_warning/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/apps/value_from_warning/out.test.toml b/acceptance/bundle/apps/value_from_warning/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/apps/value_from_warning/out.test.toml +++ b/acceptance/bundle/apps/value_from_warning/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/artifacts/ai_runtime_code_source/out.test.toml b/acceptance/bundle/artifacts/ai_runtime_code_source/out.test.toml index 51a5602947c..2c6699da193 100644 --- a/acceptance/bundle/artifacts/ai_runtime_code_source/out.test.toml +++ b/acceptance/bundle/artifacts/ai_runtime_code_source/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/artifacts/artifact_path_with_volume/volume_doesnot_exist/out.test.toml b/acceptance/bundle/artifacts/artifact_path_with_volume/volume_doesnot_exist/out.test.toml index 026bc5f1a2d..3587a9c012d 100644 --- a/acceptance/bundle/artifacts/artifact_path_with_volume/volume_doesnot_exist/out.test.toml +++ b/acceptance/bundle/artifacts/artifact_path_with_volume/volume_doesnot_exist/out.test.toml @@ -1,4 +1,4 @@ Cloud = true RequiresUnityCatalog = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/artifacts/artifact_path_with_volume/volume_not_deployed/out.test.toml b/acceptance/bundle/artifacts/artifact_path_with_volume/volume_not_deployed/out.test.toml index 026bc5f1a2d..3587a9c012d 100644 --- a/acceptance/bundle/artifacts/artifact_path_with_volume/volume_not_deployed/out.test.toml +++ b/acceptance/bundle/artifacts/artifact_path_with_volume/volume_not_deployed/out.test.toml @@ -1,4 +1,4 @@ Cloud = true RequiresUnityCatalog = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/artifacts/artifact_upload_for_volumes/out.test.toml b/acceptance/bundle/artifacts/artifact_upload_for_volumes/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/artifacts/artifact_upload_for_volumes/out.test.toml +++ b/acceptance/bundle/artifacts/artifact_upload_for_volumes/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/artifacts/artifact_upload_for_workspace/out.test.toml b/acceptance/bundle/artifacts/artifact_upload_for_workspace/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/artifacts/artifact_upload_for_workspace/out.test.toml +++ b/acceptance/bundle/artifacts/artifact_upload_for_workspace/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/artifacts/artifact_upload_with_no_library_reference/out.test.toml b/acceptance/bundle/artifacts/artifact_upload_with_no_library_reference/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/artifacts/artifact_upload_with_no_library_reference/out.test.toml +++ b/acceptance/bundle/artifacts/artifact_upload_with_no_library_reference/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/artifacts/artifacts_dynamic_version/out.test.toml b/acceptance/bundle/artifacts/artifacts_dynamic_version/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/artifacts/artifacts_dynamic_version/out.test.toml +++ b/acceptance/bundle/artifacts/artifacts_dynamic_version/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/artifacts/build_and_files/out.test.toml b/acceptance/bundle/artifacts/build_and_files/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/artifacts/build_and_files/out.test.toml +++ b/acceptance/bundle/artifacts/build_and_files/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/artifacts/build_and_files_whl/out.test.toml b/acceptance/bundle/artifacts/build_and_files_whl/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/artifacts/build_and_files_whl/out.test.toml +++ b/acceptance/bundle/artifacts/build_and_files_whl/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/artifacts/glob_exact_whl/out.test.toml b/acceptance/bundle/artifacts/glob_exact_whl/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/artifacts/glob_exact_whl/out.test.toml +++ b/acceptance/bundle/artifacts/glob_exact_whl/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/artifacts/globs_in_files/out.test.toml b/acceptance/bundle/artifacts/globs_in_files/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/artifacts/globs_in_files/out.test.toml +++ b/acceptance/bundle/artifacts/globs_in_files/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/artifacts/globs_in_files_in_include/out.test.toml b/acceptance/bundle/artifacts/globs_in_files_in_include/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/artifacts/globs_in_files_in_include/out.test.toml +++ b/acceptance/bundle/artifacts/globs_in_files_in_include/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/artifacts/globs_invalid/out.test.toml b/acceptance/bundle/artifacts/globs_invalid/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/artifacts/globs_invalid/out.test.toml +++ b/acceptance/bundle/artifacts/globs_invalid/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/artifacts/issue_3109/out.test.toml b/acceptance/bundle/artifacts/issue_3109/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/artifacts/issue_3109/out.test.toml +++ b/acceptance/bundle/artifacts/issue_3109/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/artifacts/nil_artifacts/out.test.toml b/acceptance/bundle/artifacts/nil_artifacts/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/artifacts/nil_artifacts/out.test.toml +++ b/acceptance/bundle/artifacts/nil_artifacts/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/artifacts/same_name_libraries/out.test.toml b/acceptance/bundle/artifacts/same_name_libraries/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/artifacts/same_name_libraries/out.test.toml +++ b/acceptance/bundle/artifacts/same_name_libraries/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/artifacts/shell/bash/out.test.toml b/acceptance/bundle/artifacts/shell/bash/out.test.toml index 6b4c2ab5075..0db56320cf7 100644 --- a/acceptance/bundle/artifacts/shell/bash/out.test.toml +++ b/acceptance/bundle/artifacts/shell/bash/out.test.toml @@ -1,4 +1,4 @@ Cloud = false GOOS.windows = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/artifacts/shell/basic/out.test.toml b/acceptance/bundle/artifacts/shell/basic/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/artifacts/shell/basic/out.test.toml +++ b/acceptance/bundle/artifacts/shell/basic/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/artifacts/shell/cmd/out.test.toml b/acceptance/bundle/artifacts/shell/cmd/out.test.toml index c84449a1b5b..7ea98c24589 100644 --- a/acceptance/bundle/artifacts/shell/cmd/out.test.toml +++ b/acceptance/bundle/artifacts/shell/cmd/out.test.toml @@ -1,5 +1,5 @@ Cloud = false GOOS.darwin = false GOOS.linux = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/artifacts/shell/default/out.test.toml b/acceptance/bundle/artifacts/shell/default/out.test.toml index 6b4c2ab5075..0db56320cf7 100644 --- a/acceptance/bundle/artifacts/shell/default/out.test.toml +++ b/acceptance/bundle/artifacts/shell/default/out.test.toml @@ -1,4 +1,4 @@ Cloud = false GOOS.windows = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/artifacts/shell/err-bash/out.test.toml b/acceptance/bundle/artifacts/shell/err-bash/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/artifacts/shell/err-bash/out.test.toml +++ b/acceptance/bundle/artifacts/shell/err-bash/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/artifacts/shell/err-sh/out.test.toml b/acceptance/bundle/artifacts/shell/err-sh/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/artifacts/shell/err-sh/out.test.toml +++ b/acceptance/bundle/artifacts/shell/err-sh/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/artifacts/shell/invalid/out.test.toml b/acceptance/bundle/artifacts/shell/invalid/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/artifacts/shell/invalid/out.test.toml +++ b/acceptance/bundle/artifacts/shell/invalid/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/artifacts/shell/sh/out.test.toml b/acceptance/bundle/artifacts/shell/sh/out.test.toml index 6b4c2ab5075..0db56320cf7 100644 --- a/acceptance/bundle/artifacts/shell/sh/out.test.toml +++ b/acceptance/bundle/artifacts/shell/sh/out.test.toml @@ -1,4 +1,4 @@ Cloud = false GOOS.windows = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/artifacts/unique_name_libraries/out.test.toml b/acceptance/bundle/artifacts/unique_name_libraries/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/artifacts/unique_name_libraries/out.test.toml +++ b/acceptance/bundle/artifacts/unique_name_libraries/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/artifacts/upload_multiple_libraries/out.test.toml b/acceptance/bundle/artifacts/upload_multiple_libraries/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/artifacts/upload_multiple_libraries/out.test.toml +++ b/acceptance/bundle/artifacts/upload_multiple_libraries/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/artifacts/whl_change_version/out.test.toml b/acceptance/bundle/artifacts/whl_change_version/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/artifacts/whl_change_version/out.test.toml +++ b/acceptance/bundle/artifacts/whl_change_version/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/artifacts/whl_dbfs/out.test.toml b/acceptance/bundle/artifacts/whl_dbfs/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/artifacts/whl_dbfs/out.test.toml +++ b/acceptance/bundle/artifacts/whl_dbfs/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/artifacts/whl_dynamic/out.test.toml b/acceptance/bundle/artifacts/whl_dynamic/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/artifacts/whl_dynamic/out.test.toml +++ b/acceptance/bundle/artifacts/whl_dynamic/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/artifacts/whl_explicit/out.test.toml b/acceptance/bundle/artifacts/whl_explicit/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/artifacts/whl_explicit/out.test.toml +++ b/acceptance/bundle/artifacts/whl_explicit/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/artifacts/whl_implicit/out.test.toml b/acceptance/bundle/artifacts/whl_implicit/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/artifacts/whl_implicit/out.test.toml +++ b/acceptance/bundle/artifacts/whl_implicit/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/artifacts/whl_implicit_custom_path/out.test.toml b/acceptance/bundle/artifacts/whl_implicit_custom_path/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/artifacts/whl_implicit_custom_path/out.test.toml +++ b/acceptance/bundle/artifacts/whl_implicit_custom_path/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/artifacts/whl_implicit_notebook/out.test.toml b/acceptance/bundle/artifacts/whl_implicit_notebook/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/artifacts/whl_implicit_notebook/out.test.toml +++ b/acceptance/bundle/artifacts/whl_implicit_notebook/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/artifacts/whl_multiple/out.test.toml b/acceptance/bundle/artifacts/whl_multiple/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/artifacts/whl_multiple/out.test.toml +++ b/acceptance/bundle/artifacts/whl_multiple/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/artifacts/whl_no_cleanup/out.test.toml b/acceptance/bundle/artifacts/whl_no_cleanup/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/artifacts/whl_no_cleanup/out.test.toml +++ b/acceptance/bundle/artifacts/whl_no_cleanup/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/artifacts/whl_prebuilt_multiple/out.test.toml b/acceptance/bundle/artifacts/whl_prebuilt_multiple/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/artifacts/whl_prebuilt_multiple/out.test.toml +++ b/acceptance/bundle/artifacts/whl_prebuilt_multiple/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/artifacts/whl_prebuilt_outside/out.test.toml b/acceptance/bundle/artifacts/whl_prebuilt_outside/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/artifacts/whl_prebuilt_outside/out.test.toml +++ b/acceptance/bundle/artifacts/whl_prebuilt_outside/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/artifacts/whl_prebuilt_outside_dynamic/out.test.toml b/acceptance/bundle/artifacts/whl_prebuilt_outside_dynamic/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/artifacts/whl_prebuilt_outside_dynamic/out.test.toml +++ b/acceptance/bundle/artifacts/whl_prebuilt_outside_dynamic/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/artifacts/whl_via_environment_key/out.test.toml b/acceptance/bundle/artifacts/whl_via_environment_key/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/artifacts/whl_via_environment_key/out.test.toml +++ b/acceptance/bundle/artifacts/whl_via_environment_key/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/benchmarks/deploy/out.test.toml b/acceptance/bundle/benchmarks/deploy/out.test.toml index 6b4c2ab5075..0db56320cf7 100644 --- a/acceptance/bundle/benchmarks/deploy/out.test.toml +++ b/acceptance/bundle/benchmarks/deploy/out.test.toml @@ -1,4 +1,4 @@ Cloud = false GOOS.windows = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/benchmarks/plan/out.test.toml b/acceptance/bundle/benchmarks/plan/out.test.toml index 6b4c2ab5075..0db56320cf7 100644 --- a/acceptance/bundle/benchmarks/plan/out.test.toml +++ b/acceptance/bundle/benchmarks/plan/out.test.toml @@ -1,4 +1,4 @@ Cloud = false GOOS.windows = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/benchmarks/validate/out.test.toml b/acceptance/bundle/benchmarks/validate/out.test.toml index 6b4c2ab5075..0db56320cf7 100644 --- a/acceptance/bundle/benchmarks/validate/out.test.toml +++ b/acceptance/bundle/benchmarks/validate/out.test.toml @@ -1,4 +1,4 @@ Cloud = false GOOS.windows = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/bundle_tag/id/out.test.toml b/acceptance/bundle/bundle_tag/id/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/bundle_tag/id/out.test.toml +++ b/acceptance/bundle/bundle_tag/id/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/bundle_tag/url/out.test.toml b/acceptance/bundle/bundle_tag/url/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/bundle_tag/url/out.test.toml +++ b/acceptance/bundle/bundle_tag/url/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/bundle_tag/url_ref/out.test.toml b/acceptance/bundle/bundle_tag/url_ref/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/bundle_tag/url_ref/out.test.toml +++ b/acceptance/bundle/bundle_tag/url_ref/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/config-remote-sync/cli_defaults/out.test.toml b/acceptance/bundle/config-remote-sync/cli_defaults/out.test.toml index 3e6d7149be9..1d9c95acfde 100644 --- a/acceptance/bundle/config-remote-sync/cli_defaults/out.test.toml +++ b/acceptance/bundle/config-remote-sync/cli_defaults/out.test.toml @@ -1,4 +1,4 @@ Cloud = true GOOS.windows = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/config-remote-sync/config_edits/out.test.toml b/acceptance/bundle/config-remote-sync/config_edits/out.test.toml index 3e6d7149be9..1d9c95acfde 100644 --- a/acceptance/bundle/config-remote-sync/config_edits/out.test.toml +++ b/acceptance/bundle/config-remote-sync/config_edits/out.test.toml @@ -1,4 +1,4 @@ Cloud = true GOOS.windows = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/config-remote-sync/dashboard_etag/out.test.toml b/acceptance/bundle/config-remote-sync/dashboard_etag/out.test.toml index 7dbc5a56ca1..c6ad6dab334 100644 --- a/acceptance/bundle/config-remote-sync/dashboard_etag/out.test.toml +++ b/acceptance/bundle/config-remote-sync/dashboard_etag/out.test.toml @@ -1,5 +1,5 @@ Cloud = true RequiresWarehouse = true GOOS.windows = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/config-remote-sync/flushed_cache/out.test.toml b/acceptance/bundle/config-remote-sync/flushed_cache/out.test.toml index 3e6d7149be9..1d9c95acfde 100644 --- a/acceptance/bundle/config-remote-sync/flushed_cache/out.test.toml +++ b/acceptance/bundle/config-remote-sync/flushed_cache/out.test.toml @@ -1,4 +1,4 @@ Cloud = true GOOS.windows = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/config-remote-sync/formatting_preserved/out.test.toml b/acceptance/bundle/config-remote-sync/formatting_preserved/out.test.toml index 3e6d7149be9..1d9c95acfde 100644 --- a/acceptance/bundle/config-remote-sync/formatting_preserved/out.test.toml +++ b/acceptance/bundle/config-remote-sync/formatting_preserved/out.test.toml @@ -1,4 +1,4 @@ Cloud = true GOOS.windows = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/config-remote-sync/job_fields/out.test.toml b/acceptance/bundle/config-remote-sync/job_fields/out.test.toml index 86f21360595..3bd0be00d8f 100644 --- a/acceptance/bundle/config-remote-sync/job_fields/out.test.toml +++ b/acceptance/bundle/config-remote-sync/job_fields/out.test.toml @@ -1,5 +1,5 @@ Cloud = true RequiresUnityCatalog = true GOOS.windows = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/config-remote-sync/job_multiple_tasks/out.test.toml b/acceptance/bundle/config-remote-sync/job_multiple_tasks/out.test.toml index 3e6d7149be9..1d9c95acfde 100644 --- a/acceptance/bundle/config-remote-sync/job_multiple_tasks/out.test.toml +++ b/acceptance/bundle/config-remote-sync/job_multiple_tasks/out.test.toml @@ -1,4 +1,4 @@ Cloud = true GOOS.windows = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/config-remote-sync/job_params_variables/out.test.toml b/acceptance/bundle/config-remote-sync/job_params_variables/out.test.toml index 86f21360595..3bd0be00d8f 100644 --- a/acceptance/bundle/config-remote-sync/job_params_variables/out.test.toml +++ b/acceptance/bundle/config-remote-sync/job_params_variables/out.test.toml @@ -1,5 +1,5 @@ Cloud = true RequiresUnityCatalog = true GOOS.windows = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/config-remote-sync/job_pipeline_task/out.test.toml b/acceptance/bundle/config-remote-sync/job_pipeline_task/out.test.toml index 3e6d7149be9..1d9c95acfde 100644 --- a/acceptance/bundle/config-remote-sync/job_pipeline_task/out.test.toml +++ b/acceptance/bundle/config-remote-sync/job_pipeline_task/out.test.toml @@ -1,4 +1,4 @@ Cloud = true GOOS.windows = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/config-remote-sync/multiple_files/out.test.toml b/acceptance/bundle/config-remote-sync/multiple_files/out.test.toml index 3e6d7149be9..1d9c95acfde 100644 --- a/acceptance/bundle/config-remote-sync/multiple_files/out.test.toml +++ b/acceptance/bundle/config-remote-sync/multiple_files/out.test.toml @@ -1,4 +1,4 @@ Cloud = true GOOS.windows = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/config-remote-sync/multiple_resources/out.test.toml b/acceptance/bundle/config-remote-sync/multiple_resources/out.test.toml index 86f21360595..3bd0be00d8f 100644 --- a/acceptance/bundle/config-remote-sync/multiple_resources/out.test.toml +++ b/acceptance/bundle/config-remote-sync/multiple_resources/out.test.toml @@ -1,5 +1,5 @@ Cloud = true RequiresUnityCatalog = true GOOS.windows = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/config-remote-sync/output_json/out.test.toml b/acceptance/bundle/config-remote-sync/output_json/out.test.toml index 3e6d7149be9..1d9c95acfde 100644 --- a/acceptance/bundle/config-remote-sync/output_json/out.test.toml +++ b/acceptance/bundle/config-remote-sync/output_json/out.test.toml @@ -1,4 +1,4 @@ Cloud = true GOOS.windows = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/config-remote-sync/output_no_changes/out.test.toml b/acceptance/bundle/config-remote-sync/output_no_changes/out.test.toml index 3e6d7149be9..1d9c95acfde 100644 --- a/acceptance/bundle/config-remote-sync/output_no_changes/out.test.toml +++ b/acceptance/bundle/config-remote-sync/output_no_changes/out.test.toml @@ -1,4 +1,4 @@ Cloud = true GOOS.windows = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/config-remote-sync/pipeline_fields/out.test.toml b/acceptance/bundle/config-remote-sync/pipeline_fields/out.test.toml index 86f21360595..3bd0be00d8f 100644 --- a/acceptance/bundle/config-remote-sync/pipeline_fields/out.test.toml +++ b/acceptance/bundle/config-remote-sync/pipeline_fields/out.test.toml @@ -1,5 +1,5 @@ Cloud = true RequiresUnityCatalog = true GOOS.windows = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/config-remote-sync/policy_injected_cluster_fields/out.test.toml b/acceptance/bundle/config-remote-sync/policy_injected_cluster_fields/out.test.toml index 3e6d7149be9..1d9c95acfde 100644 --- a/acceptance/bundle/config-remote-sync/policy_injected_cluster_fields/out.test.toml +++ b/acceptance/bundle/config-remote-sync/policy_injected_cluster_fields/out.test.toml @@ -1,4 +1,4 @@ Cloud = true GOOS.windows = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/config-remote-sync/resolve_variables/out.test.toml b/acceptance/bundle/config-remote-sync/resolve_variables/out.test.toml index 86f21360595..3bd0be00d8f 100644 --- a/acceptance/bundle/config-remote-sync/resolve_variables/out.test.toml +++ b/acceptance/bundle/config-remote-sync/resolve_variables/out.test.toml @@ -1,5 +1,5 @@ Cloud = true RequiresUnityCatalog = true GOOS.windows = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/config-remote-sync/select_basic/out.test.toml b/acceptance/bundle/config-remote-sync/select_basic/out.test.toml index 3e6d7149be9..1d9c95acfde 100644 --- a/acceptance/bundle/config-remote-sync/select_basic/out.test.toml +++ b/acceptance/bundle/config-remote-sync/select_basic/out.test.toml @@ -1,4 +1,4 @@ Cloud = true GOOS.windows = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/config-remote-sync/select_multiple/out.test.toml b/acceptance/bundle/config-remote-sync/select_multiple/out.test.toml index 3e6d7149be9..1d9c95acfde 100644 --- a/acceptance/bundle/config-remote-sync/select_multiple/out.test.toml +++ b/acceptance/bundle/config-remote-sync/select_multiple/out.test.toml @@ -1,4 +1,4 @@ Cloud = true GOOS.windows = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/config-remote-sync/skip_permissions/out.test.toml b/acceptance/bundle/config-remote-sync/skip_permissions/out.test.toml index 3e6d7149be9..1d9c95acfde 100644 --- a/acceptance/bundle/config-remote-sync/skip_permissions/out.test.toml +++ b/acceptance/bundle/config-remote-sync/skip_permissions/out.test.toml @@ -1,4 +1,4 @@ Cloud = true GOOS.windows = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/config-remote-sync/split/cli_default_split_element/out.test.toml b/acceptance/bundle/config-remote-sync/split/cli_default_split_element/out.test.toml index b817eb8ef24..287cde8ab5c 100644 --- a/acceptance/bundle/config-remote-sync/split/cli_default_split_element/out.test.toml +++ b/acceptance/bundle/config-remote-sync/split/cli_default_split_element/out.test.toml @@ -1,4 +1,4 @@ Cloud = false GOOS.windows = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/config-remote-sync/split/dotted_target/out.test.toml b/acceptance/bundle/config-remote-sync/split/dotted_target/out.test.toml index b817eb8ef24..287cde8ab5c 100644 --- a/acceptance/bundle/config-remote-sync/split/dotted_target/out.test.toml +++ b/acceptance/bundle/config-remote-sync/split/dotted_target/out.test.toml @@ -1,4 +1,4 @@ Cloud = false GOOS.windows = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/config-remote-sync/split/isolation/out.test.toml b/acceptance/bundle/config-remote-sync/split/isolation/out.test.toml index b817eb8ef24..287cde8ab5c 100644 --- a/acceptance/bundle/config-remote-sync/split/isolation/out.test.toml +++ b/acceptance/bundle/config-remote-sync/split/isolation/out.test.toml @@ -1,4 +1,4 @@ Cloud = false GOOS.windows = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/config-remote-sync/split/keyed_edit/out.test.toml b/acceptance/bundle/config-remote-sync/split/keyed_edit/out.test.toml index b817eb8ef24..287cde8ab5c 100644 --- a/acceptance/bundle/config-remote-sync/split/keyed_edit/out.test.toml +++ b/acceptance/bundle/config-remote-sync/split/keyed_edit/out.test.toml @@ -1,4 +1,4 @@ Cloud = false GOOS.windows = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/config-remote-sync/split/keyed_remove/out.test.toml b/acceptance/bundle/config-remote-sync/split/keyed_remove/out.test.toml index b817eb8ef24..287cde8ab5c 100644 --- a/acceptance/bundle/config-remote-sync/split/keyed_remove/out.test.toml +++ b/acceptance/bundle/config-remote-sync/split/keyed_remove/out.test.toml @@ -1,4 +1,4 @@ Cloud = false GOOS.windows = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/config-remote-sync/split/keyed_rename/out.test.toml b/acceptance/bundle/config-remote-sync/split/keyed_rename/out.test.toml index b817eb8ef24..287cde8ab5c 100644 --- a/acceptance/bundle/config-remote-sync/split/keyed_rename/out.test.toml +++ b/acceptance/bundle/config-remote-sync/split/keyed_rename/out.test.toml @@ -1,4 +1,4 @@ Cloud = false GOOS.windows = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/config-remote-sync/split/keyed_twoblock/out.test.toml b/acceptance/bundle/config-remote-sync/split/keyed_twoblock/out.test.toml index b817eb8ef24..287cde8ab5c 100644 --- a/acceptance/bundle/config-remote-sync/split/keyed_twoblock/out.test.toml +++ b/acceptance/bundle/config-remote-sync/split/keyed_twoblock/out.test.toml @@ -1,4 +1,4 @@ Cloud = false GOOS.windows = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/config-remote-sync/split/multifile/out.test.toml b/acceptance/bundle/config-remote-sync/split/multifile/out.test.toml index b817eb8ef24..287cde8ab5c 100644 --- a/acceptance/bundle/config-remote-sync/split/multifile/out.test.toml +++ b/acceptance/bundle/config-remote-sync/split/multifile/out.test.toml @@ -1,4 +1,4 @@ Cloud = false GOOS.windows = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/config-remote-sync/split/nested_add_split_parent/out.test.toml b/acceptance/bundle/config-remote-sync/split/nested_add_split_parent/out.test.toml index b817eb8ef24..287cde8ab5c 100644 --- a/acceptance/bundle/config-remote-sync/split/nested_add_split_parent/out.test.toml +++ b/acceptance/bundle/config-remote-sync/split/nested_add_split_parent/out.test.toml @@ -1,4 +1,4 @@ Cloud = false GOOS.windows = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/config-remote-sync/split/nested_sequence/out.test.toml b/acceptance/bundle/config-remote-sync/split/nested_sequence/out.test.toml index b817eb8ef24..287cde8ab5c 100644 --- a/acceptance/bundle/config-remote-sync/split/nested_sequence/out.test.toml +++ b/acceptance/bundle/config-remote-sync/split/nested_sequence/out.test.toml @@ -1,4 +1,4 @@ Cloud = false GOOS.windows = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/config-remote-sync/split/positional/out.test.toml b/acceptance/bundle/config-remote-sync/split/positional/out.test.toml index b817eb8ef24..287cde8ab5c 100644 --- a/acceptance/bundle/config-remote-sync/split/positional/out.test.toml +++ b/acceptance/bundle/config-remote-sync/split/positional/out.test.toml @@ -1,4 +1,4 @@ Cloud = false GOOS.windows = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/config-remote-sync/split/remove_field_both_blocks/out.test.toml b/acceptance/bundle/config-remote-sync/split/remove_field_both_blocks/out.test.toml index b817eb8ef24..287cde8ab5c 100644 --- a/acceptance/bundle/config-remote-sync/split/remove_field_both_blocks/out.test.toml +++ b/acceptance/bundle/config-remote-sync/split/remove_field_both_blocks/out.test.toml @@ -1,4 +1,4 @@ Cloud = false GOOS.windows = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/config-remote-sync/split/remove_with_unrelated_add/out.test.toml b/acceptance/bundle/config-remote-sync/split/remove_with_unrelated_add/out.test.toml index b817eb8ef24..287cde8ab5c 100644 --- a/acceptance/bundle/config-remote-sync/split/remove_with_unrelated_add/out.test.toml +++ b/acceptance/bundle/config-remote-sync/split/remove_with_unrelated_add/out.test.toml @@ -1,4 +1,4 @@ Cloud = false GOOS.windows = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/config-remote-sync/split/rename_ambiguous_pairing/out.test.toml b/acceptance/bundle/config-remote-sync/split/rename_ambiguous_pairing/out.test.toml index b817eb8ef24..287cde8ab5c 100644 --- a/acceptance/bundle/config-remote-sync/split/rename_ambiguous_pairing/out.test.toml +++ b/acceptance/bundle/config-remote-sync/split/rename_ambiguous_pairing/out.test.toml @@ -1,4 +1,4 @@ Cloud = false GOOS.windows = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/config-remote-sync/split/rename_ambiguous_single_block/out.test.toml b/acceptance/bundle/config-remote-sync/split/rename_ambiguous_single_block/out.test.toml index b817eb8ef24..287cde8ab5c 100644 --- a/acceptance/bundle/config-remote-sync/split/rename_ambiguous_single_block/out.test.toml +++ b/acceptance/bundle/config-remote-sync/split/rename_ambiguous_single_block/out.test.toml @@ -1,4 +1,4 @@ Cloud = false GOOS.windows = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/config-remote-sync/split/rename_two_removes_one_add/out.test.toml b/acceptance/bundle/config-remote-sync/split/rename_two_removes_one_add/out.test.toml index b817eb8ef24..287cde8ab5c 100644 --- a/acceptance/bundle/config-remote-sync/split/rename_two_removes_one_add/out.test.toml +++ b/acceptance/bundle/config-remote-sync/split/rename_two_removes_one_add/out.test.toml @@ -1,4 +1,4 @@ Cloud = false GOOS.windows = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/config-remote-sync/split/target_variable/out.test.toml b/acceptance/bundle/config-remote-sync/split/target_variable/out.test.toml index b817eb8ef24..287cde8ab5c 100644 --- a/acceptance/bundle/config-remote-sync/split/target_variable/out.test.toml +++ b/acceptance/bundle/config-remote-sync/split/target_variable/out.test.toml @@ -1,4 +1,4 @@ Cloud = false GOOS.windows = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/config-remote-sync/split/variable_file_order/out.test.toml b/acceptance/bundle/config-remote-sync/split/variable_file_order/out.test.toml index b817eb8ef24..287cde8ab5c 100644 --- a/acceptance/bundle/config-remote-sync/split/variable_file_order/out.test.toml +++ b/acceptance/bundle/config-remote-sync/split/variable_file_order/out.test.toml @@ -1,4 +1,4 @@ Cloud = false GOOS.windows = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/config-remote-sync/target_override/out.test.toml b/acceptance/bundle/config-remote-sync/target_override/out.test.toml index 3e6d7149be9..1d9c95acfde 100644 --- a/acceptance/bundle/config-remote-sync/target_override/out.test.toml +++ b/acceptance/bundle/config-remote-sync/target_override/out.test.toml @@ -1,4 +1,4 @@ Cloud = true GOOS.windows = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/config-remote-sync/task_rename_revert/out.test.toml b/acceptance/bundle/config-remote-sync/task_rename_revert/out.test.toml index 3e6d7149be9..1d9c95acfde 100644 --- a/acceptance/bundle/config-remote-sync/task_rename_revert/out.test.toml +++ b/acceptance/bundle/config-remote-sync/task_rename_revert/out.test.toml @@ -1,4 +1,4 @@ Cloud = true GOOS.windows = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/config-remote-sync/validation_errors/out.test.toml b/acceptance/bundle/config-remote-sync/validation_errors/out.test.toml index b817eb8ef24..287cde8ab5c 100644 --- a/acceptance/bundle/config-remote-sync/validation_errors/out.test.toml +++ b/acceptance/bundle/config-remote-sync/validation_errors/out.test.toml @@ -1,4 +1,4 @@ Cloud = false GOOS.windows = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/debug/list-targets/out.test.toml b/acceptance/bundle/debug/list-targets/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/debug/list-targets/out.test.toml +++ b/acceptance/bundle/debug/list-targets/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/debug/out.test.toml b/acceptance/bundle/debug/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/debug/out.test.toml +++ b/acceptance/bundle/debug/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/deploy/empty-bundle/out.test.toml b/acceptance/bundle/deploy/empty-bundle/out.test.toml index 3d9d2ed4297..a405e591abd 100644 --- a/acceptance/bundle/deploy/empty-bundle/out.test.toml +++ b/acceptance/bundle/deploy/empty-bundle/out.test.toml @@ -1,4 +1,4 @@ Cloud = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENABLE_EXPERIMENTAL_YAML_SYNC = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/deploy/experimental-python/out.test.toml b/acceptance/bundle/deploy/experimental-python/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/deploy/experimental-python/out.test.toml +++ b/acceptance/bundle/deploy/experimental-python/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/deploy/fail-on-active-runs/out.test.toml b/acceptance/bundle/deploy/fail-on-active-runs/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/deploy/fail-on-active-runs/out.test.toml +++ b/acceptance/bundle/deploy/fail-on-active-runs/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/deploy/files/no-snapshot-sync/out.test.toml b/acceptance/bundle/deploy/files/no-snapshot-sync/out.test.toml index 0fc51ee009b..264de50aa96 100644 --- a/acceptance/bundle/deploy/files/no-snapshot-sync/out.test.toml +++ b/acceptance/bundle/deploy/files/no-snapshot-sync/out.test.toml @@ -1,3 +1,3 @@ Cloud = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/deploy/files/out-of-band-delete/out.test.toml b/acceptance/bundle/deploy/files/out-of-band-delete/out.test.toml index 412d37c2f86..f61c2bccd55 100644 --- a/acceptance/bundle/deploy/files/out-of-band-delete/out.test.toml +++ b/acceptance/bundle/deploy/files/out-of-band-delete/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/deploy/files/out-of-band-delete/test.toml b/acceptance/bundle/deploy/files/out-of-band-delete/test.toml index c36335a3c48..ab2cd7ec7da 100644 --- a/acceptance/bundle/deploy/files/out-of-band-delete/test.toml +++ b/acceptance/bundle/deploy/files/out-of-band-delete/test.toml @@ -1,7 +1,7 @@ # Recording needs a bundle it has seen from the start. This test seeds a state file, # so recording refuses it. TODO(DMS): drop this once existing state can be # handed over to the service (see the TODO in dstate.Open). -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] Badness = "After the remote bundle files are deleted out-of-band, the next deploy does not re-upload them until the local sync snapshot is removed." diff --git a/acceptance/bundle/deploy/force-lock-config/out.test.toml b/acceptance/bundle/deploy/force-lock-config/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/deploy/force-lock-config/out.test.toml +++ b/acceptance/bundle/deploy/force-lock-config/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/deploy/immutable-no-artifacts/out.test.toml b/acceptance/bundle/deploy/immutable-no-artifacts/out.test.toml index 51a5602947c..2c6699da193 100644 --- a/acceptance/bundle/deploy/immutable-no-artifacts/out.test.toml +++ b/acceptance/bundle/deploy/immutable-no-artifacts/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/deploy/immutable-permissions-change/out.test.toml b/acceptance/bundle/deploy/immutable-permissions-change/out.test.toml index 51a5602947c..2c6699da193 100644 --- a/acceptance/bundle/deploy/immutable-permissions-change/out.test.toml +++ b/acceptance/bundle/deploy/immutable-permissions-change/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/deploy/immutable/out.test.toml b/acceptance/bundle/deploy/immutable/out.test.toml index 51a5602947c..2c6699da193 100644 --- a/acceptance/bundle/deploy/immutable/out.test.toml +++ b/acceptance/bundle/deploy/immutable/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/deploy/mlops-stacks/out.test.toml b/acceptance/bundle/deploy/mlops-stacks/out.test.toml index 026bc5f1a2d..3587a9c012d 100644 --- a/acceptance/bundle/deploy/mlops-stacks/out.test.toml +++ b/acceptance/bundle/deploy/mlops-stacks/out.test.toml @@ -1,4 +1,4 @@ Cloud = true RequiresUnityCatalog = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/deploy/pipeline-config-dots/out.test.toml b/acceptance/bundle/deploy/pipeline-config-dots/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/deploy/pipeline-config-dots/out.test.toml +++ b/acceptance/bundle/deploy/pipeline-config-dots/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/deploy/python-notebook/out.test.toml b/acceptance/bundle/deploy/python-notebook/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/deploy/python-notebook/out.test.toml +++ b/acceptance/bundle/deploy/python-notebook/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/deploy/readplan/basic/out.test.toml b/acceptance/bundle/deploy/readplan/basic/out.test.toml index 5ab60815a3b..7daaf6fd56a 100644 --- a/acceptance/bundle/deploy/readplan/basic/out.test.toml +++ b/acceptance/bundle/deploy/readplan/basic/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/deploy/readplan/cli-version-mismatch/out.test.toml b/acceptance/bundle/deploy/readplan/cli-version-mismatch/out.test.toml index 5ab60815a3b..7daaf6fd56a 100644 --- a/acceptance/bundle/deploy/readplan/cli-version-mismatch/out.test.toml +++ b/acceptance/bundle/deploy/readplan/cli-version-mismatch/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/deploy/readplan/grants-remove-principal/out.test.toml b/acceptance/bundle/deploy/readplan/grants-remove-principal/out.test.toml index 7508f6b670e..37356d54551 100644 --- a/acceptance/bundle/deploy/readplan/grants-remove-principal/out.test.toml +++ b/acceptance/bundle/deploy/readplan/grants-remove-principal/out.test.toml @@ -1,4 +1,4 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/deploy/readplan/invalid-plan/out.test.toml b/acceptance/bundle/deploy/readplan/invalid-plan/out.test.toml index 5ab60815a3b..7daaf6fd56a 100644 --- a/acceptance/bundle/deploy/readplan/invalid-plan/out.test.toml +++ b/acceptance/bundle/deploy/readplan/invalid-plan/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/deploy/readplan/lineage-mismatch/out.test.toml b/acceptance/bundle/deploy/readplan/lineage-mismatch/out.test.toml index 5ab60815a3b..7daaf6fd56a 100644 --- a/acceptance/bundle/deploy/readplan/lineage-mismatch/out.test.toml +++ b/acceptance/bundle/deploy/readplan/lineage-mismatch/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/deploy/readplan/plan-not-found/out.test.toml b/acceptance/bundle/deploy/readplan/plan-not-found/out.test.toml index 0007abb49a1..58b3530c41c 100644 --- a/acceptance/bundle/deploy/readplan/plan-not-found/out.test.toml +++ b/acceptance/bundle/deploy/readplan/plan-not-found/out.test.toml @@ -1,4 +1,4 @@ Cloud = false GOOS.windows = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/deploy/readplan/plan-version-mismatch/out.test.toml b/acceptance/bundle/deploy/readplan/plan-version-mismatch/out.test.toml index 5ab60815a3b..7daaf6fd56a 100644 --- a/acceptance/bundle/deploy/readplan/plan-version-mismatch/out.test.toml +++ b/acceptance/bundle/deploy/readplan/plan-version-mismatch/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/deploy/readplan/postgres_role/out.test.toml b/acceptance/bundle/deploy/readplan/postgres_role/out.test.toml index 5ab60815a3b..7daaf6fd56a 100644 --- a/acceptance/bundle/deploy/readplan/postgres_role/out.test.toml +++ b/acceptance/bundle/deploy/readplan/postgres_role/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/deploy/readplan/serial-mismatch/out.test.toml b/acceptance/bundle/deploy/readplan/serial-mismatch/out.test.toml index 5ab60815a3b..7daaf6fd56a 100644 --- a/acceptance/bundle/deploy/readplan/serial-mismatch/out.test.toml +++ b/acceptance/bundle/deploy/readplan/serial-mismatch/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/deploy/readplan/terraform-error/out.test.toml b/acceptance/bundle/deploy/readplan/terraform-error/out.test.toml index 119650f619e..5ddd8dce4c7 100644 --- a/acceptance/bundle/deploy/readplan/terraform-error/out.test.toml +++ b/acceptance/bundle/deploy/readplan/terraform-error/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/deploy/readplan/test.toml b/acceptance/bundle/deploy/readplan/test.toml index d5628ffe5fe..24b27839975 100644 --- a/acceptance/bundle/deploy/readplan/test.toml +++ b/acceptance/bundle/deploy/readplan/test.toml @@ -3,4 +3,4 @@ # leaves the field unset and applying it plans an update the next time. Same reason as # EnvMatrixExclude.dms_no_readplan in acceptance/bundle/test.toml, which only covers the # tests that take the saved-plan path through the READPLAN matrix variable. -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/deploy/readplan/unknown-field/out.test.toml b/acceptance/bundle/deploy/readplan/unknown-field/out.test.toml index 5ab60815a3b..7daaf6fd56a 100644 --- a/acceptance/bundle/deploy/readplan/unknown-field/out.test.toml +++ b/acceptance/bundle/deploy/readplan/unknown-field/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/deploy/snapshot-comparison/out.test.toml b/acceptance/bundle/deploy/snapshot-comparison/out.test.toml index 944c993b0fa..c7045d40dc0 100644 --- a/acceptance/bundle/deploy/snapshot-comparison/out.test.toml +++ b/acceptance/bundle/deploy/snapshot-comparison/out.test.toml @@ -1,3 +1,3 @@ Cloud = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/deploy/spark-jar-task/out.test.toml b/acceptance/bundle/deploy/spark-jar-task/out.test.toml index 0fc51ee009b..264de50aa96 100644 --- a/acceptance/bundle/deploy/spark-jar-task/out.test.toml +++ b/acceptance/bundle/deploy/spark-jar-task/out.test.toml @@ -1,3 +1,3 @@ Cloud = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/deploy/wal/chain-3-jobs/out.test.toml b/acceptance/bundle/deploy/wal/chain-3-jobs/out.test.toml index 51a5602947c..2c6699da193 100644 --- a/acceptance/bundle/deploy/wal/chain-3-jobs/out.test.toml +++ b/acceptance/bundle/deploy/wal/chain-3-jobs/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/deploy/wal/corrupted-wal-entry/out.test.toml b/acceptance/bundle/deploy/wal/corrupted-wal-entry/out.test.toml index 5ab60815a3b..7daaf6fd56a 100644 --- a/acceptance/bundle/deploy/wal/corrupted-wal-entry/out.test.toml +++ b/acceptance/bundle/deploy/wal/corrupted-wal-entry/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/deploy/wal/corrupted-wal-entry/test.toml b/acceptance/bundle/deploy/wal/corrupted-wal-entry/test.toml index a55b7341585..597812d5b48 100644 --- a/acceptance/bundle/deploy/wal/corrupted-wal-entry/test.toml +++ b/acceptance/bundle/deploy/wal/corrupted-wal-entry/test.toml @@ -1,4 +1,4 @@ # Recording needs a bundle it has seen from the start. This test seeds a state file, # so recording refuses it. TODO(DMS): drop this once existing state can be # handed over to the service (see the TODO in dstate.Open). -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/deploy/wal/crash-after-create/out.test.toml b/acceptance/bundle/deploy/wal/crash-after-create/out.test.toml index 750e36c7a84..c2d0c886963 100644 --- a/acceptance/bundle/deploy/wal/crash-after-create/out.test.toml +++ b/acceptance/bundle/deploy/wal/crash-after-create/out.test.toml @@ -1,4 +1,4 @@ Cloud = false EnvMatrix.COMMAND = ["plan", "deploy --force-lock", "summary"] -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/deploy/wal/empty-wal/out.test.toml b/acceptance/bundle/deploy/wal/empty-wal/out.test.toml index 51a5602947c..2c6699da193 100644 --- a/acceptance/bundle/deploy/wal/empty-wal/out.test.toml +++ b/acceptance/bundle/deploy/wal/empty-wal/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/deploy/wal/failed-plan-no-wal/out.test.toml b/acceptance/bundle/deploy/wal/failed-plan-no-wal/out.test.toml index 51a5602947c..2c6699da193 100644 --- a/acceptance/bundle/deploy/wal/failed-plan-no-wal/out.test.toml +++ b/acceptance/bundle/deploy/wal/failed-plan-no-wal/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/deploy/wal/future-serial-wal/out.test.toml b/acceptance/bundle/deploy/wal/future-serial-wal/out.test.toml index 51a5602947c..2c6699da193 100644 --- a/acceptance/bundle/deploy/wal/future-serial-wal/out.test.toml +++ b/acceptance/bundle/deploy/wal/future-serial-wal/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/deploy/wal/header-only-wal/out.test.toml b/acceptance/bundle/deploy/wal/header-only-wal/out.test.toml index 51a5602947c..2c6699da193 100644 --- a/acceptance/bundle/deploy/wal/header-only-wal/out.test.toml +++ b/acceptance/bundle/deploy/wal/header-only-wal/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/deploy/wal/lineage-mismatch/out.test.toml b/acceptance/bundle/deploy/wal/lineage-mismatch/out.test.toml index 87ffa970e97..83c77c5ad48 100644 --- a/acceptance/bundle/deploy/wal/lineage-mismatch/out.test.toml +++ b/acceptance/bundle/deploy/wal/lineage-mismatch/out.test.toml @@ -1,4 +1,4 @@ Cloud = false EnvMatrix.COMMAND = ["deploy", "plan", "summary"] -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/deploy/wal/stale-wal/out.test.toml b/acceptance/bundle/deploy/wal/stale-wal/out.test.toml index 5ab60815a3b..7daaf6fd56a 100644 --- a/acceptance/bundle/deploy/wal/stale-wal/out.test.toml +++ b/acceptance/bundle/deploy/wal/stale-wal/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/deploy/wal/stale-wal/test.toml b/acceptance/bundle/deploy/wal/stale-wal/test.toml index 09ff4752240..9ed54b6da92 100644 --- a/acceptance/bundle/deploy/wal/stale-wal/test.toml +++ b/acceptance/bundle/deploy/wal/stale-wal/test.toml @@ -1,7 +1,7 @@ # Recording needs a bundle it has seen from the start. This test seeds a state file, # so recording refuses it. TODO(DMS): drop this once existing state can be # handed over to the service (see the TODO in dstate.Open). -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] # Deploy with a stale WAL (old serial) - WAL should be deleted and ignored. diff --git a/acceptance/bundle/deploy/wal/wal-with-delete/out.test.toml b/acceptance/bundle/deploy/wal/wal-with-delete/out.test.toml index 51a5602947c..2c6699da193 100644 --- a/acceptance/bundle/deploy/wal/wal-with-delete/out.test.toml +++ b/acceptance/bundle/deploy/wal/wal-with-delete/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/deploy/yaml-sync-empty-grants/out.test.toml b/acceptance/bundle/deploy/yaml-sync-empty-grants/out.test.toml index 2563df1863f..33ed6258236 100644 --- a/acceptance/bundle/deploy/yaml-sync-empty-grants/out.test.toml +++ b/acceptance/bundle/deploy/yaml-sync-empty-grants/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/deployment/bind/alert/out.test.toml b/acceptance/bundle/deployment/bind/alert/out.test.toml index cf06746b03e..79ed01757ce 100644 --- a/acceptance/bundle/deployment/bind/alert/out.test.toml +++ b/acceptance/bundle/deployment/bind/alert/out.test.toml @@ -1,3 +1,3 @@ Cloud = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/deployment/bind/catalog/out.test.toml b/acceptance/bundle/deployment/bind/catalog/out.test.toml index fa850ed32a0..fd643f3dea8 100644 --- a/acceptance/bundle/deployment/bind/catalog/out.test.toml +++ b/acceptance/bundle/deployment/bind/catalog/out.test.toml @@ -1,4 +1,4 @@ Cloud = true RequiresUnityCatalog = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/deployment/bind/cluster/out.test.toml b/acceptance/bundle/deployment/bind/cluster/out.test.toml index 67366964738..962a600bc71 100644 --- a/acceptance/bundle/deployment/bind/cluster/out.test.toml +++ b/acceptance/bundle/deployment/bind/cluster/out.test.toml @@ -1,4 +1,4 @@ Cloud = true RequiresCluster = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/deployment/bind/dashboard/out.test.toml b/acceptance/bundle/deployment/bind/dashboard/out.test.toml index 669db43661a..1ed3c9d6855 100644 --- a/acceptance/bundle/deployment/bind/dashboard/out.test.toml +++ b/acceptance/bundle/deployment/bind/dashboard/out.test.toml @@ -1,5 +1,5 @@ Cloud = true RequiresWarehouse = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/deployment/bind/dashboard/recreation/out.test.toml b/acceptance/bundle/deployment/bind/dashboard/recreation/out.test.toml index 669db43661a..1ed3c9d6855 100644 --- a/acceptance/bundle/deployment/bind/dashboard/recreation/out.test.toml +++ b/acceptance/bundle/deployment/bind/dashboard/recreation/out.test.toml @@ -1,5 +1,5 @@ Cloud = true RequiresWarehouse = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/deployment/bind/database_instance/out.test.toml b/acceptance/bundle/deployment/bind/database_instance/out.test.toml index 412d37c2f86..f61c2bccd55 100644 --- a/acceptance/bundle/deployment/bind/database_instance/out.test.toml +++ b/acceptance/bundle/deployment/bind/database_instance/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/deployment/bind/experiment/out.test.toml b/acceptance/bundle/deployment/bind/experiment/out.test.toml index cf06746b03e..79ed01757ce 100644 --- a/acceptance/bundle/deployment/bind/experiment/out.test.toml +++ b/acceptance/bundle/deployment/bind/experiment/out.test.toml @@ -1,3 +1,3 @@ Cloud = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/deployment/bind/external_location/out.test.toml b/acceptance/bundle/deployment/bind/external_location/out.test.toml index 5ab60815a3b..7daaf6fd56a 100644 --- a/acceptance/bundle/deployment/bind/external_location/out.test.toml +++ b/acceptance/bundle/deployment/bind/external_location/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/deployment/bind/genie_space/out.test.toml b/acceptance/bundle/deployment/bind/genie_space/out.test.toml index 5ab60815a3b..7daaf6fd56a 100644 --- a/acceptance/bundle/deployment/bind/genie_space/out.test.toml +++ b/acceptance/bundle/deployment/bind/genie_space/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/deployment/bind/job/already-managed-different/out.test.toml b/acceptance/bundle/deployment/bind/job/already-managed-different/out.test.toml index 412d37c2f86..f61c2bccd55 100644 --- a/acceptance/bundle/deployment/bind/job/already-managed-different/out.test.toml +++ b/acceptance/bundle/deployment/bind/job/already-managed-different/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/deployment/bind/job/already-managed-same/out.test.toml b/acceptance/bundle/deployment/bind/job/already-managed-same/out.test.toml index 412d37c2f86..f61c2bccd55 100644 --- a/acceptance/bundle/deployment/bind/job/already-managed-same/out.test.toml +++ b/acceptance/bundle/deployment/bind/job/already-managed-same/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/deployment/bind/job/engine-from-config/out.test.toml b/acceptance/bundle/deployment/bind/job/engine-from-config/out.test.toml index 5ab60815a3b..7daaf6fd56a 100644 --- a/acceptance/bundle/deployment/bind/job/engine-from-config/out.test.toml +++ b/acceptance/bundle/deployment/bind/job/engine-from-config/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/deployment/bind/job/generate-and-bind/out.test.toml b/acceptance/bundle/deployment/bind/job/generate-and-bind/out.test.toml index cf06746b03e..79ed01757ce 100644 --- a/acceptance/bundle/deployment/bind/job/generate-and-bind/out.test.toml +++ b/acceptance/bundle/deployment/bind/job/generate-and-bind/out.test.toml @@ -1,3 +1,3 @@ Cloud = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/deployment/bind/job/job-abort-bind/out.test.toml b/acceptance/bundle/deployment/bind/job/job-abort-bind/out.test.toml index cf06746b03e..79ed01757ce 100644 --- a/acceptance/bundle/deployment/bind/job/job-abort-bind/out.test.toml +++ b/acceptance/bundle/deployment/bind/job/job-abort-bind/out.test.toml @@ -1,3 +1,3 @@ Cloud = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/deployment/bind/job/job-spark-python-task/out.test.toml b/acceptance/bundle/deployment/bind/job/job-spark-python-task/out.test.toml index cf06746b03e..79ed01757ce 100644 --- a/acceptance/bundle/deployment/bind/job/job-spark-python-task/out.test.toml +++ b/acceptance/bundle/deployment/bind/job/job-spark-python-task/out.test.toml @@ -1,3 +1,3 @@ Cloud = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/deployment/bind/job/noop-job/out.test.toml b/acceptance/bundle/deployment/bind/job/noop-job/out.test.toml index 412d37c2f86..f61c2bccd55 100644 --- a/acceptance/bundle/deployment/bind/job/noop-job/out.test.toml +++ b/acceptance/bundle/deployment/bind/job/noop-job/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/deployment/bind/job/python-job/out.test.toml b/acceptance/bundle/deployment/bind/job/python-job/out.test.toml index 412d37c2f86..f61c2bccd55 100644 --- a/acceptance/bundle/deployment/bind/job/python-job/out.test.toml +++ b/acceptance/bundle/deployment/bind/job/python-job/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/deployment/bind/job/stale-state/out.test.toml b/acceptance/bundle/deployment/bind/job/stale-state/out.test.toml index 412d37c2f86..f61c2bccd55 100644 --- a/acceptance/bundle/deployment/bind/job/stale-state/out.test.toml +++ b/acceptance/bundle/deployment/bind/job/stale-state/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/deployment/bind/model-serving-endpoint/out.test.toml b/acceptance/bundle/deployment/bind/model-serving-endpoint/out.test.toml index cf06746b03e..79ed01757ce 100644 --- a/acceptance/bundle/deployment/bind/model-serving-endpoint/out.test.toml +++ b/acceptance/bundle/deployment/bind/model-serving-endpoint/out.test.toml @@ -1,3 +1,3 @@ Cloud = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/deployment/bind/pipelines/recreate/out.test.toml b/acceptance/bundle/deployment/bind/pipelines/recreate/out.test.toml index cf06746b03e..79ed01757ce 100644 --- a/acceptance/bundle/deployment/bind/pipelines/recreate/out.test.toml +++ b/acceptance/bundle/deployment/bind/pipelines/recreate/out.test.toml @@ -1,3 +1,3 @@ Cloud = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/deployment/bind/pipelines/update/out.test.toml b/acceptance/bundle/deployment/bind/pipelines/update/out.test.toml index 412d37c2f86..f61c2bccd55 100644 --- a/acceptance/bundle/deployment/bind/pipelines/update/out.test.toml +++ b/acceptance/bundle/deployment/bind/pipelines/update/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/deployment/bind/postgres_database/out.test.toml b/acceptance/bundle/deployment/bind/postgres_database/out.test.toml index 412d37c2f86..f61c2bccd55 100644 --- a/acceptance/bundle/deployment/bind/postgres_database/out.test.toml +++ b/acceptance/bundle/deployment/bind/postgres_database/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/deployment/bind/postgres_role/out.test.toml b/acceptance/bundle/deployment/bind/postgres_role/out.test.toml index 412d37c2f86..f61c2bccd55 100644 --- a/acceptance/bundle/deployment/bind/postgres_role/out.test.toml +++ b/acceptance/bundle/deployment/bind/postgres_role/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/deployment/bind/quality-monitor/out.test.toml b/acceptance/bundle/deployment/bind/quality-monitor/out.test.toml index 412d37c2f86..f61c2bccd55 100644 --- a/acceptance/bundle/deployment/bind/quality-monitor/out.test.toml +++ b/acceptance/bundle/deployment/bind/quality-monitor/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/deployment/bind/registered-model/out.test.toml b/acceptance/bundle/deployment/bind/registered-model/out.test.toml index 8755b432692..be3f1916376 100644 --- a/acceptance/bundle/deployment/bind/registered-model/out.test.toml +++ b/acceptance/bundle/deployment/bind/registered-model/out.test.toml @@ -1,4 +1,4 @@ Cloud = true RequiresUnityCatalog = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/deployment/bind/schema/out.test.toml b/acceptance/bundle/deployment/bind/schema/out.test.toml index 8755b432692..be3f1916376 100644 --- a/acceptance/bundle/deployment/bind/schema/out.test.toml +++ b/acceptance/bundle/deployment/bind/schema/out.test.toml @@ -1,4 +1,4 @@ Cloud = true RequiresUnityCatalog = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/deployment/bind/secret-scope/out.test.toml b/acceptance/bundle/deployment/bind/secret-scope/out.test.toml index 8755b432692..be3f1916376 100644 --- a/acceptance/bundle/deployment/bind/secret-scope/out.test.toml +++ b/acceptance/bundle/deployment/bind/secret-scope/out.test.toml @@ -1,4 +1,4 @@ Cloud = true RequiresUnityCatalog = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/deployment/bind/sql_warehouse/out.test.toml b/acceptance/bundle/deployment/bind/sql_warehouse/out.test.toml index 412d37c2f86..f61c2bccd55 100644 --- a/acceptance/bundle/deployment/bind/sql_warehouse/out.test.toml +++ b/acceptance/bundle/deployment/bind/sql_warehouse/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/deployment/bind/test.toml b/acceptance/bundle/deployment/bind/test.toml index 10bef2f1ccb..13253c4e640 100644 --- a/acceptance/bundle/deployment/bind/test.toml +++ b/acceptance/bundle/deployment/bind/test.toml @@ -1,2 +1,2 @@ # Bind operations are not yet supported by the Deployment Metadata Service (DMS) -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/deployment/bind/vector_search_endpoint/out.test.toml b/acceptance/bundle/deployment/bind/vector_search_endpoint/out.test.toml index fa850ed32a0..fd643f3dea8 100644 --- a/acceptance/bundle/deployment/bind/vector_search_endpoint/out.test.toml +++ b/acceptance/bundle/deployment/bind/vector_search_endpoint/out.test.toml @@ -1,4 +1,4 @@ Cloud = true RequiresUnityCatalog = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/deployment/bind/vector_search_index/out.test.toml b/acceptance/bundle/deployment/bind/vector_search_index/out.test.toml index b946c552d4c..10ff98fa2af 100644 --- a/acceptance/bundle/deployment/bind/vector_search_index/out.test.toml +++ b/acceptance/bundle/deployment/bind/vector_search_index/out.test.toml @@ -1,5 +1,5 @@ Cloud = true CloudSlow = true RequiresUnityCatalog = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/deployment/bind/volume/out.test.toml b/acceptance/bundle/deployment/bind/volume/out.test.toml index 8755b432692..be3f1916376 100644 --- a/acceptance/bundle/deployment/bind/volume/out.test.toml +++ b/acceptance/bundle/deployment/bind/volume/out.test.toml @@ -1,4 +1,4 @@ Cloud = true RequiresUnityCatalog = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/deployment/unbind/engine-from-config/out.test.toml b/acceptance/bundle/deployment/unbind/engine-from-config/out.test.toml index 5ab60815a3b..7daaf6fd56a 100644 --- a/acceptance/bundle/deployment/unbind/engine-from-config/out.test.toml +++ b/acceptance/bundle/deployment/unbind/engine-from-config/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/deployment/unbind/grants/out.test.toml b/acceptance/bundle/deployment/unbind/grants/out.test.toml index 8755b432692..be3f1916376 100644 --- a/acceptance/bundle/deployment/unbind/grants/out.test.toml +++ b/acceptance/bundle/deployment/unbind/grants/out.test.toml @@ -1,4 +1,4 @@ Cloud = true RequiresUnityCatalog = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/deployment/unbind/job/out.test.toml b/acceptance/bundle/deployment/unbind/job/out.test.toml index 412d37c2f86..f61c2bccd55 100644 --- a/acceptance/bundle/deployment/unbind/job/out.test.toml +++ b/acceptance/bundle/deployment/unbind/job/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/deployment/unbind/permissions/out.test.toml b/acceptance/bundle/deployment/unbind/permissions/out.test.toml index cf06746b03e..79ed01757ce 100644 --- a/acceptance/bundle/deployment/unbind/permissions/out.test.toml +++ b/acceptance/bundle/deployment/unbind/permissions/out.test.toml @@ -1,3 +1,3 @@ Cloud = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/deployment/unbind/python-job/out.test.toml b/acceptance/bundle/deployment/unbind/python-job/out.test.toml index 412d37c2f86..f61c2bccd55 100644 --- a/acceptance/bundle/deployment/unbind/python-job/out.test.toml +++ b/acceptance/bundle/deployment/unbind/python-job/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/deployment/unbind/test.toml b/acceptance/bundle/deployment/unbind/test.toml index 2be1dcf74aa..cd3ddd3431d 100644 --- a/acceptance/bundle/deployment/unbind/test.toml +++ b/acceptance/bundle/deployment/unbind/test.toml @@ -1,2 +1,2 @@ # Unbind operations are not yet supported by the Deployment Metadata Service (DMS) -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/destroy/all-resources/out.test.toml b/acceptance/bundle/destroy/all-resources/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/destroy/all-resources/out.test.toml +++ b/acceptance/bundle/destroy/all-resources/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/destroy/force-lock-node-limit/out.test.toml b/acceptance/bundle/destroy/force-lock-node-limit/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/destroy/force-lock-node-limit/out.test.toml +++ b/acceptance/bundle/destroy/force-lock-node-limit/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/destroy/jobs-and-pipeline/out.test.toml b/acceptance/bundle/destroy/jobs-and-pipeline/out.test.toml index 0fc51ee009b..264de50aa96 100644 --- a/acceptance/bundle/destroy/jobs-and-pipeline/out.test.toml +++ b/acceptance/bundle/destroy/jobs-and-pipeline/out.test.toml @@ -1,3 +1,3 @@ Cloud = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/destroy/lineage-mismatch-after-redeploy/out.test.toml b/acceptance/bundle/destroy/lineage-mismatch-after-redeploy/out.test.toml index 51a5602947c..2c6699da193 100644 --- a/acceptance/bundle/destroy/lineage-mismatch-after-redeploy/out.test.toml +++ b/acceptance/bundle/destroy/lineage-mismatch-after-redeploy/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/dms/depends-on/out.test.toml b/acceptance/bundle/dms/depends-on/out.test.toml index 5ab60815a3b..7daaf6fd56a 100644 --- a/acceptance/bundle/dms/depends-on/out.test.toml +++ b/acceptance/bundle/dms/depends-on/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/dms/emptied-resource/out.test.toml b/acceptance/bundle/dms/emptied-resource/out.test.toml index 5ab60815a3b..7daaf6fd56a 100644 --- a/acceptance/bundle/dms/emptied-resource/out.test.toml +++ b/acceptance/bundle/dms/emptied-resource/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/dms/emptied-resource/output.txt b/acceptance/bundle/dms/emptied-resource/output.txt index d65afaabd78..cf9f021cc94 100644 --- a/acceptance/bundle/dms/emptied-resource/output.txt +++ b/acceptance/bundle/dms/emptied-resource/output.txt @@ -6,7 +6,7 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> update_file.py databricks.yml grants: [{ principal: someone@example.com, privileges: [USE_SCHEMA] }] grants: [] +>>> update_file.py databricks.yml grants: [{principal: someone@example.com, privileges: [USE_SCHEMA]}] grants: [] >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-emptied-resource/default/files... diff --git a/acceptance/bundle/dms/emptied-resource/script b/acceptance/bundle/dms/emptied-resource/script index b5e8ca666d0..049bbc9cd80 100644 --- a/acceptance/bundle/dms/emptied-resource/script +++ b/acceptance/bundle/dms/emptied-resource/script @@ -1,6 +1,6 @@ title "Deploy a schema with one grant, then revoke it so the grants node empties out" trace $CLI bundle deploy -trace update_file.py databricks.yml 'grants: [{ principal: someone@example.com, privileges: [USE_SCHEMA] }]' 'grants: []' +trace update_file.py databricks.yml 'grants: [{principal: someone@example.com, privileges: [USE_SCHEMA]}]' 'grants: []' trace $CLI bundle deploy # The emptied node is recorded as a delete, not as the update that emptied it: the service diff --git a/acceptance/bundle/dms/existing-state/out.test.toml b/acceptance/bundle/dms/existing-state/out.test.toml index 5ab60815a3b..7daaf6fd56a 100644 --- a/acceptance/bundle/dms/existing-state/out.test.toml +++ b/acceptance/bundle/dms/existing-state/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/dms/multiple-resources/out.test.toml b/acceptance/bundle/dms/multiple-resources/out.test.toml index 5ab60815a3b..7daaf6fd56a 100644 --- a/acceptance/bundle/dms/multiple-resources/out.test.toml +++ b/acceptance/bundle/dms/multiple-resources/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/dms/no-drift/out.test.toml b/acceptance/bundle/dms/no-drift/out.test.toml index 5ab60815a3b..7daaf6fd56a 100644 --- a/acceptance/bundle/dms/no-drift/out.test.toml +++ b/acceptance/bundle/dms/no-drift/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/dms/no-resources/out.test.toml b/acceptance/bundle/dms/no-resources/out.test.toml index 5ab60815a3b..7daaf6fd56a 100644 --- a/acceptance/bundle/dms/no-resources/out.test.toml +++ b/acceptance/bundle/dms/no-resources/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/dms/not-supported/out.test.toml b/acceptance/bundle/dms/not-supported/out.test.toml index 5ab60815a3b..7daaf6fd56a 100644 --- a/acceptance/bundle/dms/not-supported/out.test.toml +++ b/acceptance/bundle/dms/not-supported/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/dms/operation-upload-fails/out.test.toml b/acceptance/bundle/dms/operation-upload-fails/out.test.toml index 5ab60815a3b..7daaf6fd56a 100644 --- a/acceptance/bundle/dms/operation-upload-fails/out.test.toml +++ b/acceptance/bundle/dms/operation-upload-fails/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/dms/partial-update/out.test.toml b/acceptance/bundle/dms/partial-update/out.test.toml index 5ab60815a3b..7daaf6fd56a 100644 --- a/acceptance/bundle/dms/partial-update/out.test.toml +++ b/acceptance/bundle/dms/partial-update/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/dms/provenance/out.test.toml b/acceptance/bundle/dms/provenance/out.test.toml index 5ab60815a3b..7daaf6fd56a 100644 --- a/acceptance/bundle/dms/provenance/out.test.toml +++ b/acceptance/bundle/dms/provenance/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/dms/record-failure/out.test.toml b/acceptance/bundle/dms/record-failure/out.test.toml index 5ab60815a3b..7daaf6fd56a 100644 --- a/acceptance/bundle/dms/record-failure/out.test.toml +++ b/acceptance/bundle/dms/record-failure/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/dms/record/out.test.toml b/acceptance/bundle/dms/record/out.test.toml index 5ab60815a3b..7daaf6fd56a 100644 --- a/acceptance/bundle/dms/record/out.test.toml +++ b/acceptance/bundle/dms/record/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/dms/redeploy-after-destroy/out.test.toml b/acceptance/bundle/dms/redeploy-after-destroy/out.test.toml index 5ab60815a3b..7daaf6fd56a 100644 --- a/acceptance/bundle/dms/redeploy-after-destroy/out.test.toml +++ b/acceptance/bundle/dms/redeploy-after-destroy/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/dms/summary/out.test.toml b/acceptance/bundle/dms/summary/out.test.toml index 5ab60815a3b..7daaf6fd56a 100644 --- a/acceptance/bundle/dms/summary/out.test.toml +++ b/acceptance/bundle/dms/summary/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/dms/test.toml b/acceptance/bundle/dms/test.toml index f7bd9ec6fbb..041c3da7fc1 100644 --- a/acceptance/bundle/dms/test.toml +++ b/acceptance/bundle/dms/test.toml @@ -5,8 +5,9 @@ Cloud = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] # These tests enable recording through experimental.record_deployment_history, so the -# DATABRICKS_BUNDLE_DMS variant the rest of the suite adds would just duplicate them. -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] +# DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY variant the rest of the suite adds would +# just duplicate them. +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] RecordRequests = true diff --git a/acceptance/bundle/dms/version-never-created/out.test.toml b/acceptance/bundle/dms/version-never-created/out.test.toml index 5ab60815a3b..7daaf6fd56a 100644 --- a/acceptance/bundle/dms/version-never-created/out.test.toml +++ b/acceptance/bundle/dms/version-never-created/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/empty_string_dropped/out.test.toml b/acceptance/bundle/empty_string_dropped/out.test.toml index 51a5602947c..2c6699da193 100644 --- a/acceptance/bundle/empty_string_dropped/out.test.toml +++ b/acceptance/bundle/empty_string_dropped/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/empty_string_variable/out.test.toml b/acceptance/bundle/empty_string_variable/out.test.toml index 51a5602947c..2c6699da193 100644 --- a/acceptance/bundle/empty_string_variable/out.test.toml +++ b/acceptance/bundle/empty_string_variable/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/environments/dependencies/out.test.toml b/acceptance/bundle/environments/dependencies/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/environments/dependencies/out.test.toml +++ b/acceptance/bundle/environments/dependencies/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/experimental/skip_name_prefix_for_schema/out.test.toml b/acceptance/bundle/experimental/skip_name_prefix_for_schema/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/experimental/skip_name_prefix_for_schema/out.test.toml +++ b/acceptance/bundle/experimental/skip_name_prefix_for_schema/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/generate/alert/out.test.toml b/acceptance/bundle/generate/alert/out.test.toml index 0fc51ee009b..264de50aa96 100644 --- a/acceptance/bundle/generate/alert/out.test.toml +++ b/acceptance/bundle/generate/alert/out.test.toml @@ -1,3 +1,3 @@ Cloud = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/generate/alert_existing_id_not_found/out.test.toml b/acceptance/bundle/generate/alert_existing_id_not_found/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/generate/alert_existing_id_not_found/out.test.toml +++ b/acceptance/bundle/generate/alert_existing_id_not_found/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/generate/app_not_yet_deployed/out.test.toml b/acceptance/bundle/generate/app_not_yet_deployed/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/generate/app_not_yet_deployed/out.test.toml +++ b/acceptance/bundle/generate/app_not_yet_deployed/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/generate/app_subfolders/out.test.toml b/acceptance/bundle/generate/app_subfolders/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/generate/app_subfolders/out.test.toml +++ b/acceptance/bundle/generate/app_subfolders/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/generate/auto-bind/out.test.toml b/acceptance/bundle/generate/auto-bind/out.test.toml index 944c993b0fa..c7045d40dc0 100644 --- a/acceptance/bundle/generate/auto-bind/out.test.toml +++ b/acceptance/bundle/generate/auto-bind/out.test.toml @@ -1,3 +1,3 @@ Cloud = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/generate/dashboard-inplace/out.test.toml b/acceptance/bundle/generate/dashboard-inplace/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/generate/dashboard-inplace/out.test.toml +++ b/acceptance/bundle/generate/dashboard-inplace/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/generate/dashboard/out.test.toml b/acceptance/bundle/generate/dashboard/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/generate/dashboard/out.test.toml +++ b/acceptance/bundle/generate/dashboard/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/generate/dashboard_existing_id_not_found/out.test.toml b/acceptance/bundle/generate/dashboard_existing_id_not_found/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/generate/dashboard_existing_id_not_found/out.test.toml +++ b/acceptance/bundle/generate/dashboard_existing_id_not_found/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/generate/dashboard_existing_path_nominal/out.test.toml b/acceptance/bundle/generate/dashboard_existing_path_nominal/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/generate/dashboard_existing_path_nominal/out.test.toml +++ b/acceptance/bundle/generate/dashboard_existing_path_nominal/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/generate/dashboard_existing_path_not_found/out.test.toml b/acceptance/bundle/generate/dashboard_existing_path_not_found/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/generate/dashboard_existing_path_not_found/out.test.toml +++ b/acceptance/bundle/generate/dashboard_existing_path_not_found/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/generate/designer_job/out.test.toml b/acceptance/bundle/generate/designer_job/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/generate/designer_job/out.test.toml +++ b/acceptance/bundle/generate/designer_job/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/generate/genie_space/out.test.toml b/acceptance/bundle/generate/genie_space/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/generate/genie_space/out.test.toml +++ b/acceptance/bundle/generate/genie_space/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/generate/genie_space_existing_id_not_found/out.test.toml b/acceptance/bundle/generate/genie_space_existing_id_not_found/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/generate/genie_space_existing_id_not_found/out.test.toml +++ b/acceptance/bundle/generate/genie_space_existing_id_not_found/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/generate/genie_space_inplace/out.test.toml b/acceptance/bundle/generate/genie_space_inplace/out.test.toml index 51a5602947c..2c6699da193 100644 --- a/acceptance/bundle/generate/genie_space_inplace/out.test.toml +++ b/acceptance/bundle/generate/genie_space_inplace/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/generate/git_job/out.test.toml b/acceptance/bundle/generate/git_job/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/generate/git_job/out.test.toml +++ b/acceptance/bundle/generate/git_job/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/generate/include_warning/out.test.toml b/acceptance/bundle/generate/include_warning/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/generate/include_warning/out.test.toml +++ b/acceptance/bundle/generate/include_warning/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/generate/ipynb_job/out.test.toml b/acceptance/bundle/generate/ipynb_job/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/generate/ipynb_job/out.test.toml +++ b/acceptance/bundle/generate/ipynb_job/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/generate/job_nested_notebooks/out.test.toml b/acceptance/bundle/generate/job_nested_notebooks/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/generate/job_nested_notebooks/out.test.toml +++ b/acceptance/bundle/generate/job_nested_notebooks/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/generate/lakeflow_pipelines/out.test.toml b/acceptance/bundle/generate/lakeflow_pipelines/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/generate/lakeflow_pipelines/out.test.toml +++ b/acceptance/bundle/generate/lakeflow_pipelines/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/generate/pipeline/out.test.toml b/acceptance/bundle/generate/pipeline/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/generate/pipeline/out.test.toml +++ b/acceptance/bundle/generate/pipeline/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/generate/pipeline_and_deploy/out.test.toml b/acceptance/bundle/generate/pipeline_and_deploy/out.test.toml index 0fc51ee009b..264de50aa96 100644 --- a/acceptance/bundle/generate/pipeline_and_deploy/out.test.toml +++ b/acceptance/bundle/generate/pipeline_and_deploy/out.test.toml @@ -1,3 +1,3 @@ Cloud = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/generate/pipeline_with_glob/out.test.toml b/acceptance/bundle/generate/pipeline_with_glob/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/generate/pipeline_with_glob/out.test.toml +++ b/acceptance/bundle/generate/pipeline_with_glob/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/generate/pipeline_with_sql/out.test.toml b/acceptance/bundle/generate/pipeline_with_sql/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/generate/pipeline_with_sql/out.test.toml +++ b/acceptance/bundle/generate/pipeline_with_sql/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/generate/python_job/out.test.toml b/acceptance/bundle/generate/python_job/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/generate/python_job/out.test.toml +++ b/acceptance/bundle/generate/python_job/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/generate/python_job_and_deploy/out.test.toml b/acceptance/bundle/generate/python_job_and_deploy/out.test.toml index 0fc51ee009b..264de50aa96 100644 --- a/acceptance/bundle/generate/python_job_and_deploy/out.test.toml +++ b/acceptance/bundle/generate/python_job_and_deploy/out.test.toml @@ -1,3 +1,3 @@ Cloud = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/generate/spark_python_task_job/out.test.toml b/acceptance/bundle/generate/spark_python_task_job/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/generate/spark_python_task_job/out.test.toml +++ b/acceptance/bundle/generate/spark_python_task_job/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/git-permerror/out.test.toml b/acceptance/bundle/git-permerror/out.test.toml index 6b4c2ab5075..0db56320cf7 100644 --- a/acceptance/bundle/git-permerror/out.test.toml +++ b/acceptance/bundle/git-permerror/out.test.toml @@ -1,4 +1,4 @@ Cloud = false GOOS.windows = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/help/bundle-deploy/out.test.toml b/acceptance/bundle/help/bundle-deploy/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/help/bundle-deploy/out.test.toml +++ b/acceptance/bundle/help/bundle-deploy/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/help/bundle-deployment-migrate/out.test.toml b/acceptance/bundle/help/bundle-deployment-migrate/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/help/bundle-deployment-migrate/out.test.toml +++ b/acceptance/bundle/help/bundle-deployment-migrate/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/help/bundle-deployment/out.test.toml b/acceptance/bundle/help/bundle-deployment/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/help/bundle-deployment/out.test.toml +++ b/acceptance/bundle/help/bundle-deployment/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/help/bundle-destroy/out.test.toml b/acceptance/bundle/help/bundle-destroy/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/help/bundle-destroy/out.test.toml +++ b/acceptance/bundle/help/bundle-destroy/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/help/bundle-generate-dashboard/out.test.toml b/acceptance/bundle/help/bundle-generate-dashboard/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/help/bundle-generate-dashboard/out.test.toml +++ b/acceptance/bundle/help/bundle-generate-dashboard/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/help/bundle-generate-job/out.test.toml b/acceptance/bundle/help/bundle-generate-job/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/help/bundle-generate-job/out.test.toml +++ b/acceptance/bundle/help/bundle-generate-job/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/help/bundle-generate-pipeline/out.test.toml b/acceptance/bundle/help/bundle-generate-pipeline/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/help/bundle-generate-pipeline/out.test.toml +++ b/acceptance/bundle/help/bundle-generate-pipeline/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/help/bundle-generate/out.test.toml b/acceptance/bundle/help/bundle-generate/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/help/bundle-generate/out.test.toml +++ b/acceptance/bundle/help/bundle-generate/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/help/bundle-init/out.test.toml b/acceptance/bundle/help/bundle-init/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/help/bundle-init/out.test.toml +++ b/acceptance/bundle/help/bundle-init/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/help/bundle-open/out.test.toml b/acceptance/bundle/help/bundle-open/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/help/bundle-open/out.test.toml +++ b/acceptance/bundle/help/bundle-open/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/help/bundle-run/out.test.toml b/acceptance/bundle/help/bundle-run/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/help/bundle-run/out.test.toml +++ b/acceptance/bundle/help/bundle-run/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/help/bundle-schema/out.test.toml b/acceptance/bundle/help/bundle-schema/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/help/bundle-schema/out.test.toml +++ b/acceptance/bundle/help/bundle-schema/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/help/bundle-summary/out.test.toml b/acceptance/bundle/help/bundle-summary/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/help/bundle-summary/out.test.toml +++ b/acceptance/bundle/help/bundle-summary/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/help/bundle-sync/out.test.toml b/acceptance/bundle/help/bundle-sync/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/help/bundle-sync/out.test.toml +++ b/acceptance/bundle/help/bundle-sync/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/help/bundle-validate/out.test.toml b/acceptance/bundle/help/bundle-validate/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/help/bundle-validate/out.test.toml +++ b/acceptance/bundle/help/bundle-validate/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/help/bundle/out.test.toml b/acceptance/bundle/help/bundle/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/help/bundle/out.test.toml +++ b/acceptance/bundle/help/bundle/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/includes/glob_in_root_path/out.test.toml b/acceptance/bundle/includes/glob_in_root_path/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/includes/glob_in_root_path/out.test.toml +++ b/acceptance/bundle/includes/glob_in_root_path/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/includes/include_outside_root/out.test.toml b/acceptance/bundle/includes/include_outside_root/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/includes/include_outside_root/out.test.toml +++ b/acceptance/bundle/includes/include_outside_root/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/includes/non_yaml_in_include/out.test.toml b/acceptance/bundle/includes/non_yaml_in_include/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/includes/non_yaml_in_include/out.test.toml +++ b/acceptance/bundle/includes/non_yaml_in_include/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/includes/yml_outside_root/out.test.toml b/acceptance/bundle/includes/yml_outside_root/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/includes/yml_outside_root/out.test.toml +++ b/acceptance/bundle/includes/yml_outside_root/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/integration_whl/base/out.test.toml b/acceptance/bundle/integration_whl/base/out.test.toml index ef4c598cd59..d1a88727b97 100644 --- a/acceptance/bundle/integration_whl/base/out.test.toml +++ b/acceptance/bundle/integration_whl/base/out.test.toml @@ -1,4 +1,4 @@ Cloud = true CloudSlow = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/integration_whl/custom_params/out.test.toml b/acceptance/bundle/integration_whl/custom_params/out.test.toml index ef4c598cd59..d1a88727b97 100644 --- a/acceptance/bundle/integration_whl/custom_params/out.test.toml +++ b/acceptance/bundle/integration_whl/custom_params/out.test.toml @@ -1,4 +1,4 @@ Cloud = true CloudSlow = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/integration_whl/interactive_cluster/out.test.toml b/acceptance/bundle/integration_whl/interactive_cluster/out.test.toml index ef4c598cd59..d1a88727b97 100644 --- a/acceptance/bundle/integration_whl/interactive_cluster/out.test.toml +++ b/acceptance/bundle/integration_whl/interactive_cluster/out.test.toml @@ -1,4 +1,4 @@ Cloud = true CloudSlow = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/integration_whl/interactive_cluster_dynamic_version/out.test.toml b/acceptance/bundle/integration_whl/interactive_cluster_dynamic_version/out.test.toml index 07b1635cdca..e8824d9a1ac 100644 --- a/acceptance/bundle/integration_whl/interactive_cluster_dynamic_version/out.test.toml +++ b/acceptance/bundle/integration_whl/interactive_cluster_dynamic_version/out.test.toml @@ -1,5 +1,5 @@ Cloud = true CloudSlow = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] EnvMatrix.DATA_SECURITY_MODE = ["USER_ISOLATION", "SINGLE_USER"] diff --git a/acceptance/bundle/integration_whl/interactive_single_user/out.test.toml b/acceptance/bundle/integration_whl/interactive_single_user/out.test.toml index ef4c598cd59..d1a88727b97 100644 --- a/acceptance/bundle/integration_whl/interactive_single_user/out.test.toml +++ b/acceptance/bundle/integration_whl/interactive_single_user/out.test.toml @@ -1,4 +1,4 @@ Cloud = true CloudSlow = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/integration_whl/serverless/out.test.toml b/acceptance/bundle/integration_whl/serverless/out.test.toml index ee7034036dd..754a91586c5 100644 --- a/acceptance/bundle/integration_whl/serverless/out.test.toml +++ b/acceptance/bundle/integration_whl/serverless/out.test.toml @@ -1,5 +1,5 @@ Cloud = true CloudSlow = true RequiresUnityCatalog = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/integration_whl/serverless_custom_params/out.test.toml b/acceptance/bundle/integration_whl/serverless_custom_params/out.test.toml index ee7034036dd..754a91586c5 100644 --- a/acceptance/bundle/integration_whl/serverless_custom_params/out.test.toml +++ b/acceptance/bundle/integration_whl/serverless_custom_params/out.test.toml @@ -1,5 +1,5 @@ Cloud = true CloudSlow = true RequiresUnityCatalog = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/integration_whl/serverless_dynamic_version/out.test.toml b/acceptance/bundle/integration_whl/serverless_dynamic_version/out.test.toml index ee7034036dd..754a91586c5 100644 --- a/acceptance/bundle/integration_whl/serverless_dynamic_version/out.test.toml +++ b/acceptance/bundle/integration_whl/serverless_dynamic_version/out.test.toml @@ -1,5 +1,5 @@ Cloud = true CloudSlow = true RequiresUnityCatalog = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/integration_whl/wrapper/out.test.toml b/acceptance/bundle/integration_whl/wrapper/out.test.toml index ab06e47eeea..aa1a4aab182 100644 --- a/acceptance/bundle/integration_whl/wrapper/out.test.toml +++ b/acceptance/bundle/integration_whl/wrapper/out.test.toml @@ -2,5 +2,5 @@ Cloud = true CloudSlow = true CloudEnvs.aws = false CloudEnvs.gcp = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/integration_whl/wrapper_custom_params/out.test.toml b/acceptance/bundle/integration_whl/wrapper_custom_params/out.test.toml index ab06e47eeea..aa1a4aab182 100644 --- a/acceptance/bundle/integration_whl/wrapper_custom_params/out.test.toml +++ b/acceptance/bundle/integration_whl/wrapper_custom_params/out.test.toml @@ -2,5 +2,5 @@ Cloud = true CloudSlow = true CloudEnvs.aws = false CloudEnvs.gcp = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/invariant/continue_293/out.test.toml b/acceptance/bundle/invariant/continue_293/out.test.toml index 3048fdc3f19..afec0f5be81 100644 --- a/acceptance/bundle/invariant/continue_293/out.test.toml +++ b/acceptance/bundle/invariant/continue_293/out.test.toml @@ -1,7 +1,7 @@ Cloud = true RequiresUnityCatalog = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] EnvMatrix.INPUT_CONFIG = [ "alert.yml.tmpl", "app.yml.tmpl", diff --git a/acceptance/bundle/invariant/continue_293/test.toml b/acceptance/bundle/invariant/continue_293/test.toml index 5daa377779c..0766d3fc39f 100644 --- a/acceptance/bundle/invariant/continue_293/test.toml +++ b/acceptance/bundle/invariant/continue_293/test.toml @@ -2,7 +2,7 @@ # service, so the resources it creates are recorded nowhere. Reading state from the # service then finds none and plans a create on top of them. Adopting resources a # pre-DMS CLI deployed is a migration story of its own, not something this test covers. -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] # $resources references to permissions and grants are not supported on v0.293.0 EnvMatrixExclude.no_permission_ref = ["INPUT_CONFIG=job_permission_ref.yml.tmpl"] diff --git a/acceptance/bundle/invariant/delete_idempotent/out.test.toml b/acceptance/bundle/invariant/delete_idempotent/out.test.toml index 693d9534ea1..76a280fd6c3 100644 --- a/acceptance/bundle/invariant/delete_idempotent/out.test.toml +++ b/acceptance/bundle/invariant/delete_idempotent/out.test.toml @@ -1,7 +1,7 @@ Cloud = true RequiresUnityCatalog = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] EnvMatrix.INPUT_CONFIG = [ "alert.yml.tmpl", "app.yml.tmpl", diff --git a/acceptance/bundle/invariant/delete_idempotent/test.toml b/acceptance/bundle/invariant/delete_idempotent/test.toml index aab4929d07c..df880553ee2 100644 --- a/acceptance/bundle/invariant/delete_idempotent/test.toml +++ b/acceptance/bundle/invariant/delete_idempotent/test.toml @@ -2,7 +2,7 @@ # wipes the remote path the deployment record lives under, so recording refuses it. # TODO(DMS): drop this once existing state can be handed over to the service (see # the TODO in dstate.Open). -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/invariant/destroy_idempotent/out.test.toml b/acceptance/bundle/invariant/destroy_idempotent/out.test.toml index 693d9534ea1..76a280fd6c3 100644 --- a/acceptance/bundle/invariant/destroy_idempotent/out.test.toml +++ b/acceptance/bundle/invariant/destroy_idempotent/out.test.toml @@ -1,7 +1,7 @@ Cloud = true RequiresUnityCatalog = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] EnvMatrix.INPUT_CONFIG = [ "alert.yml.tmpl", "app.yml.tmpl", diff --git a/acceptance/bundle/invariant/destroy_idempotent/test.toml b/acceptance/bundle/invariant/destroy_idempotent/test.toml index ddd6d204f5d..4ee32d2cc52 100644 --- a/acceptance/bundle/invariant/destroy_idempotent/test.toml +++ b/acceptance/bundle/invariant/destroy_idempotent/test.toml @@ -2,7 +2,7 @@ # wipes the remote path the deployment record lives under, so recording refuses it. # TODO(DMS): drop this once existing state can be handed over to the service (see # the TODO in dstate.Open). -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/invariant/migrate/out.test.toml b/acceptance/bundle/invariant/migrate/out.test.toml index 7385ea3df8e..af545cef65b 100644 --- a/acceptance/bundle/invariant/migrate/out.test.toml +++ b/acceptance/bundle/invariant/migrate/out.test.toml @@ -1,7 +1,7 @@ Cloud = true RequiresUnityCatalog = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] EnvMatrix.INPUT_CONFIG = [ "alert.yml.tmpl", "app.yml.tmpl", diff --git a/acceptance/bundle/invariant/migrate/test.toml b/acceptance/bundle/invariant/migrate/test.toml index 67b5a9a6c18..972dda9a186 100644 --- a/acceptance/bundle/invariant/migrate/test.toml +++ b/acceptance/bundle/invariant/migrate/test.toml @@ -1,7 +1,7 @@ # Recording needs a bundle it has seen from the start. This test seeds a state file, # so recording refuses it. TODO(DMS): drop this once existing state can be # handed over to the service (see the TODO in dstate.Open). -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] # vector_search_endpoints and vector_search_indexes have no terraform converter EnvMatrixExclude.no_vector_search_endpoint = ["INPUT_CONFIG=vector_search_endpoint.yml.tmpl"] diff --git a/acceptance/bundle/invariant/no_drift/out.test.toml b/acceptance/bundle/invariant/no_drift/out.test.toml index 2643e6319a9..4f11eb25291 100644 --- a/acceptance/bundle/invariant/no_drift/out.test.toml +++ b/acceptance/bundle/invariant/no_drift/out.test.toml @@ -1,7 +1,7 @@ Cloud = true RequiresUnityCatalog = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] EnvMatrix.INPUT_CONFIG = [ "alert.yml.tmpl", "app.yml.tmpl", diff --git a/acceptance/bundle/invariant/no_drift/test.toml b/acceptance/bundle/invariant/no_drift/test.toml index ddcf203eb9a..097cf7f8207 100644 --- a/acceptance/bundle/invariant/no_drift/test.toml +++ b/acceptance/bundle/invariant/no_drift/test.toml @@ -4,4 +4,4 @@ EnvMatrix.READPLAN = ["", "1"] # deployment metadata service accepts. Recording skips the resource with a warning, so it # is absent from the state the service reports and the next plan wants to create it again. # Raising the limit or splitting the state is a service-side decision. -EnvMatrixExclude.dms_state_too_large = ["DATABRICKS_BUNDLE_DMS=true", "INPUT_CONFIG=job_pydabs_1000_tasks.yml.tmpl"] +EnvMatrixExclude.dms_state_too_large = ["DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY=true", "INPUT_CONFIG=job_pydabs_1000_tasks.yml.tmpl"] diff --git a/acceptance/bundle/libraries/maven/out.test.toml b/acceptance/bundle/libraries/maven/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/libraries/maven/out.test.toml +++ b/acceptance/bundle/libraries/maven/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/libraries/outside_of_bundle_root/out.test.toml b/acceptance/bundle/libraries/outside_of_bundle_root/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/libraries/outside_of_bundle_root/out.test.toml +++ b/acceptance/bundle/libraries/outside_of_bundle_root/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/libraries/pypi/out.test.toml b/acceptance/bundle/libraries/pypi/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/libraries/pypi/out.test.toml +++ b/acceptance/bundle/libraries/pypi/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/lifecycle/prevent-destroy/out.test.toml b/acceptance/bundle/lifecycle/prevent-destroy/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/lifecycle/prevent-destroy/out.test.toml +++ b/acceptance/bundle/lifecycle/prevent-destroy/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/lifecycle/started-validation/out.test.toml b/acceptance/bundle/lifecycle/started-validation/out.test.toml index 7bd72308cef..65479977129 100644 --- a/acceptance/bundle/lifecycle/started-validation/out.test.toml +++ b/acceptance/bundle/lifecycle/started-validation/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/lifecycle/started/out.test.toml b/acceptance/bundle/lifecycle/started/out.test.toml index 51a5602947c..2c6699da193 100644 --- a/acceptance/bundle/lifecycle/started/out.test.toml +++ b/acceptance/bundle/lifecycle/started/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/local_state_staleness/out.test.toml b/acceptance/bundle/local_state_staleness/out.test.toml index 0fc51ee009b..264de50aa96 100644 --- a/acceptance/bundle/local_state_staleness/out.test.toml +++ b/acceptance/bundle/local_state_staleness/out.test.toml @@ -1,3 +1,3 @@ Cloud = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/migrate/added/out.test.toml b/acceptance/bundle/migrate/added/out.test.toml index 5ab60815a3b..7daaf6fd56a 100644 --- a/acceptance/bundle/migrate/added/out.test.toml +++ b/acceptance/bundle/migrate/added/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/migrate/auto-migrate-clean/out.test.toml b/acceptance/bundle/migrate/auto-migrate-clean/out.test.toml index 7508f6b670e..37356d54551 100644 --- a/acceptance/bundle/migrate/auto-migrate-clean/out.test.toml +++ b/acceptance/bundle/migrate/auto-migrate-clean/out.test.toml @@ -1,4 +1,4 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/migrate/auto-migrate-empty-tfstate/out.test.toml b/acceptance/bundle/migrate/auto-migrate-empty-tfstate/out.test.toml index 5ab60815a3b..7daaf6fd56a 100644 --- a/acceptance/bundle/migrate/auto-migrate-empty-tfstate/out.test.toml +++ b/acceptance/bundle/migrate/auto-migrate-empty-tfstate/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/migrate/auto-migrate-envvar/out.test.toml b/acceptance/bundle/migrate/auto-migrate-envvar/out.test.toml index 5ab60815a3b..7daaf6fd56a 100644 --- a/acceptance/bundle/migrate/auto-migrate-envvar/out.test.toml +++ b/acceptance/bundle/migrate/auto-migrate-envvar/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/migrate/auto-migrate-push-failure/out.test.toml b/acceptance/bundle/migrate/auto-migrate-push-failure/out.test.toml index 5ab60815a3b..7daaf6fd56a 100644 --- a/acceptance/bundle/migrate/auto-migrate-push-failure/out.test.toml +++ b/acceptance/bundle/migrate/auto-migrate-push-failure/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/migrate/auto-migrate-tfbackup-failure/out.test.toml b/acceptance/bundle/migrate/auto-migrate-tfbackup-failure/out.test.toml index 5ab60815a3b..7daaf6fd56a 100644 --- a/acceptance/bundle/migrate/auto-migrate-tfbackup-failure/out.test.toml +++ b/acceptance/bundle/migrate/auto-migrate-tfbackup-failure/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/migrate/basic/out.test.toml b/acceptance/bundle/migrate/basic/out.test.toml index 5ab60815a3b..7daaf6fd56a 100644 --- a/acceptance/bundle/migrate/basic/out.test.toml +++ b/acceptance/bundle/migrate/basic/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/migrate/dashboards/out.test.toml b/acceptance/bundle/migrate/dashboards/out.test.toml index 5ab60815a3b..7daaf6fd56a 100644 --- a/acceptance/bundle/migrate/dashboards/out.test.toml +++ b/acceptance/bundle/migrate/dashboards/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/migrate/default-python/out.test.toml b/acceptance/bundle/migrate/default-python/out.test.toml index 5ab60815a3b..7daaf6fd56a 100644 --- a/acceptance/bundle/migrate/default-python/out.test.toml +++ b/acceptance/bundle/migrate/default-python/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/migrate/engine-config-direct/out.test.toml b/acceptance/bundle/migrate/engine-config-direct/out.test.toml index 5ab60815a3b..7daaf6fd56a 100644 --- a/acceptance/bundle/migrate/engine-config-direct/out.test.toml +++ b/acceptance/bundle/migrate/engine-config-direct/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/migrate/engine-config-terraform/out.test.toml b/acceptance/bundle/migrate/engine-config-terraform/out.test.toml index 5ab60815a3b..7daaf6fd56a 100644 --- a/acceptance/bundle/migrate/engine-config-terraform/out.test.toml +++ b/acceptance/bundle/migrate/engine-config-terraform/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/migrate/grants/out.test.toml b/acceptance/bundle/migrate/grants/out.test.toml index 5ab60815a3b..7daaf6fd56a 100644 --- a/acceptance/bundle/migrate/grants/out.test.toml +++ b/acceptance/bundle/migrate/grants/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/migrate/permissions/out.test.toml b/acceptance/bundle/migrate/permissions/out.test.toml index 5ab60815a3b..7daaf6fd56a 100644 --- a/acceptance/bundle/migrate/permissions/out.test.toml +++ b/acceptance/bundle/migrate/permissions/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/migrate/profile_arg/out.test.toml b/acceptance/bundle/migrate/profile_arg/out.test.toml index 5ab60815a3b..7daaf6fd56a 100644 --- a/acceptance/bundle/migrate/profile_arg/out.test.toml +++ b/acceptance/bundle/migrate/profile_arg/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/migrate/removed/out.test.toml b/acceptance/bundle/migrate/removed/out.test.toml index 5ab60815a3b..7daaf6fd56a 100644 --- a/acceptance/bundle/migrate/removed/out.test.toml +++ b/acceptance/bundle/migrate/removed/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/migrate/runas/out.test.toml b/acceptance/bundle/migrate/runas/out.test.toml index 5ab60815a3b..7daaf6fd56a 100644 --- a/acceptance/bundle/migrate/runas/out.test.toml +++ b/acceptance/bundle/migrate/runas/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/migrate/test.toml b/acceptance/bundle/migrate/test.toml index 964175b8b01..375f5445d1a 100644 --- a/acceptance/bundle/migrate/test.toml +++ b/acceptance/bundle/migrate/test.toml @@ -12,4 +12,4 @@ EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] # direct-engine feature, so the resources the terraform half creates are recorded nowhere # and the migration reads state the service does not have. Migrating a deployment onto the # service is a story of its own; these tests are not it. -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/migrate/var_arg/out.test.toml b/acceptance/bundle/migrate/var_arg/out.test.toml index 5ab60815a3b..7daaf6fd56a 100644 --- a/acceptance/bundle/migrate/var_arg/out.test.toml +++ b/acceptance/bundle/migrate/var_arg/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/multi_profile/auto_select/out.test.toml b/acceptance/bundle/multi_profile/auto_select/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/multi_profile/auto_select/out.test.toml +++ b/acceptance/bundle/multi_profile/auto_select/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/multi_profile/env_auth_skip/out.test.toml b/acceptance/bundle/multi_profile/env_auth_skip/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/multi_profile/env_auth_skip/out.test.toml +++ b/acceptance/bundle/multi_profile/env_auth_skip/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/multi_profile/no_workspace_profiles/out.test.toml b/acceptance/bundle/multi_profile/no_workspace_profiles/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/multi_profile/no_workspace_profiles/out.test.toml +++ b/acceptance/bundle/multi_profile/no_workspace_profiles/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/multi_profile/non_interactive_error/out.test.toml b/acceptance/bundle/multi_profile/non_interactive_error/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/multi_profile/non_interactive_error/out.test.toml +++ b/acceptance/bundle/multi_profile/non_interactive_error/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/open/out.test.toml b/acceptance/bundle/open/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/open/out.test.toml +++ b/acceptance/bundle/open/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/override/clusters/out.test.toml b/acceptance/bundle/override/clusters/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/override/clusters/out.test.toml +++ b/acceptance/bundle/override/clusters/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/override/job_cluster/out.test.toml b/acceptance/bundle/override/job_cluster/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/override/job_cluster/out.test.toml +++ b/acceptance/bundle/override/job_cluster/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/override/job_cluster_var/out.test.toml b/acceptance/bundle/override/job_cluster_var/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/override/job_cluster_var/out.test.toml +++ b/acceptance/bundle/override/job_cluster_var/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/override/job_tasks/out.test.toml b/acceptance/bundle/override/job_tasks/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/override/job_tasks/out.test.toml +++ b/acceptance/bundle/override/job_tasks/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/override/merge-string-map/out.test.toml b/acceptance/bundle/override/merge-string-map/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/override/merge-string-map/out.test.toml +++ b/acceptance/bundle/override/merge-string-map/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/override/pipeline_cluster/out.test.toml b/acceptance/bundle/override/pipeline_cluster/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/override/pipeline_cluster/out.test.toml +++ b/acceptance/bundle/override/pipeline_cluster/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/paths/designer_notebook/out.test.toml b/acceptance/bundle/paths/designer_notebook/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/paths/designer_notebook/out.test.toml +++ b/acceptance/bundle/paths/designer_notebook/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/paths/fallback/out.test.toml b/acceptance/bundle/paths/fallback/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/paths/fallback/out.test.toml +++ b/acceptance/bundle/paths/fallback/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/paths/git_source_jobs/out.test.toml b/acceptance/bundle/paths/git_source_jobs/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/paths/git_source_jobs/out.test.toml +++ b/acceptance/bundle/paths/git_source_jobs/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/paths/invalid_pipeline_globs/out.test.toml b/acceptance/bundle/paths/invalid_pipeline_globs/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/paths/invalid_pipeline_globs/out.test.toml +++ b/acceptance/bundle/paths/invalid_pipeline_globs/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/paths/nominal/out.test.toml b/acceptance/bundle/paths/nominal/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/paths/nominal/out.test.toml +++ b/acceptance/bundle/paths/nominal/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/paths/outside_root_no_sync/out.test.toml b/acceptance/bundle/paths/outside_root_no_sync/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/paths/outside_root_no_sync/out.test.toml +++ b/acceptance/bundle/paths/outside_root_no_sync/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/paths/pipeline_expected_file_got_notebook/out.test.toml b/acceptance/bundle/paths/pipeline_expected_file_got_notebook/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/paths/pipeline_expected_file_got_notebook/out.test.toml +++ b/acceptance/bundle/paths/pipeline_expected_file_got_notebook/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/paths/pipeline_globs/out.test.toml b/acceptance/bundle/paths/pipeline_globs/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/paths/pipeline_globs/out.test.toml +++ b/acceptance/bundle/paths/pipeline_globs/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/paths/pipeline_root_path_doesnotexist/out.test.toml b/acceptance/bundle/paths/pipeline_root_path_doesnotexist/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/paths/pipeline_root_path_doesnotexist/out.test.toml +++ b/acceptance/bundle/paths/pipeline_root_path_doesnotexist/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/paths/pipelines_glob_include_and_root_path/out.test.toml b/acceptance/bundle/paths/pipelines_glob_include_and_root_path/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/paths/pipelines_glob_include_and_root_path/out.test.toml +++ b/acceptance/bundle/paths/pipelines_glob_include_and_root_path/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/paths/pipelines_root_path_outside_sync_root/out.test.toml b/acceptance/bundle/paths/pipelines_root_path_outside_sync_root/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/paths/pipelines_root_path_outside_sync_root/out.test.toml +++ b/acceptance/bundle/paths/pipelines_root_path_outside_sync_root/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/paths/relative_path_outside_root/out.test.toml b/acceptance/bundle/paths/relative_path_outside_root/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/paths/relative_path_outside_root/out.test.toml +++ b/acceptance/bundle/paths/relative_path_outside_root/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/paths/relative_path_translation/out.test.toml b/acceptance/bundle/paths/relative_path_translation/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/paths/relative_path_translation/out.test.toml +++ b/acceptance/bundle/paths/relative_path_translation/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/plan/no_upload/out.test.toml b/acceptance/bundle/plan/no_upload/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/plan/no_upload/out.test.toml +++ b/acceptance/bundle/plan/no_upload/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/presets/preset_vs_dev_mode/out.test.toml b/acceptance/bundle/presets/preset_vs_dev_mode/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/presets/preset_vs_dev_mode/out.test.toml +++ b/acceptance/bundle/presets/preset_vs_dev_mode/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/python/experimental-compatibility-both-equal/out.test.toml b/acceptance/bundle/python/experimental-compatibility-both-equal/out.test.toml index cb31f1360de..fbae8dd6138 100644 --- a/acceptance/bundle/python/experimental-compatibility-both-equal/out.test.toml +++ b/acceptance/bundle/python/experimental-compatibility-both-equal/out.test.toml @@ -1,4 +1,4 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] EnvMatrix.PYDAB_VERSION = ["0.266.0", "current"] diff --git a/acceptance/bundle/python/experimental-compatibility-both-error/out.test.toml b/acceptance/bundle/python/experimental-compatibility-both-error/out.test.toml index cb31f1360de..fbae8dd6138 100644 --- a/acceptance/bundle/python/experimental-compatibility-both-error/out.test.toml +++ b/acceptance/bundle/python/experimental-compatibility-both-error/out.test.toml @@ -1,4 +1,4 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] EnvMatrix.PYDAB_VERSION = ["0.266.0", "current"] diff --git a/acceptance/bundle/python/experimental-compatibility/out.test.toml b/acceptance/bundle/python/experimental-compatibility/out.test.toml index cb31f1360de..fbae8dd6138 100644 --- a/acceptance/bundle/python/experimental-compatibility/out.test.toml +++ b/acceptance/bundle/python/experimental-compatibility/out.test.toml @@ -1,4 +1,4 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] EnvMatrix.PYDAB_VERSION = ["0.266.0", "current"] diff --git a/acceptance/bundle/python/grants-aliases/out.test.toml b/acceptance/bundle/python/grants-aliases/out.test.toml index c5fdcb720f2..20eb81d55ff 100644 --- a/acceptance/bundle/python/grants-aliases/out.test.toml +++ b/acceptance/bundle/python/grants-aliases/out.test.toml @@ -1,4 +1,4 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] EnvMatrix.PYDAB_VERSION = ["current"] diff --git a/acceptance/bundle/python/mutator-ordering/out.test.toml b/acceptance/bundle/python/mutator-ordering/out.test.toml index cb31f1360de..fbae8dd6138 100644 --- a/acceptance/bundle/python/mutator-ordering/out.test.toml +++ b/acceptance/bundle/python/mutator-ordering/out.test.toml @@ -1,4 +1,4 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] EnvMatrix.PYDAB_VERSION = ["0.266.0", "current"] diff --git a/acceptance/bundle/python/mutator-permissions-owner-5682/out.test.toml b/acceptance/bundle/python/mutator-permissions-owner-5682/out.test.toml index 4f3fa2d5201..782880010e5 100644 --- a/acceptance/bundle/python/mutator-permissions-owner-5682/out.test.toml +++ b/acceptance/bundle/python/mutator-permissions-owner-5682/out.test.toml @@ -1,5 +1,5 @@ Cloud = true RequiresUnityCatalog = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] EnvMatrix.PYDAB_VERSION = ["0.266.0", "current"] diff --git a/acceptance/bundle/python/pipelines-support/out.test.toml b/acceptance/bundle/python/pipelines-support/out.test.toml index cb31f1360de..fbae8dd6138 100644 --- a/acceptance/bundle/python/pipelines-support/out.test.toml +++ b/acceptance/bundle/python/pipelines-support/out.test.toml @@ -1,4 +1,4 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] EnvMatrix.PYDAB_VERSION = ["0.266.0", "current"] diff --git a/acceptance/bundle/python/propagates-auth-env/out.test.toml b/acceptance/bundle/python/propagates-auth-env/out.test.toml index cb31f1360de..fbae8dd6138 100644 --- a/acceptance/bundle/python/propagates-auth-env/out.test.toml +++ b/acceptance/bundle/python/propagates-auth-env/out.test.toml @@ -1,4 +1,4 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] EnvMatrix.PYDAB_VERSION = ["0.266.0", "current"] diff --git a/acceptance/bundle/python/resolve-variable/out.test.toml b/acceptance/bundle/python/resolve-variable/out.test.toml index cb31f1360de..fbae8dd6138 100644 --- a/acceptance/bundle/python/resolve-variable/out.test.toml +++ b/acceptance/bundle/python/resolve-variable/out.test.toml @@ -1,4 +1,4 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] EnvMatrix.PYDAB_VERSION = ["0.266.0", "current"] diff --git a/acceptance/bundle/python/resource-loading/out.test.toml b/acceptance/bundle/python/resource-loading/out.test.toml index cb31f1360de..fbae8dd6138 100644 --- a/acceptance/bundle/python/resource-loading/out.test.toml +++ b/acceptance/bundle/python/resource-loading/out.test.toml @@ -1,4 +1,4 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] EnvMatrix.PYDAB_VERSION = ["0.266.0", "current"] diff --git a/acceptance/bundle/python/restricted-execution/out.test.toml b/acceptance/bundle/python/restricted-execution/out.test.toml index cb31f1360de..fbae8dd6138 100644 --- a/acceptance/bundle/python/restricted-execution/out.test.toml +++ b/acceptance/bundle/python/restricted-execution/out.test.toml @@ -1,4 +1,4 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] EnvMatrix.PYDAB_VERSION = ["0.266.0", "current"] diff --git a/acceptance/bundle/python/schemas-support/out.test.toml b/acceptance/bundle/python/schemas-support/out.test.toml index c5fdcb720f2..20eb81d55ff 100644 --- a/acceptance/bundle/python/schemas-support/out.test.toml +++ b/acceptance/bundle/python/schemas-support/out.test.toml @@ -1,4 +1,4 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] EnvMatrix.PYDAB_VERSION = ["current"] diff --git a/acceptance/bundle/python/unicode-support/out.test.toml b/acceptance/bundle/python/unicode-support/out.test.toml index cb31f1360de..fbae8dd6138 100644 --- a/acceptance/bundle/python/unicode-support/out.test.toml +++ b/acceptance/bundle/python/unicode-support/out.test.toml @@ -1,4 +1,4 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] EnvMatrix.PYDAB_VERSION = ["0.266.0", "current"] diff --git a/acceptance/bundle/python/volumes-support/out.test.toml b/acceptance/bundle/python/volumes-support/out.test.toml index cb31f1360de..fbae8dd6138 100644 --- a/acceptance/bundle/python/volumes-support/out.test.toml +++ b/acceptance/bundle/python/volumes-support/out.test.toml @@ -1,4 +1,4 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] EnvMatrix.PYDAB_VERSION = ["0.266.0", "current"] diff --git a/acceptance/bundle/quality_monitor/out.test.toml b/acceptance/bundle/quality_monitor/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/quality_monitor/out.test.toml +++ b/acceptance/bundle/quality_monitor/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/refschema/out.test.toml b/acceptance/bundle/refschema/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/refschema/out.test.toml +++ b/acceptance/bundle/refschema/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resource_deps/bad_ref_string_to_int/out.test.toml b/acceptance/bundle/resource_deps/bad_ref_string_to_int/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/resource_deps/bad_ref_string_to_int/out.test.toml +++ b/acceptance/bundle/resource_deps/bad_ref_string_to_int/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resource_deps/bad_syntax/out.test.toml b/acceptance/bundle/resource_deps/bad_syntax/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/resource_deps/bad_syntax/out.test.toml +++ b/acceptance/bundle/resource_deps/bad_syntax/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resource_deps/computed_volume_path/out.test.toml b/acceptance/bundle/resource_deps/computed_volume_path/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/resource_deps/computed_volume_path/out.test.toml +++ b/acceptance/bundle/resource_deps/computed_volume_path/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resource_deps/create_error/out.test.toml b/acceptance/bundle/resource_deps/create_error/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/resource_deps/create_error/out.test.toml +++ b/acceptance/bundle/resource_deps/create_error/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resource_deps/duplicate_ref/out.test.toml b/acceptance/bundle/resource_deps/duplicate_ref/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/resource_deps/duplicate_ref/out.test.toml +++ b/acceptance/bundle/resource_deps/duplicate_ref/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resource_deps/grant_ref/out.test.toml b/acceptance/bundle/resource_deps/grant_ref/out.test.toml index 51a5602947c..2c6699da193 100644 --- a/acceptance/bundle/resource_deps/grant_ref/out.test.toml +++ b/acceptance/bundle/resource_deps/grant_ref/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resource_deps/id_chain/out.test.toml b/acceptance/bundle/resource_deps/id_chain/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/resource_deps/id_chain/out.test.toml +++ b/acceptance/bundle/resource_deps/id_chain/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resource_deps/id_star/out.test.toml b/acceptance/bundle/resource_deps/id_star/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/resource_deps/id_star/out.test.toml +++ b/acceptance/bundle/resource_deps/id_star/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resource_deps/immutable_field_ref/out.test.toml b/acceptance/bundle/resource_deps/immutable_field_ref/out.test.toml index 51a5602947c..2c6699da193 100644 --- a/acceptance/bundle/resource_deps/immutable_field_ref/out.test.toml +++ b/acceptance/bundle/resource_deps/immutable_field_ref/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resource_deps/implicit_deps_model_serving_endpoint/out.test.toml b/acceptance/bundle/resource_deps/implicit_deps_model_serving_endpoint/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/resource_deps/implicit_deps_model_serving_endpoint/out.test.toml +++ b/acceptance/bundle/resource_deps/implicit_deps_model_serving_endpoint/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resource_deps/implicit_deps_quality_monitor/out.test.toml b/acceptance/bundle/resource_deps/implicit_deps_quality_monitor/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/resource_deps/implicit_deps_quality_monitor/out.test.toml +++ b/acceptance/bundle/resource_deps/implicit_deps_quality_monitor/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resource_deps/implicit_deps_registered_model/out.test.toml b/acceptance/bundle/resource_deps/implicit_deps_registered_model/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/resource_deps/implicit_deps_registered_model/out.test.toml +++ b/acceptance/bundle/resource_deps/implicit_deps_registered_model/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resource_deps/implicit_deps_volume/out.test.toml b/acceptance/bundle/resource_deps/implicit_deps_volume/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/resource_deps/implicit_deps_volume/out.test.toml +++ b/acceptance/bundle/resource_deps/implicit_deps_volume/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resource_deps/job_id/out.test.toml b/acceptance/bundle/resource_deps/job_id/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/resource_deps/job_id/out.test.toml +++ b/acceptance/bundle/resource_deps/job_id/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resource_deps/job_id_big_graph/delete_all/out.test.toml b/acceptance/bundle/resource_deps/job_id_big_graph/delete_all/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/resource_deps/job_id_big_graph/delete_all/out.test.toml +++ b/acceptance/bundle/resource_deps/job_id_big_graph/delete_all/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resource_deps/job_id_big_graph/destroy/out.test.toml b/acceptance/bundle/resource_deps/job_id_big_graph/destroy/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/resource_deps/job_id_big_graph/destroy/out.test.toml +++ b/acceptance/bundle/resource_deps/job_id_big_graph/destroy/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resource_deps/job_id_delete_bar/out.test.toml b/acceptance/bundle/resource_deps/job_id_delete_bar/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/resource_deps/job_id_delete_bar/out.test.toml +++ b/acceptance/bundle/resource_deps/job_id_delete_bar/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resource_deps/job_id_delete_foo/out.test.toml b/acceptance/bundle/resource_deps/job_id_delete_foo/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/resource_deps/job_id_delete_foo/out.test.toml +++ b/acceptance/bundle/resource_deps/job_id_delete_foo/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resource_deps/job_tasks/out.test.toml b/acceptance/bundle/resource_deps/job_tasks/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/resource_deps/job_tasks/out.test.toml +++ b/acceptance/bundle/resource_deps/job_tasks/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resource_deps/jobs_update/out.test.toml b/acceptance/bundle/resource_deps/jobs_update/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/resource_deps/jobs_update/out.test.toml +++ b/acceptance/bundle/resource_deps/jobs_update/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resource_deps/jobs_update_remote/out.test.toml b/acceptance/bundle/resource_deps/jobs_update_remote/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/resource_deps/jobs_update_remote/out.test.toml +++ b/acceptance/bundle/resource_deps/jobs_update_remote/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resource_deps/loop_jobs/out.test.toml b/acceptance/bundle/resource_deps/loop_jobs/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/resource_deps/loop_jobs/out.test.toml +++ b/acceptance/bundle/resource_deps/loop_jobs/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resource_deps/loop_self/out.test.toml b/acceptance/bundle/resource_deps/loop_self/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/resource_deps/loop_self/out.test.toml +++ b/acceptance/bundle/resource_deps/loop_self/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resource_deps/missing_ingestion_definition/out.test.toml b/acceptance/bundle/resource_deps/missing_ingestion_definition/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/resource_deps/missing_ingestion_definition/out.test.toml +++ b/acceptance/bundle/resource_deps/missing_ingestion_definition/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resource_deps/missing_map_key/out.test.toml b/acceptance/bundle/resource_deps/missing_map_key/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/resource_deps/missing_map_key/out.test.toml +++ b/acceptance/bundle/resource_deps/missing_map_key/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resource_deps/missing_string_field/out.test.toml b/acceptance/bundle/resource_deps/missing_string_field/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/resource_deps/missing_string_field/out.test.toml +++ b/acceptance/bundle/resource_deps/missing_string_field/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resource_deps/model_id_ref/out.test.toml b/acceptance/bundle/resource_deps/model_id_ref/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/resource_deps/model_id_ref/out.test.toml +++ b/acceptance/bundle/resource_deps/model_id_ref/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resource_deps/non_existent_field/out.test.toml b/acceptance/bundle/resource_deps/non_existent_field/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/resource_deps/non_existent_field/out.test.toml +++ b/acceptance/bundle/resource_deps/non_existent_field/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resource_deps/permission_ref/out.test.toml b/acceptance/bundle/resource_deps/permission_ref/out.test.toml index 51a5602947c..2c6699da193 100644 --- a/acceptance/bundle/resource_deps/permission_ref/out.test.toml +++ b/acceptance/bundle/resource_deps/permission_ref/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resource_deps/pipelines_recreate/out.test.toml b/acceptance/bundle/resource_deps/pipelines_recreate/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/resource_deps/pipelines_recreate/out.test.toml +++ b/acceptance/bundle/resource_deps/pipelines_recreate/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resource_deps/present_ingestion_definition/out.test.toml b/acceptance/bundle/resource_deps/present_ingestion_definition/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/resource_deps/present_ingestion_definition/out.test.toml +++ b/acceptance/bundle/resource_deps/present_ingestion_definition/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resource_deps/remote_app_url/out.test.toml b/acceptance/bundle/resource_deps/remote_app_url/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/resource_deps/remote_app_url/out.test.toml +++ b/acceptance/bundle/resource_deps/remote_app_url/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resource_deps/remote_field_storage_location/out.test.toml b/acceptance/bundle/resource_deps/remote_field_storage_location/out.test.toml index cc2fc7c517f..a3b8091ddab 100644 --- a/acceptance/bundle/resource_deps/remote_field_storage_location/out.test.toml +++ b/acceptance/bundle/resource_deps/remote_field_storage_location/out.test.toml @@ -2,5 +2,5 @@ Cloud = true RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resource_deps/remote_pipeline/out.test.toml b/acceptance/bundle/resource_deps/remote_pipeline/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/resource_deps/remote_pipeline/out.test.toml +++ b/acceptance/bundle/resource_deps/remote_pipeline/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resource_deps/resources_var/out.test.toml b/acceptance/bundle/resource_deps/resources_var/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/resource_deps/resources_var/out.test.toml +++ b/acceptance/bundle/resource_deps/resources_var/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resource_deps/resources_var_presets/out.test.toml b/acceptance/bundle/resource_deps/resources_var_presets/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/resource_deps/resources_var_presets/out.test.toml +++ b/acceptance/bundle/resource_deps/resources_var_presets/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resource_deps/resources_var_presets_implicit_deps/out.test.toml b/acceptance/bundle/resource_deps/resources_var_presets_implicit_deps/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/resource_deps/resources_var_presets_implicit_deps/out.test.toml +++ b/acceptance/bundle/resource_deps/resources_var_presets_implicit_deps/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resource_deps/tf_path_only_error/out.test.toml b/acceptance/bundle/resource_deps/tf_path_only_error/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/resource_deps/tf_path_only_error/out.test.toml +++ b/acceptance/bundle/resource_deps/tf_path_only_error/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resource_deps/tf_path_renames/out.test.toml b/acceptance/bundle/resource_deps/tf_path_renames/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/resource_deps/tf_path_renames/out.test.toml +++ b/acceptance/bundle/resource_deps/tf_path_renames/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resource_deps/unicode_reference/out.test.toml b/acceptance/bundle/resource_deps/unicode_reference/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/resource_deps/unicode_reference/out.test.toml +++ b/acceptance/bundle/resource_deps/unicode_reference/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resource_deps/volume_path_contains_id/out.test.toml b/acceptance/bundle/resource_deps/volume_path_contains_id/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/resource_deps/volume_path_contains_id/out.test.toml +++ b/acceptance/bundle/resource_deps/volume_path_contains_id/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resource_deps/volume_path_job_ref/out.test.toml b/acceptance/bundle/resource_deps/volume_path_job_ref/out.test.toml index 51a5602947c..2c6699da193 100644 --- a/acceptance/bundle/resource_deps/volume_path_job_ref/out.test.toml +++ b/acceptance/bundle/resource_deps/volume_path_job_ref/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/alerts/basic/out.test.toml b/acceptance/bundle/resources/alerts/basic/out.test.toml index 56a8bf0b677..d8a18a01a89 100644 --- a/acceptance/bundle/resources/alerts/basic/out.test.toml +++ b/acceptance/bundle/resources/alerts/basic/out.test.toml @@ -1,4 +1,4 @@ Cloud = true RunsOnDbr = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/alerts/with_file/out.test.toml b/acceptance/bundle/resources/alerts/with_file/out.test.toml index 0fc51ee009b..264de50aa96 100644 --- a/acceptance/bundle/resources/alerts/with_file/out.test.toml +++ b/acceptance/bundle/resources/alerts/with_file/out.test.toml @@ -1,3 +1,3 @@ Cloud = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/alerts/with_file_not_allowed_field_error/out.test.toml b/acceptance/bundle/resources/alerts/with_file_not_allowed_field_error/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/resources/alerts/with_file_not_allowed_field_error/out.test.toml +++ b/acceptance/bundle/resources/alerts/with_file_not_allowed_field_error/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/alerts/with_file_run_from_subdir/out.test.toml b/acceptance/bundle/resources/alerts/with_file_run_from_subdir/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/resources/alerts/with_file_run_from_subdir/out.test.toml +++ b/acceptance/bundle/resources/alerts/with_file_run_from_subdir/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/alerts/with_file_variable_interpolation_error/out.test.toml b/acceptance/bundle/resources/alerts/with_file_variable_interpolation_error/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/resources/alerts/with_file_variable_interpolation_error/out.test.toml +++ b/acceptance/bundle/resources/alerts/with_file_variable_interpolation_error/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/apps/config-drift-stopped/out.test.toml b/acceptance/bundle/resources/apps/config-drift-stopped/out.test.toml index 51a5602947c..2c6699da193 100644 --- a/acceptance/bundle/resources/apps/config-drift-stopped/out.test.toml +++ b/acceptance/bundle/resources/apps/config-drift-stopped/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/apps/config-drift/out.test.toml b/acceptance/bundle/resources/apps/config-drift/out.test.toml index 51a5602947c..2c6699da193 100644 --- a/acceptance/bundle/resources/apps/config-drift/out.test.toml +++ b/acceptance/bundle/resources/apps/config-drift/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/apps/config-no-deployment/out.test.toml b/acceptance/bundle/resources/apps/config-no-deployment/out.test.toml index 51a5602947c..2c6699da193 100644 --- a/acceptance/bundle/resources/apps/config-no-deployment/out.test.toml +++ b/acceptance/bundle/resources/apps/config-no-deployment/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/apps/create_already_exists/out.test.toml b/acceptance/bundle/resources/apps/create_already_exists/out.test.toml index 51a5602947c..2c6699da193 100644 --- a/acceptance/bundle/resources/apps/create_already_exists/out.test.toml +++ b/acceptance/bundle/resources/apps/create_already_exists/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/apps/default_description/out.test.toml b/acceptance/bundle/resources/apps/default_description/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/resources/apps/default_description/out.test.toml +++ b/acceptance/bundle/resources/apps/default_description/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/apps/git-source-no-deployment/out.test.toml b/acceptance/bundle/resources/apps/git-source-no-deployment/out.test.toml index 51a5602947c..2c6699da193 100644 --- a/acceptance/bundle/resources/apps/git-source-no-deployment/out.test.toml +++ b/acceptance/bundle/resources/apps/git-source-no-deployment/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/apps/immutable/out.test.toml b/acceptance/bundle/resources/apps/immutable/out.test.toml index 51a5602947c..2c6699da193 100644 --- a/acceptance/bundle/resources/apps/immutable/out.test.toml +++ b/acceptance/bundle/resources/apps/immutable/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/apps/inline_config/out.test.toml b/acceptance/bundle/resources/apps/inline_config/out.test.toml index 0fc51ee009b..264de50aa96 100644 --- a/acceptance/bundle/resources/apps/inline_config/out.test.toml +++ b/acceptance/bundle/resources/apps/inline_config/out.test.toml @@ -1,3 +1,3 @@ Cloud = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/apps/lifecycle-started-omitted/out.test.toml b/acceptance/bundle/resources/apps/lifecycle-started-omitted/out.test.toml index a40e88fc275..89d861bf4d0 100644 --- a/acceptance/bundle/resources/apps/lifecycle-started-omitted/out.test.toml +++ b/acceptance/bundle/resources/apps/lifecycle-started-omitted/out.test.toml @@ -1,3 +1,3 @@ Cloud = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/apps/lifecycle-started-terraform-error/out.test.toml b/acceptance/bundle/resources/apps/lifecycle-started-terraform-error/out.test.toml index 944c993b0fa..c7045d40dc0 100644 --- a/acceptance/bundle/resources/apps/lifecycle-started-terraform-error/out.test.toml +++ b/acceptance/bundle/resources/apps/lifecycle-started-terraform-error/out.test.toml @@ -1,3 +1,3 @@ Cloud = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/apps/lifecycle-started-toggle/out.test.toml b/acceptance/bundle/resources/apps/lifecycle-started-toggle/out.test.toml index a40e88fc275..89d861bf4d0 100644 --- a/acceptance/bundle/resources/apps/lifecycle-started-toggle/out.test.toml +++ b/acceptance/bundle/resources/apps/lifecycle-started-toggle/out.test.toml @@ -1,3 +1,3 @@ Cloud = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/apps/lifecycle-started/out.test.toml b/acceptance/bundle/resources/apps/lifecycle-started/out.test.toml index a40e88fc275..89d861bf4d0 100644 --- a/acceptance/bundle/resources/apps/lifecycle-started/out.test.toml +++ b/acceptance/bundle/resources/apps/lifecycle-started/out.test.toml @@ -1,3 +1,3 @@ Cloud = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/apps/readplan-lifecycle/out.test.toml b/acceptance/bundle/resources/apps/readplan-lifecycle/out.test.toml index ecdf1da5791..71b97f1370d 100644 --- a/acceptance/bundle/resources/apps/readplan-lifecycle/out.test.toml +++ b/acceptance/bundle/resources/apps/readplan-lifecycle/out.test.toml @@ -1,4 +1,4 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/resources/apps/resource-refs/out.test.toml b/acceptance/bundle/resources/apps/resource-refs/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/resources/apps/resource-refs/out.test.toml +++ b/acceptance/bundle/resources/apps/resource-refs/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/apps/update/out.test.toml b/acceptance/bundle/resources/apps/update/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/resources/apps/update/out.test.toml +++ b/acceptance/bundle/resources/apps/update/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/catalogs/auto-approve/out.test.toml b/acceptance/bundle/resources/catalogs/auto-approve/out.test.toml index 8197bd50f28..6c47aa9e8d1 100644 --- a/acceptance/bundle/resources/catalogs/auto-approve/out.test.toml +++ b/acceptance/bundle/resources/catalogs/auto-approve/out.test.toml @@ -1,4 +1,4 @@ Cloud = true RequiresUnityCatalog = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/catalogs/basic/out.test.toml b/acceptance/bundle/resources/catalogs/basic/out.test.toml index 8197bd50f28..6c47aa9e8d1 100644 --- a/acceptance/bundle/resources/catalogs/basic/out.test.toml +++ b/acceptance/bundle/resources/catalogs/basic/out.test.toml @@ -1,4 +1,4 @@ Cloud = true RequiresUnityCatalog = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/catalogs/drift/managed_properties/out.test.toml b/acceptance/bundle/resources/catalogs/drift/managed_properties/out.test.toml index 51a5602947c..2c6699da193 100644 --- a/acceptance/bundle/resources/catalogs/drift/managed_properties/out.test.toml +++ b/acceptance/bundle/resources/catalogs/drift/managed_properties/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/catalogs/empty-name/out.test.toml b/acceptance/bundle/resources/catalogs/empty-name/out.test.toml index 8197bd50f28..6c47aa9e8d1 100644 --- a/acceptance/bundle/resources/catalogs/empty-name/out.test.toml +++ b/acceptance/bundle/resources/catalogs/empty-name/out.test.toml @@ -1,4 +1,4 @@ Cloud = true RequiresUnityCatalog = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/catalogs/with-schemas/out.test.toml b/acceptance/bundle/resources/catalogs/with-schemas/out.test.toml index 8197bd50f28..6c47aa9e8d1 100644 --- a/acceptance/bundle/resources/catalogs/with-schemas/out.test.toml +++ b/acceptance/bundle/resources/catalogs/with-schemas/out.test.toml @@ -1,4 +1,4 @@ Cloud = true RequiresUnityCatalog = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/clusters/deploy/data_security_mode/out.test.toml b/acceptance/bundle/resources/clusters/deploy/data_security_mode/out.test.toml index f9a87c23500..681e90697f0 100644 --- a/acceptance/bundle/resources/clusters/deploy/data_security_mode/out.test.toml +++ b/acceptance/bundle/resources/clusters/deploy/data_security_mode/out.test.toml @@ -1,4 +1,4 @@ Cloud = true RunsOnDbr = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/clusters/deploy/instance_pool/out.test.toml b/acceptance/bundle/resources/clusters/deploy/instance_pool/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/resources/clusters/deploy/instance_pool/out.test.toml +++ b/acceptance/bundle/resources/clusters/deploy/instance_pool/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/clusters/deploy/instance_pool_and_node_type/out.test.toml b/acceptance/bundle/resources/clusters/deploy/instance_pool_and_node_type/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/resources/clusters/deploy/instance_pool_and_node_type/out.test.toml +++ b/acceptance/bundle/resources/clusters/deploy/instance_pool_and_node_type/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/clusters/deploy/local_ssd_count/out.test.toml b/acceptance/bundle/resources/clusters/deploy/local_ssd_count/out.test.toml index 762b8cc2ce9..9399b9ff114 100644 --- a/acceptance/bundle/resources/clusters/deploy/local_ssd_count/out.test.toml +++ b/acceptance/bundle/resources/clusters/deploy/local_ssd_count/out.test.toml @@ -2,5 +2,5 @@ Cloud = true CloudEnvs.aws = false CloudEnvs.azure = false CloudEnvs.gcp = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/clusters/deploy/num_workers_absent/out.test.toml b/acceptance/bundle/resources/clusters/deploy/num_workers_absent/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/resources/clusters/deploy/num_workers_absent/out.test.toml +++ b/acceptance/bundle/resources/clusters/deploy/num_workers_absent/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/clusters/deploy/simple/out.test.toml b/acceptance/bundle/resources/clusters/deploy/simple/out.test.toml index f9a87c23500..681e90697f0 100644 --- a/acceptance/bundle/resources/clusters/deploy/simple/out.test.toml +++ b/acceptance/bundle/resources/clusters/deploy/simple/out.test.toml @@ -1,4 +1,4 @@ Cloud = true RunsOnDbr = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/clusters/deploy/update-after-create/out.test.toml b/acceptance/bundle/resources/clusters/deploy/update-after-create/out.test.toml index 0fc51ee009b..264de50aa96 100644 --- a/acceptance/bundle/resources/clusters/deploy/update-after-create/out.test.toml +++ b/acceptance/bundle/resources/clusters/deploy/update-after-create/out.test.toml @@ -1,3 +1,3 @@ Cloud = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/out.test.toml b/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/out.test.toml +++ b/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/clusters/deploy/update-and-resize/out.test.toml b/acceptance/bundle/resources/clusters/deploy/update-and-resize/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/resources/clusters/deploy/update-and-resize/out.test.toml +++ b/acceptance/bundle/resources/clusters/deploy/update-and-resize/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/clusters/deploy/workload_type/out.test.toml b/acceptance/bundle/resources/clusters/deploy/workload_type/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/resources/clusters/deploy/workload_type/out.test.toml +++ b/acceptance/bundle/resources/clusters/deploy/workload_type/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/clusters/lifecycle-started-terraform-error/out.test.toml b/acceptance/bundle/resources/clusters/lifecycle-started-terraform-error/out.test.toml index 944c993b0fa..c7045d40dc0 100644 --- a/acceptance/bundle/resources/clusters/lifecycle-started-terraform-error/out.test.toml +++ b/acceptance/bundle/resources/clusters/lifecycle-started-terraform-error/out.test.toml @@ -1,3 +1,3 @@ Cloud = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/clusters/lifecycle-started-toggle/out.test.toml b/acceptance/bundle/resources/clusters/lifecycle-started-toggle/out.test.toml index a40e88fc275..89d861bf4d0 100644 --- a/acceptance/bundle/resources/clusters/lifecycle-started-toggle/out.test.toml +++ b/acceptance/bundle/resources/clusters/lifecycle-started-toggle/out.test.toml @@ -1,3 +1,3 @@ Cloud = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/clusters/lifecycle-started/out.test.toml b/acceptance/bundle/resources/clusters/lifecycle-started/out.test.toml index a40e88fc275..89d861bf4d0 100644 --- a/acceptance/bundle/resources/clusters/lifecycle-started/out.test.toml +++ b/acceptance/bundle/resources/clusters/lifecycle-started/out.test.toml @@ -1,3 +1,3 @@ Cloud = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/clusters/readplan-lifecycle/out.test.toml b/acceptance/bundle/resources/clusters/readplan-lifecycle/out.test.toml index ecdf1da5791..71b97f1370d 100644 --- a/acceptance/bundle/resources/clusters/readplan-lifecycle/out.test.toml +++ b/acceptance/bundle/resources/clusters/readplan-lifecycle/out.test.toml @@ -1,4 +1,4 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/resources/clusters/resize-terminated-fallback/out.test.toml b/acceptance/bundle/resources/clusters/resize-terminated-fallback/out.test.toml index a40e88fc275..89d861bf4d0 100644 --- a/acceptance/bundle/resources/clusters/resize-terminated-fallback/out.test.toml +++ b/acceptance/bundle/resources/clusters/resize-terminated-fallback/out.test.toml @@ -1,3 +1,3 @@ Cloud = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/clusters/run/spark_python_task/out.test.toml b/acceptance/bundle/resources/clusters/run/spark_python_task/out.test.toml index 47ea1e4aa9b..e9b1abe7d7c 100644 --- a/acceptance/bundle/resources/clusters/run/spark_python_task/out.test.toml +++ b/acceptance/bundle/resources/clusters/run/spark_python_task/out.test.toml @@ -1,5 +1,5 @@ Cloud = true CloudSlow = true RunsOnDbr = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/dashboards/change-embed-credentials/out.test.toml b/acceptance/bundle/resources/dashboards/change-embed-credentials/out.test.toml index 3a19b9e8c33..fb23fa08c1e 100644 --- a/acceptance/bundle/resources/dashboards/change-embed-credentials/out.test.toml +++ b/acceptance/bundle/resources/dashboards/change-embed-credentials/out.test.toml @@ -1,4 +1,4 @@ Cloud = true RequiresWarehouse = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/dashboards/change-name/out.test.toml b/acceptance/bundle/resources/dashboards/change-name/out.test.toml index 3a19b9e8c33..fb23fa08c1e 100644 --- a/acceptance/bundle/resources/dashboards/change-name/out.test.toml +++ b/acceptance/bundle/resources/dashboards/change-name/out.test.toml @@ -1,4 +1,4 @@ Cloud = true RequiresWarehouse = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/dashboards/change-parent-path/out.test.toml b/acceptance/bundle/resources/dashboards/change-parent-path/out.test.toml index 3a19b9e8c33..fb23fa08c1e 100644 --- a/acceptance/bundle/resources/dashboards/change-parent-path/out.test.toml +++ b/acceptance/bundle/resources/dashboards/change-parent-path/out.test.toml @@ -1,4 +1,4 @@ Cloud = true RequiresWarehouse = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/dashboards/change-serialized-dashboard/out.test.toml b/acceptance/bundle/resources/dashboards/change-serialized-dashboard/out.test.toml index 9f30bf4a2b2..7098ee15b5e 100644 --- a/acceptance/bundle/resources/dashboards/change-serialized-dashboard/out.test.toml +++ b/acceptance/bundle/resources/dashboards/change-serialized-dashboard/out.test.toml @@ -1,5 +1,5 @@ Cloud = true RequiresWarehouse = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/resources/dashboards/dataset-catalog-schema/out.test.toml b/acceptance/bundle/resources/dashboards/dataset-catalog-schema/out.test.toml index a83d82c12c5..d94a4f1d500 100644 --- a/acceptance/bundle/resources/dashboards/dataset-catalog-schema/out.test.toml +++ b/acceptance/bundle/resources/dashboards/dataset-catalog-schema/out.test.toml @@ -1,4 +1,4 @@ Cloud = true RequiresWarehouse = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/dashboards/delete-trashed-out-of-band/out.test.toml b/acceptance/bundle/resources/dashboards/delete-trashed-out-of-band/out.test.toml index c42373c3478..db7a7fc7904 100644 --- a/acceptance/bundle/resources/dashboards/delete-trashed-out-of-band/out.test.toml +++ b/acceptance/bundle/resources/dashboards/delete-trashed-out-of-band/out.test.toml @@ -1,5 +1,5 @@ Cloud = true RequiresWarehouse = true RunsOnDbr = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/dashboards/destroy/out.test.toml b/acceptance/bundle/resources/dashboards/destroy/out.test.toml index c42373c3478..db7a7fc7904 100644 --- a/acceptance/bundle/resources/dashboards/destroy/out.test.toml +++ b/acceptance/bundle/resources/dashboards/destroy/out.test.toml @@ -1,5 +1,5 @@ Cloud = true RequiresWarehouse = true RunsOnDbr = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/dashboards/detect-change/out.test.toml b/acceptance/bundle/resources/dashboards/detect-change/out.test.toml index 3a19b9e8c33..fb23fa08c1e 100644 --- a/acceptance/bundle/resources/dashboards/detect-change/out.test.toml +++ b/acceptance/bundle/resources/dashboards/detect-change/out.test.toml @@ -1,4 +1,4 @@ Cloud = true RequiresWarehouse = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/dashboards/generate_inplace/out.test.toml b/acceptance/bundle/resources/dashboards/generate_inplace/out.test.toml index c42373c3478..db7a7fc7904 100644 --- a/acceptance/bundle/resources/dashboards/generate_inplace/out.test.toml +++ b/acceptance/bundle/resources/dashboards/generate_inplace/out.test.toml @@ -1,5 +1,5 @@ Cloud = true RequiresWarehouse = true RunsOnDbr = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/dashboards/nested-folders/out.test.toml b/acceptance/bundle/resources/dashboards/nested-folders/out.test.toml index 31f0c073e7f..1e2baa8951e 100644 --- a/acceptance/bundle/resources/dashboards/nested-folders/out.test.toml +++ b/acceptance/bundle/resources/dashboards/nested-folders/out.test.toml @@ -1,6 +1,6 @@ Cloud = true RequiresWarehouse = true RunsOnDbr = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/resources/dashboards/publish-failure-cleans-up-dashboard/out.test.toml b/acceptance/bundle/resources/dashboards/publish-failure-cleans-up-dashboard/out.test.toml index 1c1f58253c8..7c9f593dda8 100644 --- a/acceptance/bundle/resources/dashboards/publish-failure-cleans-up-dashboard/out.test.toml +++ b/acceptance/bundle/resources/dashboards/publish-failure-cleans-up-dashboard/out.test.toml @@ -1,5 +1,5 @@ Cloud = false RequiresWarehouse = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/resources/dashboards/publish-failure-stale-content/out.test.toml b/acceptance/bundle/resources/dashboards/publish-failure-stale-content/out.test.toml index b3b039ee392..b423ee4c554 100644 --- a/acceptance/bundle/resources/dashboards/publish-failure-stale-content/out.test.toml +++ b/acceptance/bundle/resources/dashboards/publish-failure-stale-content/out.test.toml @@ -1,4 +1,4 @@ Cloud = false RequiresWarehouse = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/dashboards/republish-after-draft-update/out.test.toml b/acceptance/bundle/resources/dashboards/republish-after-draft-update/out.test.toml index b3b039ee392..b423ee4c554 100644 --- a/acceptance/bundle/resources/dashboards/republish-after-draft-update/out.test.toml +++ b/acceptance/bundle/resources/dashboards/republish-after-draft-update/out.test.toml @@ -1,4 +1,4 @@ Cloud = false RequiresWarehouse = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/dashboards/simple/out.test.toml b/acceptance/bundle/resources/dashboards/simple/out.test.toml index c42373c3478..db7a7fc7904 100644 --- a/acceptance/bundle/resources/dashboards/simple/out.test.toml +++ b/acceptance/bundle/resources/dashboards/simple/out.test.toml @@ -1,5 +1,5 @@ Cloud = true RequiresWarehouse = true RunsOnDbr = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/dashboards/simple_outside_bundle_root/out.test.toml b/acceptance/bundle/resources/dashboards/simple_outside_bundle_root/out.test.toml index c42373c3478..db7a7fc7904 100644 --- a/acceptance/bundle/resources/dashboards/simple_outside_bundle_root/out.test.toml +++ b/acceptance/bundle/resources/dashboards/simple_outside_bundle_root/out.test.toml @@ -1,5 +1,5 @@ Cloud = true RequiresWarehouse = true RunsOnDbr = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/dashboards/simple_syncroot/out.test.toml b/acceptance/bundle/resources/dashboards/simple_syncroot/out.test.toml index c42373c3478..db7a7fc7904 100644 --- a/acceptance/bundle/resources/dashboards/simple_syncroot/out.test.toml +++ b/acceptance/bundle/resources/dashboards/simple_syncroot/out.test.toml @@ -1,5 +1,5 @@ Cloud = true RequiresWarehouse = true RunsOnDbr = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/dashboards/unpublish-out-of-band/out.test.toml b/acceptance/bundle/resources/dashboards/unpublish-out-of-band/out.test.toml index c42373c3478..db7a7fc7904 100644 --- a/acceptance/bundle/resources/dashboards/unpublish-out-of-band/out.test.toml +++ b/acceptance/bundle/resources/dashboards/unpublish-out-of-band/out.test.toml @@ -1,5 +1,5 @@ Cloud = true RequiresWarehouse = true RunsOnDbr = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/database_catalogs/basic/out.test.toml b/acceptance/bundle/resources/database_catalogs/basic/out.test.toml index 0227211aff5..8e07072d596 100644 --- a/acceptance/bundle/resources/database_catalogs/basic/out.test.toml +++ b/acceptance/bundle/resources/database_catalogs/basic/out.test.toml @@ -3,5 +3,5 @@ CloudSlow = true RequiresUnityCatalog = true RunsOnDbr = true CloudEnvs.gcp = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/database_catalogs/recreate/out.test.toml b/acceptance/bundle/resources/database_catalogs/recreate/out.test.toml index bffdd4c91db..1b77ea7eec0 100644 --- a/acceptance/bundle/resources/database_catalogs/recreate/out.test.toml +++ b/acceptance/bundle/resources/database_catalogs/recreate/out.test.toml @@ -1,5 +1,5 @@ Cloud = false RequiresUnityCatalog = true CloudEnvs.gcp = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/database_instances/recreate/out.test.toml b/acceptance/bundle/resources/database_instances/recreate/out.test.toml index bffdd4c91db..1b77ea7eec0 100644 --- a/acceptance/bundle/resources/database_instances/recreate/out.test.toml +++ b/acceptance/bundle/resources/database_instances/recreate/out.test.toml @@ -1,5 +1,5 @@ Cloud = false RequiresUnityCatalog = true CloudEnvs.gcp = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/database_instances/single-instance/out.test.toml b/acceptance/bundle/resources/database_instances/single-instance/out.test.toml index 75800313cf0..d5bc9dfb6b2 100644 --- a/acceptance/bundle/resources/database_instances/single-instance/out.test.toml +++ b/acceptance/bundle/resources/database_instances/single-instance/out.test.toml @@ -2,5 +2,5 @@ Cloud = true CloudSlow = true RequiresUnityCatalog = true CloudEnvs.gcp = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/experiments/basic/out.test.toml b/acceptance/bundle/resources/experiments/basic/out.test.toml index f9a87c23500..681e90697f0 100644 --- a/acceptance/bundle/resources/experiments/basic/out.test.toml +++ b/acceptance/bundle/resources/experiments/basic/out.test.toml @@ -1,4 +1,4 @@ Cloud = true RunsOnDbr = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/external_locations/out.test.toml b/acceptance/bundle/resources/external_locations/out.test.toml index 3fa8da1cda1..a2e41fad08d 100644 --- a/acceptance/bundle/resources/external_locations/out.test.toml +++ b/acceptance/bundle/resources/external_locations/out.test.toml @@ -1,4 +1,4 @@ Cloud = false RequiresUnityCatalog = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/genie_spaces/delete_warning/out.test.toml b/acceptance/bundle/resources/genie_spaces/delete_warning/out.test.toml index b3b039ee392..b423ee4c554 100644 --- a/acceptance/bundle/resources/genie_spaces/delete_warning/out.test.toml +++ b/acceptance/bundle/resources/genie_spaces/delete_warning/out.test.toml @@ -1,4 +1,4 @@ Cloud = false RequiresWarehouse = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/genie_spaces/inline/out.test.toml b/acceptance/bundle/resources/genie_spaces/inline/out.test.toml index b3b039ee392..b423ee4c554 100644 --- a/acceptance/bundle/resources/genie_spaces/inline/out.test.toml +++ b/acceptance/bundle/resources/genie_spaces/inline/out.test.toml @@ -1,4 +1,4 @@ Cloud = false RequiresWarehouse = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/genie_spaces/parent_path_update/out.test.toml b/acceptance/bundle/resources/genie_spaces/parent_path_update/out.test.toml index fb7c5e7772d..77f42f8ab8b 100644 --- a/acceptance/bundle/resources/genie_spaces/parent_path_update/out.test.toml +++ b/acceptance/bundle/resources/genie_spaces/parent_path_update/out.test.toml @@ -1,4 +1,4 @@ Cloud = true RequiresWarehouse = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/genie_spaces/recreate_when_gone/out.test.toml b/acceptance/bundle/resources/genie_spaces/recreate_when_gone/out.test.toml index fb7c5e7772d..77f42f8ab8b 100644 --- a/acceptance/bundle/resources/genie_spaces/recreate_when_gone/out.test.toml +++ b/acceptance/bundle/resources/genie_spaces/recreate_when_gone/out.test.toml @@ -1,4 +1,4 @@ Cloud = true RequiresWarehouse = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/genie_spaces/serialized_space/out.test.toml b/acceptance/bundle/resources/genie_spaces/serialized_space/out.test.toml index b3b039ee392..b423ee4c554 100644 --- a/acceptance/bundle/resources/genie_spaces/serialized_space/out.test.toml +++ b/acceptance/bundle/resources/genie_spaces/serialized_space/out.test.toml @@ -1,4 +1,4 @@ Cloud = false RequiresWarehouse = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/genie_spaces/simple/out.test.toml b/acceptance/bundle/resources/genie_spaces/simple/out.test.toml index fb7c5e7772d..77f42f8ab8b 100644 --- a/acceptance/bundle/resources/genie_spaces/simple/out.test.toml +++ b/acceptance/bundle/resources/genie_spaces/simple/out.test.toml @@ -1,4 +1,4 @@ Cloud = true RequiresWarehouse = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/genie_spaces/version_migration/out.test.toml b/acceptance/bundle/resources/genie_spaces/version_migration/out.test.toml index fb7c5e7772d..77f42f8ab8b 100644 --- a/acceptance/bundle/resources/genie_spaces/version_migration/out.test.toml +++ b/acceptance/bundle/resources/genie_spaces/version_migration/out.test.toml @@ -1,4 +1,4 @@ Cloud = true RequiresWarehouse = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/grants/catalogs/out.test.toml b/acceptance/bundle/resources/grants/catalogs/out.test.toml index 8197bd50f28..6c47aa9e8d1 100644 --- a/acceptance/bundle/resources/grants/catalogs/out.test.toml +++ b/acceptance/bundle/resources/grants/catalogs/out.test.toml @@ -1,4 +1,4 @@ Cloud = true RequiresUnityCatalog = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/grants/registered_models/out.test.toml b/acceptance/bundle/resources/grants/registered_models/out.test.toml index 026bc5f1a2d..3587a9c012d 100644 --- a/acceptance/bundle/resources/grants/registered_models/out.test.toml +++ b/acceptance/bundle/resources/grants/registered_models/out.test.toml @@ -1,4 +1,4 @@ Cloud = true RequiresUnityCatalog = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/grants/schemas/all_privileges/out.test.toml b/acceptance/bundle/resources/grants/schemas/all_privileges/out.test.toml index 026bc5f1a2d..3587a9c012d 100644 --- a/acceptance/bundle/resources/grants/schemas/all_privileges/out.test.toml +++ b/acceptance/bundle/resources/grants/schemas/all_privileges/out.test.toml @@ -1,4 +1,4 @@ Cloud = true RequiresUnityCatalog = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/grants/schemas/all_privileges_coexist/out.test.toml b/acceptance/bundle/resources/grants/schemas/all_privileges_coexist/out.test.toml index 026bc5f1a2d..3587a9c012d 100644 --- a/acceptance/bundle/resources/grants/schemas/all_privileges_coexist/out.test.toml +++ b/acceptance/bundle/resources/grants/schemas/all_privileges_coexist/out.test.toml @@ -1,4 +1,4 @@ Cloud = true RequiresUnityCatalog = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/grants/schemas/change_privilege/out.test.toml b/acceptance/bundle/resources/grants/schemas/change_privilege/out.test.toml index 026bc5f1a2d..3587a9c012d 100644 --- a/acceptance/bundle/resources/grants/schemas/change_privilege/out.test.toml +++ b/acceptance/bundle/resources/grants/schemas/change_privilege/out.test.toml @@ -1,4 +1,4 @@ Cloud = true RequiresUnityCatalog = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/grants/schemas/duplicate_principals/out.test.toml b/acceptance/bundle/resources/grants/schemas/duplicate_principals/out.test.toml index 026bc5f1a2d..3587a9c012d 100644 --- a/acceptance/bundle/resources/grants/schemas/duplicate_principals/out.test.toml +++ b/acceptance/bundle/resources/grants/schemas/duplicate_principals/out.test.toml @@ -1,4 +1,4 @@ Cloud = true RequiresUnityCatalog = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/grants/schemas/duplicate_privileges/out.test.toml b/acceptance/bundle/resources/grants/schemas/duplicate_privileges/out.test.toml index 026bc5f1a2d..3587a9c012d 100644 --- a/acceptance/bundle/resources/grants/schemas/duplicate_privileges/out.test.toml +++ b/acceptance/bundle/resources/grants/schemas/duplicate_privileges/out.test.toml @@ -1,4 +1,4 @@ Cloud = true RequiresUnityCatalog = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/grants/schemas/empty_array/out.test.toml b/acceptance/bundle/resources/grants/schemas/empty_array/out.test.toml index 026bc5f1a2d..3587a9c012d 100644 --- a/acceptance/bundle/resources/grants/schemas/empty_array/out.test.toml +++ b/acceptance/bundle/resources/grants/schemas/empty_array/out.test.toml @@ -1,4 +1,4 @@ Cloud = true RequiresUnityCatalog = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/grants/schemas/out_of_band_principal/out.test.toml b/acceptance/bundle/resources/grants/schemas/out_of_band_principal/out.test.toml index 026bc5f1a2d..3587a9c012d 100644 --- a/acceptance/bundle/resources/grants/schemas/out_of_band_principal/out.test.toml +++ b/acceptance/bundle/resources/grants/schemas/out_of_band_principal/out.test.toml @@ -1,4 +1,4 @@ Cloud = true RequiresUnityCatalog = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/grants/schemas/remove_all/out.test.toml b/acceptance/bundle/resources/grants/schemas/remove_all/out.test.toml index 026bc5f1a2d..3587a9c012d 100644 --- a/acceptance/bundle/resources/grants/schemas/remove_all/out.test.toml +++ b/acceptance/bundle/resources/grants/schemas/remove_all/out.test.toml @@ -1,4 +1,4 @@ Cloud = true RequiresUnityCatalog = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/grants/schemas/remove_principal/out.test.toml b/acceptance/bundle/resources/grants/schemas/remove_principal/out.test.toml index 026bc5f1a2d..3587a9c012d 100644 --- a/acceptance/bundle/resources/grants/schemas/remove_principal/out.test.toml +++ b/acceptance/bundle/resources/grants/schemas/remove_principal/out.test.toml @@ -1,4 +1,4 @@ Cloud = true RequiresUnityCatalog = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/grants/volumes/out.test.toml b/acceptance/bundle/resources/grants/volumes/out.test.toml index 026bc5f1a2d..3587a9c012d 100644 --- a/acceptance/bundle/resources/grants/volumes/out.test.toml +++ b/acceptance/bundle/resources/grants/volumes/out.test.toml @@ -1,4 +1,4 @@ Cloud = true RequiresUnityCatalog = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/independent/out.test.toml b/acceptance/bundle/resources/independent/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/resources/independent/out.test.toml +++ b/acceptance/bundle/resources/independent/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/instance_pools/out.test.toml b/acceptance/bundle/resources/instance_pools/out.test.toml index 51a5602947c..2c6699da193 100644 --- a/acceptance/bundle/resources/instance_pools/out.test.toml +++ b/acceptance/bundle/resources/instance_pools/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/job_runs/basic/out.test.toml b/acceptance/bundle/resources/job_runs/basic/out.test.toml index 51a5602947c..2c6699da193 100644 --- a/acceptance/bundle/resources/job_runs/basic/out.test.toml +++ b/acceptance/bundle/resources/job_runs/basic/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/job_runs/job_parameters/out.test.toml b/acceptance/bundle/resources/job_runs/job_parameters/out.test.toml index 51a5602947c..2c6699da193 100644 --- a/acceptance/bundle/resources/job_runs/job_parameters/out.test.toml +++ b/acceptance/bundle/resources/job_runs/job_parameters/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/job_runs/redeploy/out.test.toml b/acceptance/bundle/resources/job_runs/redeploy/out.test.toml index 51a5602947c..2c6699da193 100644 --- a/acceptance/bundle/resources/job_runs/redeploy/out.test.toml +++ b/acceptance/bundle/resources/job_runs/redeploy/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/jobs/alert-task/out.test.toml b/acceptance/bundle/resources/jobs/alert-task/out.test.toml index 0fc51ee009b..264de50aa96 100644 --- a/acceptance/bundle/resources/jobs/alert-task/out.test.toml +++ b/acceptance/bundle/resources/jobs/alert-task/out.test.toml @@ -1,3 +1,3 @@ Cloud = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/jobs/big_id/out.test.toml b/acceptance/bundle/resources/jobs/big_id/out.test.toml index 7508f6b670e..37356d54551 100644 --- a/acceptance/bundle/resources/jobs/big_id/out.test.toml +++ b/acceptance/bundle/resources/jobs/big_id/out.test.toml @@ -1,4 +1,4 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/resources/jobs/big_id/test.toml b/acceptance/bundle/resources/jobs/big_id/test.toml index 467d0855fa4..bc436b2d544 100644 --- a/acceptance/bundle/resources/jobs/big_id/test.toml +++ b/acceptance/bundle/resources/jobs/big_id/test.toml @@ -7,7 +7,7 @@ EnvMatrix.READPLAN = ["", "1"] # is written before the deployment version exists, so the deployment stamp never reaches # the applied resource and the next plan reports it as a change. Recording is skipped here # until the stamp is written into the saved plan too. -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] [[Repls]] Old = '9223372036854775807' diff --git a/acceptance/bundle/resources/jobs/check-metadata/out.test.toml b/acceptance/bundle/resources/jobs/check-metadata/out.test.toml index 0fc51ee009b..264de50aa96 100644 --- a/acceptance/bundle/resources/jobs/check-metadata/out.test.toml +++ b/acceptance/bundle/resources/jobs/check-metadata/out.test.toml @@ -1,3 +1,3 @@ Cloud = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/jobs/create-error/out.test.toml b/acceptance/bundle/resources/jobs/create-error/out.test.toml index 51a5602947c..2c6699da193 100644 --- a/acceptance/bundle/resources/jobs/create-error/out.test.toml +++ b/acceptance/bundle/resources/jobs/create-error/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/jobs/delete_job/out.test.toml b/acceptance/bundle/resources/jobs/delete_job/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/resources/jobs/delete_job/out.test.toml +++ b/acceptance/bundle/resources/jobs/delete_job/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/jobs/delete_task/out.test.toml b/acceptance/bundle/resources/jobs/delete_task/out.test.toml index 428575c5aa3..491291484b8 100644 --- a/acceptance/bundle/resources/jobs/delete_task/out.test.toml +++ b/acceptance/bundle/resources/jobs/delete_task/out.test.toml @@ -1,4 +1,4 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/resources/jobs/delete_task/test.toml b/acceptance/bundle/resources/jobs/delete_task/test.toml index cca4fd1ac9c..57ec9bccd00 100644 --- a/acceptance/bundle/resources/jobs/delete_task/test.toml +++ b/acceptance/bundle/resources/jobs/delete_task/test.toml @@ -4,5 +4,5 @@ EnvMatrix.READPLAN = ["", "1"] # is written before the deployment version exists, so the deployment stamp never reaches # the applied resource and the next plan reports it as a change. Recording is skipped here # until the stamp is written into the saved plan too. -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/resources/jobs/double-underscore-keys/out.test.toml b/acceptance/bundle/resources/jobs/double-underscore-keys/out.test.toml index f9a87c23500..681e90697f0 100644 --- a/acceptance/bundle/resources/jobs/double-underscore-keys/out.test.toml +++ b/acceptance/bundle/resources/jobs/double-underscore-keys/out.test.toml @@ -1,4 +1,4 @@ Cloud = true RunsOnDbr = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/jobs/fail-on-active-runs/out.test.toml b/acceptance/bundle/resources/jobs/fail-on-active-runs/out.test.toml index f9a87c23500..681e90697f0 100644 --- a/acceptance/bundle/resources/jobs/fail-on-active-runs/out.test.toml +++ b/acceptance/bundle/resources/jobs/fail-on-active-runs/out.test.toml @@ -1,4 +1,4 @@ Cloud = true RunsOnDbr = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/jobs/instance_pool_and_node_type/out.test.toml b/acceptance/bundle/resources/jobs/instance_pool_and_node_type/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/resources/jobs/instance_pool_and_node_type/out.test.toml +++ b/acceptance/bundle/resources/jobs/instance_pool_and_node_type/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/jobs/no-git-provider/out.test.toml b/acceptance/bundle/resources/jobs/no-git-provider/out.test.toml index f9a87c23500..681e90697f0 100644 --- a/acceptance/bundle/resources/jobs/no-git-provider/out.test.toml +++ b/acceptance/bundle/resources/jobs/no-git-provider/out.test.toml @@ -1,4 +1,4 @@ Cloud = true RunsOnDbr = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/jobs/num_workers/out.test.toml b/acceptance/bundle/resources/jobs/num_workers/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/resources/jobs/num_workers/out.test.toml +++ b/acceptance/bundle/resources/jobs/num_workers/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/jobs/on_failure_empty_slice/out.test.toml b/acceptance/bundle/resources/jobs/on_failure_empty_slice/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/resources/jobs/on_failure_empty_slice/out.test.toml +++ b/acceptance/bundle/resources/jobs/on_failure_empty_slice/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/jobs/remote_add_tag/out.test.toml b/acceptance/bundle/resources/jobs/remote_add_tag/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/resources/jobs/remote_add_tag/out.test.toml +++ b/acceptance/bundle/resources/jobs/remote_add_tag/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/jobs/remote_delete/deploy/out.test.toml b/acceptance/bundle/resources/jobs/remote_delete/deploy/out.test.toml index 428575c5aa3..491291484b8 100644 --- a/acceptance/bundle/resources/jobs/remote_delete/deploy/out.test.toml +++ b/acceptance/bundle/resources/jobs/remote_delete/deploy/out.test.toml @@ -1,4 +1,4 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/resources/jobs/remote_delete/deploy/test.toml b/acceptance/bundle/resources/jobs/remote_delete/deploy/test.toml index cca4fd1ac9c..57ec9bccd00 100644 --- a/acceptance/bundle/resources/jobs/remote_delete/deploy/test.toml +++ b/acceptance/bundle/resources/jobs/remote_delete/deploy/test.toml @@ -4,5 +4,5 @@ EnvMatrix.READPLAN = ["", "1"] # is written before the deployment version exists, so the deployment stamp never reaches # the applied resource and the next plan reports it as a change. Recording is skipped here # until the stamp is written into the saved plan too. -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/resources/jobs/remote_delete/destroy/out.test.toml b/acceptance/bundle/resources/jobs/remote_delete/destroy/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/resources/jobs/remote_delete/destroy/out.test.toml +++ b/acceptance/bundle/resources/jobs/remote_delete/destroy/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/jobs/remote_delete/removed_from_config/out.test.toml b/acceptance/bundle/resources/jobs/remote_delete/removed_from_config/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/resources/jobs/remote_delete/removed_from_config/out.test.toml +++ b/acceptance/bundle/resources/jobs/remote_delete/removed_from_config/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/jobs/remote_matches_config/out.test.toml b/acceptance/bundle/resources/jobs/remote_matches_config/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/resources/jobs/remote_matches_config/out.test.toml +++ b/acceptance/bundle/resources/jobs/remote_matches_config/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/jobs/shared-root-path/out.test.toml b/acceptance/bundle/resources/jobs/shared-root-path/out.test.toml index f9a87c23500..681e90697f0 100644 --- a/acceptance/bundle/resources/jobs/shared-root-path/out.test.toml +++ b/acceptance/bundle/resources/jobs/shared-root-path/out.test.toml @@ -1,4 +1,4 @@ Cloud = true RunsOnDbr = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/jobs/tags_empty_map/out.test.toml b/acceptance/bundle/resources/jobs/tags_empty_map/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/resources/jobs/tags_empty_map/out.test.toml +++ b/acceptance/bundle/resources/jobs/tags_empty_map/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/jobs/task-source/out.test.toml b/acceptance/bundle/resources/jobs/task-source/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/resources/jobs/task-source/out.test.toml +++ b/acceptance/bundle/resources/jobs/task-source/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/jobs/tasks-reorder-locally/out.test.toml b/acceptance/bundle/resources/jobs/tasks-reorder-locally/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/resources/jobs/tasks-reorder-locally/out.test.toml +++ b/acceptance/bundle/resources/jobs/tasks-reorder-locally/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/jobs/unknown-terraform-field/out.test.toml b/acceptance/bundle/resources/jobs/unknown-terraform-field/out.test.toml index 2563df1863f..33ed6258236 100644 --- a/acceptance/bundle/resources/jobs/unknown-terraform-field/out.test.toml +++ b/acceptance/bundle/resources/jobs/unknown-terraform-field/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/jobs/update/out.test.toml b/acceptance/bundle/resources/jobs/update/out.test.toml index 428575c5aa3..491291484b8 100644 --- a/acceptance/bundle/resources/jobs/update/out.test.toml +++ b/acceptance/bundle/resources/jobs/update/out.test.toml @@ -1,4 +1,4 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/resources/jobs/update/test.toml b/acceptance/bundle/resources/jobs/update/test.toml index a8a9a1e90b9..56096b047d3 100644 --- a/acceptance/bundle/resources/jobs/update/test.toml +++ b/acceptance/bundle/resources/jobs/update/test.toml @@ -3,5 +3,5 @@ EnvMatrix.READPLAN = ["", "1"] # is written before the deployment version exists, so the deployment stamp never reaches # the applied resource and the next plan reports it as a change. Recording is skipped here # until the stamp is written into the saved plan too. -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/resources/jobs/update_single_node/out.test.toml b/acceptance/bundle/resources/jobs/update_single_node/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/resources/jobs/update_single_node/out.test.toml +++ b/acceptance/bundle/resources/jobs/update_single_node/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/jobs/webhook-reorder-remote/out.test.toml b/acceptance/bundle/resources/jobs/webhook-reorder-remote/out.test.toml index 51a5602947c..2c6699da193 100644 --- a/acceptance/bundle/resources/jobs/webhook-reorder-remote/out.test.toml +++ b/acceptance/bundle/resources/jobs/webhook-reorder-remote/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/model_serving_endpoints/basic/out.test.toml b/acceptance/bundle/resources/model_serving_endpoints/basic/out.test.toml index 026bc5f1a2d..3587a9c012d 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/basic/out.test.toml +++ b/acceptance/bundle/resources/model_serving_endpoints/basic/out.test.toml @@ -1,4 +1,4 @@ Cloud = true RequiresUnityCatalog = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/model_serving_endpoints/drift/recreated_same_name/out.test.toml b/acceptance/bundle/resources/model_serving_endpoints/drift/recreated_same_name/out.test.toml index 8197bd50f28..6c47aa9e8d1 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/drift/recreated_same_name/out.test.toml +++ b/acceptance/bundle/resources/model_serving_endpoints/drift/recreated_same_name/out.test.toml @@ -1,4 +1,4 @@ Cloud = true RequiresUnityCatalog = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_unmanaged/out.test.toml b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_unmanaged/out.test.toml index 51a5602947c..2c6699da193 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_unmanaged/out.test.toml +++ b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_unmanaged/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_with_config_update/out.test.toml b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_with_config_update/out.test.toml index 51a5602947c..2c6699da193 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_with_config_update/out.test.toml +++ b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_with_config_update/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/model_serving_endpoints/drift/write_only/out.test.toml b/acceptance/bundle/resources/model_serving_endpoints/drift/write_only/out.test.toml index 51a5602947c..2c6699da193 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/drift/write_only/out.test.toml +++ b/acceptance/bundle/resources/model_serving_endpoints/drift/write_only/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/model_serving_endpoints/recreate/catalog-name/out.test.toml b/acceptance/bundle/resources/model_serving_endpoints/recreate/catalog-name/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/recreate/catalog-name/out.test.toml +++ b/acceptance/bundle/resources/model_serving_endpoints/recreate/catalog-name/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/model_serving_endpoints/recreate/name-change/out.test.toml b/acceptance/bundle/resources/model_serving_endpoints/recreate/name-change/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/recreate/name-change/out.test.toml +++ b/acceptance/bundle/resources/model_serving_endpoints/recreate/name-change/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/model_serving_endpoints/recreate/route-optimized/out.test.toml b/acceptance/bundle/resources/model_serving_endpoints/recreate/route-optimized/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/recreate/route-optimized/out.test.toml +++ b/acceptance/bundle/resources/model_serving_endpoints/recreate/route-optimized/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/model_serving_endpoints/recreate/schema-name/out.test.toml b/acceptance/bundle/resources/model_serving_endpoints/recreate/schema-name/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/recreate/schema-name/out.test.toml +++ b/acceptance/bundle/resources/model_serving_endpoints/recreate/schema-name/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/model_serving_endpoints/recreate/table-prefix/out.test.toml b/acceptance/bundle/resources/model_serving_endpoints/recreate/table-prefix/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/recreate/table-prefix/out.test.toml +++ b/acceptance/bundle/resources/model_serving_endpoints/recreate/table-prefix/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/model_serving_endpoints/running-endpoint/out.test.toml b/acceptance/bundle/resources/model_serving_endpoints/running-endpoint/out.test.toml index 75800313cf0..d5bc9dfb6b2 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/running-endpoint/out.test.toml +++ b/acceptance/bundle/resources/model_serving_endpoints/running-endpoint/out.test.toml @@ -2,5 +2,5 @@ Cloud = true CloudSlow = true RequiresUnityCatalog = true CloudEnvs.gcp = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/model_serving_endpoints/update/ai-gateway/out.test.toml b/acceptance/bundle/resources/model_serving_endpoints/update/ai-gateway/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/update/ai-gateway/out.test.toml +++ b/acceptance/bundle/resources/model_serving_endpoints/update/ai-gateway/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/model_serving_endpoints/update/both_gateway_and_tags/out.test.toml b/acceptance/bundle/resources/model_serving_endpoints/update/both_gateway_and_tags/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/update/both_gateway_and_tags/out.test.toml +++ b/acceptance/bundle/resources/model_serving_endpoints/update/both_gateway_and_tags/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/model_serving_endpoints/update/config/out.test.toml b/acceptance/bundle/resources/model_serving_endpoints/update/config/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/update/config/out.test.toml +++ b/acceptance/bundle/resources/model_serving_endpoints/update/config/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/model_serving_endpoints/update/email-notifications/out.test.toml b/acceptance/bundle/resources/model_serving_endpoints/update/email-notifications/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/update/email-notifications/out.test.toml +++ b/acceptance/bundle/resources/model_serving_endpoints/update/email-notifications/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/model_serving_endpoints/update/tags/out.test.toml b/acceptance/bundle/resources/model_serving_endpoints/update/tags/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/update/tags/out.test.toml +++ b/acceptance/bundle/resources/model_serving_endpoints/update/tags/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/models/basic/out.test.toml b/acceptance/bundle/resources/models/basic/out.test.toml index f9a87c23500..681e90697f0 100644 --- a/acceptance/bundle/resources/models/basic/out.test.toml +++ b/acceptance/bundle/resources/models/basic/out.test.toml @@ -1,4 +1,4 @@ Cloud = true RunsOnDbr = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/models/empty-name/out.test.toml b/acceptance/bundle/resources/models/empty-name/out.test.toml index 0fc51ee009b..264de50aa96 100644 --- a/acceptance/bundle/resources/models/empty-name/out.test.toml +++ b/acceptance/bundle/resources/models/empty-name/out.test.toml @@ -1,3 +1,3 @@ Cloud = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/models/readplan-permissions/out.test.toml b/acceptance/bundle/resources/models/readplan-permissions/out.test.toml index ecdf1da5791..71b97f1370d 100644 --- a/acceptance/bundle/resources/models/readplan-permissions/out.test.toml +++ b/acceptance/bundle/resources/models/readplan-permissions/out.test.toml @@ -1,4 +1,4 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/resources/permissions/apps/current_can_manage/out.test.toml b/acceptance/bundle/resources/permissions/apps/current_can_manage/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/resources/permissions/apps/current_can_manage/out.test.toml +++ b/acceptance/bundle/resources/permissions/apps/current_can_manage/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/permissions/apps/other_can_manage/out.test.toml b/acceptance/bundle/resources/permissions/apps/other_can_manage/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/resources/permissions/apps/other_can_manage/out.test.toml +++ b/acceptance/bundle/resources/permissions/apps/other_can_manage/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/permissions/clusters/current_can_manage/out.test.toml b/acceptance/bundle/resources/permissions/clusters/current_can_manage/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/resources/permissions/clusters/current_can_manage/out.test.toml +++ b/acceptance/bundle/resources/permissions/clusters/current_can_manage/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/permissions/clusters/target/out.test.toml b/acceptance/bundle/resources/permissions/clusters/target/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/resources/permissions/clusters/target/out.test.toml +++ b/acceptance/bundle/resources/permissions/clusters/target/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/permissions/dashboards/create/out.test.toml b/acceptance/bundle/resources/permissions/dashboards/create/out.test.toml index 4ecffd9ce40..456831a3552 100644 --- a/acceptance/bundle/resources/permissions/dashboards/create/out.test.toml +++ b/acceptance/bundle/resources/permissions/dashboards/create/out.test.toml @@ -1,5 +1,5 @@ Cloud = true RequiresWarehouse = true CloudEnvs.gcp = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/permissions/database_instances/current_can_manage/out.test.toml b/acceptance/bundle/resources/permissions/database_instances/current_can_manage/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/resources/permissions/database_instances/current_can_manage/out.test.toml +++ b/acceptance/bundle/resources/permissions/database_instances/current_can_manage/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/permissions/experiments/current_can_manage/out.test.toml b/acceptance/bundle/resources/permissions/experiments/current_can_manage/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/resources/permissions/experiments/current_can_manage/out.test.toml +++ b/acceptance/bundle/resources/permissions/experiments/current_can_manage/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/permissions/factcheck/out.test.toml b/acceptance/bundle/resources/permissions/factcheck/out.test.toml index aefc8292031..174ddbce3d9 100644 --- a/acceptance/bundle/resources/permissions/factcheck/out.test.toml +++ b/acceptance/bundle/resources/permissions/factcheck/out.test.toml @@ -2,5 +2,5 @@ Cloud = true CloudSlow = true RunsOnDbr = true CloudEnvs.gcp = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/out.test.toml b/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/out.test.toml index 51a5602947c..2c6699da193 100644 --- a/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/out.test.toml +++ b/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/permissions/genie_spaces/out_of_band_deletion/out.test.toml b/acceptance/bundle/resources/permissions/genie_spaces/out_of_band_deletion/out.test.toml index 51a5602947c..2c6699da193 100644 --- a/acceptance/bundle/resources/permissions/genie_spaces/out_of_band_deletion/out.test.toml +++ b/acceptance/bundle/resources/permissions/genie_spaces/out_of_band_deletion/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/permissions/jobs/added_remotely/out.test.toml b/acceptance/bundle/resources/permissions/jobs/added_remotely/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/resources/permissions/jobs/added_remotely/out.test.toml +++ b/acceptance/bundle/resources/permissions/jobs/added_remotely/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/permissions/jobs/current_can_manage/out.test.toml b/acceptance/bundle/resources/permissions/jobs/current_can_manage/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/resources/permissions/jobs/current_can_manage/out.test.toml +++ b/acceptance/bundle/resources/permissions/jobs/current_can_manage/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/permissions/jobs/current_can_manage_run/out.test.toml b/acceptance/bundle/resources/permissions/jobs/current_can_manage_run/out.test.toml index d18df0cfbd2..27bdd93332c 100644 --- a/acceptance/bundle/resources/permissions/jobs/current_can_manage_run/out.test.toml +++ b/acceptance/bundle/resources/permissions/jobs/current_can_manage_run/out.test.toml @@ -1,4 +1,4 @@ Cloud = true CloudEnvs.gcp = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/permissions/jobs/current_is_owner/out.test.toml b/acceptance/bundle/resources/permissions/jobs/current_is_owner/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/resources/permissions/jobs/current_is_owner/out.test.toml +++ b/acceptance/bundle/resources/permissions/jobs/current_is_owner/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/permissions/jobs/delete_one/out.test.toml b/acceptance/bundle/resources/permissions/jobs/delete_one/out.test.toml index 026bc5f1a2d..3587a9c012d 100644 --- a/acceptance/bundle/resources/permissions/jobs/delete_one/out.test.toml +++ b/acceptance/bundle/resources/permissions/jobs/delete_one/out.test.toml @@ -1,4 +1,4 @@ Cloud = true RequiresUnityCatalog = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/permissions/jobs/deleted_remotely/out.test.toml b/acceptance/bundle/resources/permissions/jobs/deleted_remotely/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/resources/permissions/jobs/deleted_remotely/out.test.toml +++ b/acceptance/bundle/resources/permissions/jobs/deleted_remotely/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/permissions/jobs/destroy_without_mgmtperms/test.toml b/acceptance/bundle/resources/permissions/jobs/destroy_without_mgmtperms/test.toml index a55b7341585..597812d5b48 100644 --- a/acceptance/bundle/resources/permissions/jobs/destroy_without_mgmtperms/test.toml +++ b/acceptance/bundle/resources/permissions/jobs/destroy_without_mgmtperms/test.toml @@ -1,4 +1,4 @@ # Recording needs a bundle it has seen from the start. This test seeds a state file, # so recording refuses it. TODO(DMS): drop this once existing state can be # handed over to the service (see the TODO in dstate.Open). -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/resources/permissions/jobs/destroy_without_mgmtperms/with_permissions/out.test.toml b/acceptance/bundle/resources/permissions/jobs/destroy_without_mgmtperms/with_permissions/out.test.toml index 4d60b937f26..cf654d42d69 100644 --- a/acceptance/bundle/resources/permissions/jobs/destroy_without_mgmtperms/with_permissions/out.test.toml +++ b/acceptance/bundle/resources/permissions/jobs/destroy_without_mgmtperms/with_permissions/out.test.toml @@ -2,5 +2,5 @@ Cloud = true RunsOnDbr = false CloudEnvs.azure = false CloudEnvs.gcp = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/resources/permissions/jobs/destroy_without_mgmtperms/without_permissions/out.test.toml b/acceptance/bundle/resources/permissions/jobs/destroy_without_mgmtperms/without_permissions/out.test.toml index 4d60b937f26..cf654d42d69 100644 --- a/acceptance/bundle/resources/permissions/jobs/destroy_without_mgmtperms/without_permissions/out.test.toml +++ b/acceptance/bundle/resources/permissions/jobs/destroy_without_mgmtperms/without_permissions/out.test.toml @@ -2,5 +2,5 @@ Cloud = true RunsOnDbr = false CloudEnvs.azure = false CloudEnvs.gcp = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/resources/permissions/jobs/empty_list/out.test.toml b/acceptance/bundle/resources/permissions/jobs/empty_list/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/resources/permissions/jobs/empty_list/out.test.toml +++ b/acceptance/bundle/resources/permissions/jobs/empty_list/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/permissions/jobs/other_can_manage/out.test.toml b/acceptance/bundle/resources/permissions/jobs/other_can_manage/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/resources/permissions/jobs/other_can_manage/out.test.toml +++ b/acceptance/bundle/resources/permissions/jobs/other_can_manage/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/permissions/jobs/other_can_manage_run/out.test.toml b/acceptance/bundle/resources/permissions/jobs/other_can_manage_run/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/resources/permissions/jobs/other_can_manage_run/out.test.toml +++ b/acceptance/bundle/resources/permissions/jobs/other_can_manage_run/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/permissions/jobs/other_is_owner/out.test.toml b/acceptance/bundle/resources/permissions/jobs/other_is_owner/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/resources/permissions/jobs/other_is_owner/out.test.toml +++ b/acceptance/bundle/resources/permissions/jobs/other_is_owner/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/permissions/jobs/reorder_locally/out.test.toml b/acceptance/bundle/resources/permissions/jobs/reorder_locally/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/resources/permissions/jobs/reorder_locally/out.test.toml +++ b/acceptance/bundle/resources/permissions/jobs/reorder_locally/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/permissions/jobs/reorder_remotely/out.test.toml b/acceptance/bundle/resources/permissions/jobs/reorder_remotely/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/resources/permissions/jobs/reorder_remotely/out.test.toml +++ b/acceptance/bundle/resources/permissions/jobs/reorder_remotely/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/permissions/jobs/update/out.test.toml b/acceptance/bundle/resources/permissions/jobs/update/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/resources/permissions/jobs/update/out.test.toml +++ b/acceptance/bundle/resources/permissions/jobs/update/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/permissions/jobs/viewers/out.test.toml b/acceptance/bundle/resources/permissions/jobs/viewers/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/resources/permissions/jobs/viewers/out.test.toml +++ b/acceptance/bundle/resources/permissions/jobs/viewers/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/permissions/models/current_can_manage/out.test.toml b/acceptance/bundle/resources/permissions/models/current_can_manage/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/resources/permissions/models/current_can_manage/out.test.toml +++ b/acceptance/bundle/resources/permissions/models/current_can_manage/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/permissions/out.test.toml b/acceptance/bundle/resources/permissions/out.test.toml index 5636693f291..54c616ad472 100644 --- a/acceptance/bundle/resources/permissions/out.test.toml +++ b/acceptance/bundle/resources/permissions/out.test.toml @@ -1,4 +1,4 @@ Cloud = false Phase = 1 -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/permissions/pipelines/504/create/out.test.toml b/acceptance/bundle/resources/permissions/pipelines/504/create/out.test.toml index 5356bd3fdf7..473e201c5c2 100644 --- a/acceptance/bundle/resources/permissions/pipelines/504/create/out.test.toml +++ b/acceptance/bundle/resources/permissions/pipelines/504/create/out.test.toml @@ -1,4 +1,4 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] EnvMatrix.READPLAN = [] diff --git a/acceptance/bundle/resources/permissions/pipelines/504/plan/out.test.toml b/acceptance/bundle/resources/permissions/pipelines/504/plan/out.test.toml index 5356bd3fdf7..473e201c5c2 100644 --- a/acceptance/bundle/resources/permissions/pipelines/504/plan/out.test.toml +++ b/acceptance/bundle/resources/permissions/pipelines/504/plan/out.test.toml @@ -1,4 +1,4 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] EnvMatrix.READPLAN = [] diff --git a/acceptance/bundle/resources/permissions/pipelines/504/update/out.test.toml b/acceptance/bundle/resources/permissions/pipelines/504/update/out.test.toml index 5356bd3fdf7..473e201c5c2 100644 --- a/acceptance/bundle/resources/permissions/pipelines/504/update/out.test.toml +++ b/acceptance/bundle/resources/permissions/pipelines/504/update/out.test.toml @@ -1,4 +1,4 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] EnvMatrix.READPLAN = [] diff --git a/acceptance/bundle/resources/permissions/pipelines/current_can_manage/out.test.toml b/acceptance/bundle/resources/permissions/pipelines/current_can_manage/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/resources/permissions/pipelines/current_can_manage/out.test.toml +++ b/acceptance/bundle/resources/permissions/pipelines/current_can_manage/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/permissions/pipelines/current_is_owner/out.test.toml b/acceptance/bundle/resources/permissions/pipelines/current_is_owner/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/resources/permissions/pipelines/current_is_owner/out.test.toml +++ b/acceptance/bundle/resources/permissions/pipelines/current_is_owner/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/permissions/pipelines/empty_list/out.test.toml b/acceptance/bundle/resources/permissions/pipelines/empty_list/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/resources/permissions/pipelines/empty_list/out.test.toml +++ b/acceptance/bundle/resources/permissions/pipelines/empty_list/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/permissions/pipelines/other_can_manage/out.test.toml b/acceptance/bundle/resources/permissions/pipelines/other_can_manage/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/resources/permissions/pipelines/other_can_manage/out.test.toml +++ b/acceptance/bundle/resources/permissions/pipelines/other_can_manage/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/permissions/pipelines/other_is_owner/out.test.toml b/acceptance/bundle/resources/permissions/pipelines/other_is_owner/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/resources/permissions/pipelines/other_is_owner/out.test.toml +++ b/acceptance/bundle/resources/permissions/pipelines/other_is_owner/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/permissions/pipelines/update/out.test.toml b/acceptance/bundle/resources/permissions/pipelines/update/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/resources/permissions/pipelines/update/out.test.toml +++ b/acceptance/bundle/resources/permissions/pipelines/update/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/permissions/postgres_projects/current_can_manage/out.test.toml b/acceptance/bundle/resources/permissions/postgres_projects/current_can_manage/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/resources/permissions/postgres_projects/current_can_manage/out.test.toml +++ b/acceptance/bundle/resources/permissions/postgres_projects/current_can_manage/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/permissions/sql_warehouses/current_can_manage/out.test.toml b/acceptance/bundle/resources/permissions/sql_warehouses/current_can_manage/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/resources/permissions/sql_warehouses/current_can_manage/out.test.toml +++ b/acceptance/bundle/resources/permissions/sql_warehouses/current_can_manage/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/permissions/target_permissions/out.test.toml b/acceptance/bundle/resources/permissions/target_permissions/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/resources/permissions/target_permissions/out.test.toml +++ b/acceptance/bundle/resources/permissions/target_permissions/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/permissions/vector_search_endpoints/current_can_manage/out.test.toml b/acceptance/bundle/resources/permissions/vector_search_endpoints/current_can_manage/out.test.toml index 51a5602947c..2c6699da193 100644 --- a/acceptance/bundle/resources/permissions/vector_search_endpoints/current_can_manage/out.test.toml +++ b/acceptance/bundle/resources/permissions/vector_search_endpoints/current_can_manage/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/pipelines/allow-duplicate-names/out.test.toml b/acceptance/bundle/resources/pipelines/allow-duplicate-names/out.test.toml index 0fc51ee009b..264de50aa96 100644 --- a/acceptance/bundle/resources/pipelines/allow-duplicate-names/out.test.toml +++ b/acceptance/bundle/resources/pipelines/allow-duplicate-names/out.test.toml @@ -1,3 +1,3 @@ Cloud = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/pipelines/auto-approve/out.test.toml b/acceptance/bundle/resources/pipelines/auto-approve/out.test.toml index f9a87c23500..681e90697f0 100644 --- a/acceptance/bundle/resources/pipelines/auto-approve/out.test.toml +++ b/acceptance/bundle/resources/pipelines/auto-approve/out.test.toml @@ -1,4 +1,4 @@ Cloud = true RunsOnDbr = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/pipelines/drift/parameters/out.test.toml b/acceptance/bundle/resources/pipelines/drift/parameters/out.test.toml index 51a5602947c..2c6699da193 100644 --- a/acceptance/bundle/resources/pipelines/drift/parameters/out.test.toml +++ b/acceptance/bundle/resources/pipelines/drift/parameters/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/pipelines/lakeflow-pipeline/out.test.toml b/acceptance/bundle/resources/pipelines/lakeflow-pipeline/out.test.toml index 0fc51ee009b..264de50aa96 100644 --- a/acceptance/bundle/resources/pipelines/lakeflow-pipeline/out.test.toml +++ b/acceptance/bundle/resources/pipelines/lakeflow-pipeline/out.test.toml @@ -1,3 +1,3 @@ Cloud = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/pipelines/num-workers-zero/out.test.toml b/acceptance/bundle/resources/pipelines/num-workers-zero/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/resources/pipelines/num-workers-zero/out.test.toml +++ b/acceptance/bundle/resources/pipelines/num-workers-zero/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/pipelines/photon-true/out.test.toml b/acceptance/bundle/resources/pipelines/photon-true/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/resources/pipelines/photon-true/out.test.toml +++ b/acceptance/bundle/resources/pipelines/photon-true/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/out.test.toml b/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/out.test.toml +++ b/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/pipelines/recreate-keys/change-storage/out.test.toml b/acceptance/bundle/resources/pipelines/recreate-keys/change-storage/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/resources/pipelines/recreate-keys/change-storage/out.test.toml +++ b/acceptance/bundle/resources/pipelines/recreate-keys/change-storage/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/pipelines/recreate/out.test.toml b/acceptance/bundle/resources/pipelines/recreate/out.test.toml index b481e57b2fa..4401dadbd4e 100644 --- a/acceptance/bundle/resources/pipelines/recreate/out.test.toml +++ b/acceptance/bundle/resources/pipelines/recreate/out.test.toml @@ -1,5 +1,5 @@ Cloud = true RequiresUnityCatalog = true RunsOnDbr = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/pipelines/remote_matches_config/out.test.toml b/acceptance/bundle/resources/pipelines/remote_matches_config/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/resources/pipelines/remote_matches_config/out.test.toml +++ b/acceptance/bundle/resources/pipelines/remote_matches_config/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/pipelines/update/out.test.toml b/acceptance/bundle/resources/pipelines/update/out.test.toml index 0fc51ee009b..264de50aa96 100644 --- a/acceptance/bundle/resources/pipelines/update/out.test.toml +++ b/acceptance/bundle/resources/pipelines/update/out.test.toml @@ -1,3 +1,3 @@ Cloud = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/pipelines/zero-value-fields/out.test.toml b/acceptance/bundle/resources/pipelines/zero-value-fields/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/resources/pipelines/zero-value-fields/out.test.toml +++ b/acceptance/bundle/resources/pipelines/zero-value-fields/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/postgres_branches/basic/out.test.toml b/acceptance/bundle/resources/postgres_branches/basic/out.test.toml index 8ce0c4c33e2..c09534f265e 100644 --- a/acceptance/bundle/resources/postgres_branches/basic/out.test.toml +++ b/acceptance/bundle/resources/postgres_branches/basic/out.test.toml @@ -2,5 +2,5 @@ Cloud = true RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/postgres_branches/purge_on_delete/out.test.toml b/acceptance/bundle/resources/postgres_branches/purge_on_delete/out.test.toml index 8ce0c4c33e2..c09534f265e 100644 --- a/acceptance/bundle/resources/postgres_branches/purge_on_delete/out.test.toml +++ b/acceptance/bundle/resources/postgres_branches/purge_on_delete/out.test.toml @@ -2,5 +2,5 @@ Cloud = true RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/postgres_branches/purge_on_delete_transitions/out.test.toml b/acceptance/bundle/resources/postgres_branches/purge_on_delete_transitions/out.test.toml index 3c6ef20a158..5a29113cb28 100644 --- a/acceptance/bundle/resources/postgres_branches/purge_on_delete_transitions/out.test.toml +++ b/acceptance/bundle/resources/postgres_branches/purge_on_delete_transitions/out.test.toml @@ -2,5 +2,5 @@ Cloud = true RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/postgres_branches/recreate/out.test.toml b/acceptance/bundle/resources/postgres_branches/recreate/out.test.toml index 8ce0c4c33e2..c09534f265e 100644 --- a/acceptance/bundle/resources/postgres_branches/recreate/out.test.toml +++ b/acceptance/bundle/resources/postgres_branches/recreate/out.test.toml @@ -2,5 +2,5 @@ Cloud = true RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/postgres_branches/replace_existing/out.test.toml b/acceptance/bundle/resources/postgres_branches/replace_existing/out.test.toml index 8ce0c4c33e2..c09534f265e 100644 --- a/acceptance/bundle/resources/postgres_branches/replace_existing/out.test.toml +++ b/acceptance/bundle/resources/postgres_branches/replace_existing/out.test.toml @@ -2,5 +2,5 @@ Cloud = true RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/postgres_branches/update_protected/out.test.toml b/acceptance/bundle/resources/postgres_branches/update_protected/out.test.toml index 8ce0c4c33e2..c09534f265e 100644 --- a/acceptance/bundle/resources/postgres_branches/update_protected/out.test.toml +++ b/acceptance/bundle/resources/postgres_branches/update_protected/out.test.toml @@ -2,5 +2,5 @@ Cloud = true RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/postgres_branches/without_branch_id/out.test.toml b/acceptance/bundle/resources/postgres_branches/without_branch_id/out.test.toml index 8ce0c4c33e2..c09534f265e 100644 --- a/acceptance/bundle/resources/postgres_branches/without_branch_id/out.test.toml +++ b/acceptance/bundle/resources/postgres_branches/without_branch_id/out.test.toml @@ -2,5 +2,5 @@ Cloud = true RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/postgres_catalogs/basic/out.test.toml b/acceptance/bundle/resources/postgres_catalogs/basic/out.test.toml index 8ce0c4c33e2..c09534f265e 100644 --- a/acceptance/bundle/resources/postgres_catalogs/basic/out.test.toml +++ b/acceptance/bundle/resources/postgres_catalogs/basic/out.test.toml @@ -2,5 +2,5 @@ Cloud = true RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/postgres_catalogs/recreate/out.test.toml b/acceptance/bundle/resources/postgres_catalogs/recreate/out.test.toml index 8ce0c4c33e2..c09534f265e 100644 --- a/acceptance/bundle/resources/postgres_catalogs/recreate/out.test.toml +++ b/acceptance/bundle/resources/postgres_catalogs/recreate/out.test.toml @@ -2,5 +2,5 @@ Cloud = true RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/postgres_databases/basic/out.test.toml b/acceptance/bundle/resources/postgres_databases/basic/out.test.toml index 8ce0c4c33e2..c09534f265e 100644 --- a/acceptance/bundle/resources/postgres_databases/basic/out.test.toml +++ b/acceptance/bundle/resources/postgres_databases/basic/out.test.toml @@ -2,5 +2,5 @@ Cloud = true RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/postgres_databases/live_errors/bad_database_id/out.test.toml b/acceptance/bundle/resources/postgres_databases/live_errors/bad_database_id/out.test.toml index 3c6ef20a158..5a29113cb28 100644 --- a/acceptance/bundle/resources/postgres_databases/live_errors/bad_database_id/out.test.toml +++ b/acceptance/bundle/resources/postgres_databases/live_errors/bad_database_id/out.test.toml @@ -2,5 +2,5 @@ Cloud = true RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/postgres_databases/live_errors/bad_role_ref/out.test.toml b/acceptance/bundle/resources/postgres_databases/live_errors/bad_role_ref/out.test.toml index 3c6ef20a158..5a29113cb28 100644 --- a/acceptance/bundle/resources/postgres_databases/live_errors/bad_role_ref/out.test.toml +++ b/acceptance/bundle/resources/postgres_databases/live_errors/bad_role_ref/out.test.toml @@ -2,5 +2,5 @@ Cloud = true RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/postgres_databases/recreate/out.test.toml b/acceptance/bundle/resources/postgres_databases/recreate/out.test.toml index 8ce0c4c33e2..c09534f265e 100644 --- a/acceptance/bundle/resources/postgres_databases/recreate/out.test.toml +++ b/acceptance/bundle/resources/postgres_databases/recreate/out.test.toml @@ -2,5 +2,5 @@ Cloud = true RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/postgres_databases/replace_existing/out.test.toml b/acceptance/bundle/resources/postgres_databases/replace_existing/out.test.toml index f2f500f9a7d..69be6ba8159 100644 --- a/acceptance/bundle/resources/postgres_databases/replace_existing/out.test.toml +++ b/acceptance/bundle/resources/postgres_databases/replace_existing/out.test.toml @@ -2,5 +2,5 @@ Cloud = true RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/resources/postgres_databases/replace_existing/test.toml b/acceptance/bundle/resources/postgres_databases/replace_existing/test.toml index 31e3a00a75e..bb93cb1ad2e 100644 --- a/acceptance/bundle/resources/postgres_databases/replace_existing/test.toml +++ b/acceptance/bundle/resources/postgres_databases/replace_existing/test.toml @@ -1,3 +1,3 @@ # `bundle unbind` does not yet drop the resource from what the deployment metadata service # reports, so the plan after the unbind still sees the database as tracked. -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/resources/postgres_databases/update/out.test.toml b/acceptance/bundle/resources/postgres_databases/update/out.test.toml index 8ce0c4c33e2..c09534f265e 100644 --- a/acceptance/bundle/resources/postgres_databases/update/out.test.toml +++ b/acceptance/bundle/resources/postgres_databases/update/out.test.toml @@ -2,5 +2,5 @@ Cloud = true RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/postgres_endpoints/basic/out.test.toml b/acceptance/bundle/resources/postgres_endpoints/basic/out.test.toml index 8ce0c4c33e2..c09534f265e 100644 --- a/acceptance/bundle/resources/postgres_endpoints/basic/out.test.toml +++ b/acceptance/bundle/resources/postgres_endpoints/basic/out.test.toml @@ -2,5 +2,5 @@ Cloud = true RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/postgres_endpoints/recreate/out.test.toml b/acceptance/bundle/resources/postgres_endpoints/recreate/out.test.toml index 3c6ef20a158..5a29113cb28 100644 --- a/acceptance/bundle/resources/postgres_endpoints/recreate/out.test.toml +++ b/acceptance/bundle/resources/postgres_endpoints/recreate/out.test.toml @@ -2,5 +2,5 @@ Cloud = true RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/postgres_endpoints/replace_existing/out.test.toml b/acceptance/bundle/resources/postgres_endpoints/replace_existing/out.test.toml index 8ce0c4c33e2..c09534f265e 100644 --- a/acceptance/bundle/resources/postgres_endpoints/replace_existing/out.test.toml +++ b/acceptance/bundle/resources/postgres_endpoints/replace_existing/out.test.toml @@ -2,5 +2,5 @@ Cloud = true RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/postgres_endpoints/update_autoscaling/out.test.toml b/acceptance/bundle/resources/postgres_endpoints/update_autoscaling/out.test.toml index 8ce0c4c33e2..c09534f265e 100644 --- a/acceptance/bundle/resources/postgres_endpoints/update_autoscaling/out.test.toml +++ b/acceptance/bundle/resources/postgres_endpoints/update_autoscaling/out.test.toml @@ -2,5 +2,5 @@ Cloud = true RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/postgres_endpoints/without_endpoint_id/out.test.toml b/acceptance/bundle/resources/postgres_endpoints/without_endpoint_id/out.test.toml index 8ce0c4c33e2..c09534f265e 100644 --- a/acceptance/bundle/resources/postgres_endpoints/without_endpoint_id/out.test.toml +++ b/acceptance/bundle/resources/postgres_endpoints/without_endpoint_id/out.test.toml @@ -2,5 +2,5 @@ Cloud = true RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/postgres_projects/basic/out.test.toml b/acceptance/bundle/resources/postgres_projects/basic/out.test.toml index 8ce0c4c33e2..c09534f265e 100644 --- a/acceptance/bundle/resources/postgres_projects/basic/out.test.toml +++ b/acceptance/bundle/resources/postgres_projects/basic/out.test.toml @@ -2,5 +2,5 @@ Cloud = true RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/postgres_projects/purge_on_delete/out.test.toml b/acceptance/bundle/resources/postgres_projects/purge_on_delete/out.test.toml index 8ce0c4c33e2..c09534f265e 100644 --- a/acceptance/bundle/resources/postgres_projects/purge_on_delete/out.test.toml +++ b/acceptance/bundle/resources/postgres_projects/purge_on_delete/out.test.toml @@ -2,5 +2,5 @@ Cloud = true RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/postgres_projects/purge_on_delete_transitions/out.test.toml b/acceptance/bundle/resources/postgres_projects/purge_on_delete_transitions/out.test.toml index 3c6ef20a158..5a29113cb28 100644 --- a/acceptance/bundle/resources/postgres_projects/purge_on_delete_transitions/out.test.toml +++ b/acceptance/bundle/resources/postgres_projects/purge_on_delete_transitions/out.test.toml @@ -2,5 +2,5 @@ Cloud = true RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/postgres_projects/recreate/out.test.toml b/acceptance/bundle/resources/postgres_projects/recreate/out.test.toml index 8ce0c4c33e2..c09534f265e 100644 --- a/acceptance/bundle/resources/postgres_projects/recreate/out.test.toml +++ b/acceptance/bundle/resources/postgres_projects/recreate/out.test.toml @@ -2,5 +2,5 @@ Cloud = true RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/postgres_projects/update_display_name/out.test.toml b/acceptance/bundle/resources/postgres_projects/update_display_name/out.test.toml index 8ce0c4c33e2..c09534f265e 100644 --- a/acceptance/bundle/resources/postgres_projects/update_display_name/out.test.toml +++ b/acceptance/bundle/resources/postgres_projects/update_display_name/out.test.toml @@ -2,5 +2,5 @@ Cloud = true RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/postgres_projects/without_project_id/out.test.toml b/acceptance/bundle/resources/postgres_projects/without_project_id/out.test.toml index 8ce0c4c33e2..c09534f265e 100644 --- a/acceptance/bundle/resources/postgres_projects/without_project_id/out.test.toml +++ b/acceptance/bundle/resources/postgres_projects/without_project_id/out.test.toml @@ -2,5 +2,5 @@ Cloud = true RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/postgres_roles/basic/out.test.toml b/acceptance/bundle/resources/postgres_roles/basic/out.test.toml index 8ce0c4c33e2..c09534f265e 100644 --- a/acceptance/bundle/resources/postgres_roles/basic/out.test.toml +++ b/acceptance/bundle/resources/postgres_roles/basic/out.test.toml @@ -2,5 +2,5 @@ Cloud = true RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/postgres_roles/inherited-role-bind/out.test.toml b/acceptance/bundle/resources/postgres_roles/inherited-role-bind/out.test.toml index 4a55ea95ecf..f1c3b60146c 100644 --- a/acceptance/bundle/resources/postgres_roles/inherited-role-bind/out.test.toml +++ b/acceptance/bundle/resources/postgres_roles/inherited-role-bind/out.test.toml @@ -2,5 +2,5 @@ Cloud = false RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/postgres_roles/inherited-role-conflict/out.test.toml b/acceptance/bundle/resources/postgres_roles/inherited-role-conflict/out.test.toml index 12a96d6c83a..9608aa781f5 100644 --- a/acceptance/bundle/resources/postgres_roles/inherited-role-conflict/out.test.toml +++ b/acceptance/bundle/resources/postgres_roles/inherited-role-conflict/out.test.toml @@ -2,5 +2,5 @@ Cloud = false RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/resources/postgres_roles/inherited-role-conflict/test.toml b/acceptance/bundle/resources/postgres_roles/inherited-role-conflict/test.toml index 18183d0f8f0..156ca740490 100644 --- a/acceptance/bundle/resources/postgres_roles/inherited-role-conflict/test.toml +++ b/acceptance/bundle/resources/postgres_roles/inherited-role-conflict/test.toml @@ -8,4 +8,4 @@ EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] # `bundle unbind` does not yet drop the resource from what the deployment metadata service # reports, so the role staged above still looks tracked. -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/resources/postgres_roles/recreate-postgres-role/out.test.toml b/acceptance/bundle/resources/postgres_roles/recreate-postgres-role/out.test.toml index 3c6ef20a158..5a29113cb28 100644 --- a/acceptance/bundle/resources/postgres_roles/recreate-postgres-role/out.test.toml +++ b/acceptance/bundle/resources/postgres_roles/recreate-postgres-role/out.test.toml @@ -2,5 +2,5 @@ Cloud = true RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/postgres_roles/recreate/out.test.toml b/acceptance/bundle/resources/postgres_roles/recreate/out.test.toml index 8ce0c4c33e2..c09534f265e 100644 --- a/acceptance/bundle/resources/postgres_roles/recreate/out.test.toml +++ b/acceptance/bundle/resources/postgres_roles/recreate/out.test.toml @@ -2,5 +2,5 @@ Cloud = true RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/postgres_roles/replace_existing/out.test.toml b/acceptance/bundle/resources/postgres_roles/replace_existing/out.test.toml index f2f500f9a7d..69be6ba8159 100644 --- a/acceptance/bundle/resources/postgres_roles/replace_existing/out.test.toml +++ b/acceptance/bundle/resources/postgres_roles/replace_existing/out.test.toml @@ -2,5 +2,5 @@ Cloud = true RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/resources/postgres_roles/replace_existing/test.toml b/acceptance/bundle/resources/postgres_roles/replace_existing/test.toml index 2123f2f1544..e7ff603eff2 100644 --- a/acceptance/bundle/resources/postgres_roles/replace_existing/test.toml +++ b/acceptance/bundle/resources/postgres_roles/replace_existing/test.toml @@ -1,3 +1,3 @@ # `bundle unbind` does not yet drop the resource from what the deployment metadata service # reports, so the plan after the unbind still sees the role as tracked. -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/resources/postgres_roles/update/out.test.toml b/acceptance/bundle/resources/postgres_roles/update/out.test.toml index 8ce0c4c33e2..c09534f265e 100644 --- a/acceptance/bundle/resources/postgres_roles/update/out.test.toml +++ b/acceptance/bundle/resources/postgres_roles/update/out.test.toml @@ -2,5 +2,5 @@ Cloud = true RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/postgres_synced_tables/basic/out.test.toml b/acceptance/bundle/resources/postgres_synced_tables/basic/out.test.toml index 8ce0c4c33e2..c09534f265e 100644 --- a/acceptance/bundle/resources/postgres_synced_tables/basic/out.test.toml +++ b/acceptance/bundle/resources/postgres_synced_tables/basic/out.test.toml @@ -2,5 +2,5 @@ Cloud = true RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/postgres_synced_tables/recreate/out.test.toml b/acceptance/bundle/resources/postgres_synced_tables/recreate/out.test.toml index 8ce0c4c33e2..c09534f265e 100644 --- a/acceptance/bundle/resources/postgres_synced_tables/recreate/out.test.toml +++ b/acceptance/bundle/resources/postgres_synced_tables/recreate/out.test.toml @@ -2,5 +2,5 @@ Cloud = true RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/quality_monitors/change_assets_dir/out.test.toml b/acceptance/bundle/resources/quality_monitors/change_assets_dir/out.test.toml index 026bc5f1a2d..3587a9c012d 100644 --- a/acceptance/bundle/resources/quality_monitors/change_assets_dir/out.test.toml +++ b/acceptance/bundle/resources/quality_monitors/change_assets_dir/out.test.toml @@ -1,4 +1,4 @@ Cloud = true RequiresUnityCatalog = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/quality_monitors/change_output_schema_name/out.test.toml b/acceptance/bundle/resources/quality_monitors/change_output_schema_name/out.test.toml index 026bc5f1a2d..3587a9c012d 100644 --- a/acceptance/bundle/resources/quality_monitors/change_output_schema_name/out.test.toml +++ b/acceptance/bundle/resources/quality_monitors/change_output_schema_name/out.test.toml @@ -1,4 +1,4 @@ Cloud = true RequiresUnityCatalog = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/quality_monitors/change_table_name/out.test.toml b/acceptance/bundle/resources/quality_monitors/change_table_name/out.test.toml index 8197bd50f28..6c47aa9e8d1 100644 --- a/acceptance/bundle/resources/quality_monitors/change_table_name/out.test.toml +++ b/acceptance/bundle/resources/quality_monitors/change_table_name/out.test.toml @@ -1,4 +1,4 @@ Cloud = true RequiresUnityCatalog = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/quality_monitors/create/out.test.toml b/acceptance/bundle/resources/quality_monitors/create/out.test.toml index 026bc5f1a2d..3587a9c012d 100644 --- a/acceptance/bundle/resources/quality_monitors/create/out.test.toml +++ b/acceptance/bundle/resources/quality_monitors/create/out.test.toml @@ -1,4 +1,4 @@ Cloud = true RequiresUnityCatalog = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/registered_models/aliases_converge/out.test.toml b/acceptance/bundle/resources/registered_models/aliases_converge/out.test.toml index 51a5602947c..2c6699da193 100644 --- a/acceptance/bundle/resources/registered_models/aliases_converge/out.test.toml +++ b/acceptance/bundle/resources/registered_models/aliases_converge/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/registered_models/basic/out.test.toml b/acceptance/bundle/resources/registered_models/basic/out.test.toml index b481e57b2fa..4401dadbd4e 100644 --- a/acceptance/bundle/resources/registered_models/basic/out.test.toml +++ b/acceptance/bundle/resources/registered_models/basic/out.test.toml @@ -1,5 +1,5 @@ Cloud = true RequiresUnityCatalog = true RunsOnDbr = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/registered_models/drift/browse_only/out.test.toml b/acceptance/bundle/resources/registered_models/drift/browse_only/out.test.toml index 51a5602947c..2c6699da193 100644 --- a/acceptance/bundle/resources/registered_models/drift/browse_only/out.test.toml +++ b/acceptance/bundle/resources/registered_models/drift/browse_only/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/schemas/auto-approve/out.test.toml b/acceptance/bundle/resources/schemas/auto-approve/out.test.toml index b481e57b2fa..4401dadbd4e 100644 --- a/acceptance/bundle/resources/schemas/auto-approve/out.test.toml +++ b/acceptance/bundle/resources/schemas/auto-approve/out.test.toml @@ -1,5 +1,5 @@ Cloud = true RequiresUnityCatalog = true RunsOnDbr = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/schemas/drift/managed_properties/out.test.toml b/acceptance/bundle/resources/schemas/drift/managed_properties/out.test.toml index 51a5602947c..2c6699da193 100644 --- a/acceptance/bundle/resources/schemas/drift/managed_properties/out.test.toml +++ b/acceptance/bundle/resources/schemas/drift/managed_properties/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/schemas/recreate/out.test.toml b/acceptance/bundle/resources/schemas/recreate/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/resources/schemas/recreate/out.test.toml +++ b/acceptance/bundle/resources/schemas/recreate/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/schemas/update/out.test.toml b/acceptance/bundle/resources/schemas/update/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/resources/schemas/update/out.test.toml +++ b/acceptance/bundle/resources/schemas/update/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/secret_scopes/backend-type/out.test.toml b/acceptance/bundle/resources/secret_scopes/backend-type/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/resources/secret_scopes/backend-type/out.test.toml +++ b/acceptance/bundle/resources/secret_scopes/backend-type/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/secret_scopes/basic/out.test.toml b/acceptance/bundle/resources/secret_scopes/basic/out.test.toml index 0fc51ee009b..264de50aa96 100644 --- a/acceptance/bundle/resources/secret_scopes/basic/out.test.toml +++ b/acceptance/bundle/resources/secret_scopes/basic/out.test.toml @@ -1,3 +1,3 @@ Cloud = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/secret_scopes/delete_scope/out.test.toml b/acceptance/bundle/resources/secret_scopes/delete_scope/out.test.toml index 0fc51ee009b..264de50aa96 100644 --- a/acceptance/bundle/resources/secret_scopes/delete_scope/out.test.toml +++ b/acceptance/bundle/resources/secret_scopes/delete_scope/out.test.toml @@ -1,3 +1,3 @@ Cloud = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/secret_scopes/permissions-collapse/out.test.toml b/acceptance/bundle/resources/secret_scopes/permissions-collapse/out.test.toml index 606f603fda8..828b5a6af1e 100644 --- a/acceptance/bundle/resources/secret_scopes/permissions-collapse/out.test.toml +++ b/acceptance/bundle/resources/secret_scopes/permissions-collapse/out.test.toml @@ -1,5 +1,5 @@ Cloud = true RunsOnDbr = true CloudEnvs.gcp = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/secret_scopes/permissions/out.test.toml b/acceptance/bundle/resources/secret_scopes/permissions/out.test.toml index df704765038..e3bca616b46 100644 --- a/acceptance/bundle/resources/secret_scopes/permissions/out.test.toml +++ b/acceptance/bundle/resources/secret_scopes/permissions/out.test.toml @@ -1,5 +1,5 @@ Cloud = true RunsOnDbr = true CloudEnvs.gcp = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/secrets/basic/out.test.toml b/acceptance/bundle/resources/secrets/basic/out.test.toml index 8197bd50f28..6c47aa9e8d1 100644 --- a/acceptance/bundle/resources/secrets/basic/out.test.toml +++ b/acceptance/bundle/resources/secrets/basic/out.test.toml @@ -1,4 +1,4 @@ Cloud = true RequiresUnityCatalog = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/secrets/direct-only/out.test.toml b/acceptance/bundle/resources/secrets/direct-only/out.test.toml index 2563df1863f..33ed6258236 100644 --- a/acceptance/bundle/resources/secrets/direct-only/out.test.toml +++ b/acceptance/bundle/resources/secrets/direct-only/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/secrets/update-value/out.test.toml b/acceptance/bundle/resources/secrets/update-value/out.test.toml index 51a5602947c..2c6699da193 100644 --- a/acceptance/bundle/resources/secrets/update-value/out.test.toml +++ b/acceptance/bundle/resources/secrets/update-value/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/secrets/validate-no-plain-text-default/out.test.toml b/acceptance/bundle/resources/secrets/validate-no-plain-text-default/out.test.toml index 51a5602947c..2c6699da193 100644 --- a/acceptance/bundle/resources/secrets/validate-no-plain-text-default/out.test.toml +++ b/acceptance/bundle/resources/secrets/validate-no-plain-text-default/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/secrets/validate-no-plain-text/out.test.toml b/acceptance/bundle/resources/secrets/validate-no-plain-text/out.test.toml index 51a5602947c..2c6699da193 100644 --- a/acceptance/bundle/resources/secrets/validate-no-plain-text/out.test.toml +++ b/acceptance/bundle/resources/secrets/validate-no-plain-text/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/sql_warehouses/lifecycle-started-edit/out.test.toml b/acceptance/bundle/resources/sql_warehouses/lifecycle-started-edit/out.test.toml index 18e87acca1c..11a76398855 100644 --- a/acceptance/bundle/resources/sql_warehouses/lifecycle-started-edit/out.test.toml +++ b/acceptance/bundle/resources/sql_warehouses/lifecycle-started-edit/out.test.toml @@ -1,4 +1,4 @@ Cloud = true CloudSlow = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/sql_warehouses/lifecycle-started-terraform-error/out.test.toml b/acceptance/bundle/resources/sql_warehouses/lifecycle-started-terraform-error/out.test.toml index c21e4f2b571..035ed14e0e9 100644 --- a/acceptance/bundle/resources/sql_warehouses/lifecycle-started-terraform-error/out.test.toml +++ b/acceptance/bundle/resources/sql_warehouses/lifecycle-started-terraform-error/out.test.toml @@ -1,4 +1,4 @@ Cloud = false CloudSlow = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/sql_warehouses/lifecycle-started-toggle/out.test.toml b/acceptance/bundle/resources/sql_warehouses/lifecycle-started-toggle/out.test.toml index b891255e63e..4ad035d970b 100644 --- a/acceptance/bundle/resources/sql_warehouses/lifecycle-started-toggle/out.test.toml +++ b/acceptance/bundle/resources/sql_warehouses/lifecycle-started-toggle/out.test.toml @@ -1,4 +1,4 @@ Cloud = false CloudSlow = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/sql_warehouses/lifecycle-started/out.test.toml b/acceptance/bundle/resources/sql_warehouses/lifecycle-started/out.test.toml index b891255e63e..4ad035d970b 100644 --- a/acceptance/bundle/resources/sql_warehouses/lifecycle-started/out.test.toml +++ b/acceptance/bundle/resources/sql_warehouses/lifecycle-started/out.test.toml @@ -1,4 +1,4 @@ Cloud = false CloudSlow = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/sql_warehouses/out.test.toml b/acceptance/bundle/resources/sql_warehouses/out.test.toml index 6ebf01d9f53..2ebb8922cab 100644 --- a/acceptance/bundle/resources/sql_warehouses/out.test.toml +++ b/acceptance/bundle/resources/sql_warehouses/out.test.toml @@ -1,4 +1,4 @@ Cloud = false CloudSlow = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/synced_database_tables/basic/out.test.toml b/acceptance/bundle/resources/synced_database_tables/basic/out.test.toml index 94ce6673e5d..798db302b7d 100644 --- a/acceptance/bundle/resources/synced_database_tables/basic/out.test.toml +++ b/acceptance/bundle/resources/synced_database_tables/basic/out.test.toml @@ -2,5 +2,5 @@ Cloud = true RequiresUnityCatalog = true RunsOnDbr = false CloudEnvs.gcp = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/synced_database_tables/recreate/out.test.toml b/acceptance/bundle/resources/synced_database_tables/recreate/out.test.toml index bffdd4c91db..1b77ea7eec0 100644 --- a/acceptance/bundle/resources/synced_database_tables/recreate/out.test.toml +++ b/acceptance/bundle/resources/synced_database_tables/recreate/out.test.toml @@ -1,5 +1,5 @@ Cloud = false RequiresUnityCatalog = true CloudEnvs.gcp = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/vector_search_endpoints/basic/out.test.toml b/acceptance/bundle/resources/vector_search_endpoints/basic/out.test.toml index 8197bd50f28..6c47aa9e8d1 100644 --- a/acceptance/bundle/resources/vector_search_endpoints/basic/out.test.toml +++ b/acceptance/bundle/resources/vector_search_endpoints/basic/out.test.toml @@ -1,4 +1,4 @@ Cloud = true RequiresUnityCatalog = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/vector_search_endpoints/drift/budget_policy/out.test.toml b/acceptance/bundle/resources/vector_search_endpoints/drift/budget_policy/out.test.toml index 3fa8da1cda1..a2e41fad08d 100644 --- a/acceptance/bundle/resources/vector_search_endpoints/drift/budget_policy/out.test.toml +++ b/acceptance/bundle/resources/vector_search_endpoints/drift/budget_policy/out.test.toml @@ -1,4 +1,4 @@ Cloud = false RequiresUnityCatalog = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/vector_search_endpoints/drift/recreated_same_name/out.test.toml b/acceptance/bundle/resources/vector_search_endpoints/drift/recreated_same_name/out.test.toml index 8197bd50f28..6c47aa9e8d1 100644 --- a/acceptance/bundle/resources/vector_search_endpoints/drift/recreated_same_name/out.test.toml +++ b/acceptance/bundle/resources/vector_search_endpoints/drift/recreated_same_name/out.test.toml @@ -1,4 +1,4 @@ Cloud = true RequiresUnityCatalog = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/vector_search_endpoints/drift/target_qps/out.test.toml b/acceptance/bundle/resources/vector_search_endpoints/drift/target_qps/out.test.toml index 3fa8da1cda1..a2e41fad08d 100644 --- a/acceptance/bundle/resources/vector_search_endpoints/drift/target_qps/out.test.toml +++ b/acceptance/bundle/resources/vector_search_endpoints/drift/target_qps/out.test.toml @@ -1,4 +1,4 @@ Cloud = false RequiresUnityCatalog = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/vector_search_endpoints/recreate/create-fails/out.test.toml b/acceptance/bundle/resources/vector_search_endpoints/recreate/create-fails/out.test.toml index 3fa8da1cda1..a2e41fad08d 100644 --- a/acceptance/bundle/resources/vector_search_endpoints/recreate/create-fails/out.test.toml +++ b/acceptance/bundle/resources/vector_search_endpoints/recreate/create-fails/out.test.toml @@ -1,4 +1,4 @@ Cloud = false RequiresUnityCatalog = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/vector_search_endpoints/recreate/endpoint_type/out.test.toml b/acceptance/bundle/resources/vector_search_endpoints/recreate/endpoint_type/out.test.toml index 3fa8da1cda1..a2e41fad08d 100644 --- a/acceptance/bundle/resources/vector_search_endpoints/recreate/endpoint_type/out.test.toml +++ b/acceptance/bundle/resources/vector_search_endpoints/recreate/endpoint_type/out.test.toml @@ -1,4 +1,4 @@ Cloud = false RequiresUnityCatalog = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/vector_search_endpoints/update/budget_policy/out.test.toml b/acceptance/bundle/resources/vector_search_endpoints/update/budget_policy/out.test.toml index 3fa8da1cda1..a2e41fad08d 100644 --- a/acceptance/bundle/resources/vector_search_endpoints/update/budget_policy/out.test.toml +++ b/acceptance/bundle/resources/vector_search_endpoints/update/budget_policy/out.test.toml @@ -1,4 +1,4 @@ Cloud = false RequiresUnityCatalog = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/vector_search_endpoints/update/target_qps/out.test.toml b/acceptance/bundle/resources/vector_search_endpoints/update/target_qps/out.test.toml index 3fa8da1cda1..a2e41fad08d 100644 --- a/acceptance/bundle/resources/vector_search_endpoints/update/target_qps/out.test.toml +++ b/acceptance/bundle/resources/vector_search_endpoints/update/target_qps/out.test.toml @@ -1,4 +1,4 @@ Cloud = false RequiresUnityCatalog = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/vector_search_indexes/basic/out.test.toml b/acceptance/bundle/resources/vector_search_indexes/basic/out.test.toml index a2189ded973..cf858e749e1 100644 --- a/acceptance/bundle/resources/vector_search_indexes/basic/out.test.toml +++ b/acceptance/bundle/resources/vector_search_indexes/basic/out.test.toml @@ -1,5 +1,5 @@ Cloud = true CloudSlow = true RequiresUnityCatalog = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/vector_search_indexes/drift/deleted_remotely/out.test.toml b/acceptance/bundle/resources/vector_search_indexes/drift/deleted_remotely/out.test.toml index cbe4a8c07b4..19a39c76fd4 100644 --- a/acceptance/bundle/resources/vector_search_indexes/drift/deleted_remotely/out.test.toml +++ b/acceptance/bundle/resources/vector_search_indexes/drift/deleted_remotely/out.test.toml @@ -1,5 +1,5 @@ Cloud = false CloudSlow = false RequiresUnityCatalog = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/vector_search_indexes/drift/orphaned_endpoint/out.test.toml b/acceptance/bundle/resources/vector_search_indexes/drift/orphaned_endpoint/out.test.toml index cbe4a8c07b4..19a39c76fd4 100644 --- a/acceptance/bundle/resources/vector_search_indexes/drift/orphaned_endpoint/out.test.toml +++ b/acceptance/bundle/resources/vector_search_indexes/drift/orphaned_endpoint/out.test.toml @@ -1,5 +1,5 @@ Cloud = false CloudSlow = false RequiresUnityCatalog = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/vector_search_indexes/grants/select/out.test.toml b/acceptance/bundle/resources/vector_search_indexes/grants/select/out.test.toml index a2189ded973..cf858e749e1 100644 --- a/acceptance/bundle/resources/vector_search_indexes/grants/select/out.test.toml +++ b/acceptance/bundle/resources/vector_search_indexes/grants/select/out.test.toml @@ -1,5 +1,5 @@ Cloud = true CloudSlow = true RequiresUnityCatalog = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/vector_search_indexes/recreate/embedding_dimension/out.test.toml b/acceptance/bundle/resources/vector_search_indexes/recreate/embedding_dimension/out.test.toml index a2189ded973..cf858e749e1 100644 --- a/acceptance/bundle/resources/vector_search_indexes/recreate/embedding_dimension/out.test.toml +++ b/acceptance/bundle/resources/vector_search_indexes/recreate/embedding_dimension/out.test.toml @@ -1,5 +1,5 @@ Cloud = true CloudSlow = true RequiresUnityCatalog = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/vector_search_indexes/recreate/pending_deletion/out.test.toml b/acceptance/bundle/resources/vector_search_indexes/recreate/pending_deletion/out.test.toml index cbe4a8c07b4..19a39c76fd4 100644 --- a/acceptance/bundle/resources/vector_search_indexes/recreate/pending_deletion/out.test.toml +++ b/acceptance/bundle/resources/vector_search_indexes/recreate/pending_deletion/out.test.toml @@ -1,5 +1,5 @@ Cloud = false CloudSlow = false RequiresUnityCatalog = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/vector_search_indexes/recreate/with_endpoint/out.test.toml b/acceptance/bundle/resources/vector_search_indexes/recreate/with_endpoint/out.test.toml index cbe4a8c07b4..19a39c76fd4 100644 --- a/acceptance/bundle/resources/vector_search_indexes/recreate/with_endpoint/out.test.toml +++ b/acceptance/bundle/resources/vector_search_indexes/recreate/with_endpoint/out.test.toml @@ -1,5 +1,5 @@ Cloud = false CloudSlow = false RequiresUnityCatalog = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/vector_search_indexes/schema_normalization/out.test.toml b/acceptance/bundle/resources/vector_search_indexes/schema_normalization/out.test.toml index a2189ded973..cf858e749e1 100644 --- a/acceptance/bundle/resources/vector_search_indexes/schema_normalization/out.test.toml +++ b/acceptance/bundle/resources/vector_search_indexes/schema_normalization/out.test.toml @@ -1,5 +1,5 @@ Cloud = true CloudSlow = true RequiresUnityCatalog = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/volumes/catalog-var-ref/out.test.toml b/acceptance/bundle/resources/volumes/catalog-var-ref/out.test.toml index 51a5602947c..2c6699da193 100644 --- a/acceptance/bundle/resources/volumes/catalog-var-ref/out.test.toml +++ b/acceptance/bundle/resources/volumes/catalog-var-ref/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/volumes/change-comment/out.test.toml b/acceptance/bundle/resources/volumes/change-comment/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/resources/volumes/change-comment/out.test.toml +++ b/acceptance/bundle/resources/volumes/change-comment/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/volumes/change-name/out.test.toml b/acceptance/bundle/resources/volumes/change-name/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/resources/volumes/change-name/out.test.toml +++ b/acceptance/bundle/resources/volumes/change-name/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/volumes/change-schema-name/out.test.toml b/acceptance/bundle/resources/volumes/change-schema-name/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/resources/volumes/change-schema-name/out.test.toml +++ b/acceptance/bundle/resources/volumes/change-schema-name/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/volumes/recreate/out.test.toml b/acceptance/bundle/resources/volumes/recreate/out.test.toml index b481e57b2fa..4401dadbd4e 100644 --- a/acceptance/bundle/resources/volumes/recreate/out.test.toml +++ b/acceptance/bundle/resources/volumes/recreate/out.test.toml @@ -1,5 +1,5 @@ Cloud = true RequiresUnityCatalog = true RunsOnDbr = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/volumes/remote-change-name/out.test.toml b/acceptance/bundle/resources/volumes/remote-change-name/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/resources/volumes/remote-change-name/out.test.toml +++ b/acceptance/bundle/resources/volumes/remote-change-name/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/volumes/remote-delete/out.test.toml b/acceptance/bundle/resources/volumes/remote-delete/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/resources/volumes/remote-delete/out.test.toml +++ b/acceptance/bundle/resources/volumes/remote-delete/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/volumes/set-storage-location/out.test.toml b/acceptance/bundle/resources/volumes/set-storage-location/out.test.toml index cc2fc7c517f..a3b8091ddab 100644 --- a/acceptance/bundle/resources/volumes/set-storage-location/out.test.toml +++ b/acceptance/bundle/resources/volumes/set-storage-location/out.test.toml @@ -2,5 +2,5 @@ Cloud = true RequiresUnityCatalog = true CloudEnvs.azure = false CloudEnvs.gcp = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/volumes/set-volume-path/out.test.toml b/acceptance/bundle/resources/volumes/set-volume-path/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/resources/volumes/set-volume-path/out.test.toml +++ b/acceptance/bundle/resources/volumes/set-volume-path/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/volumes/uppercase-name/out.test.toml b/acceptance/bundle/resources/volumes/uppercase-name/out.test.toml index 026bc5f1a2d..3587a9c012d 100644 --- a/acceptance/bundle/resources/volumes/uppercase-name/out.test.toml +++ b/acceptance/bundle/resources/volumes/uppercase-name/out.test.toml @@ -1,4 +1,4 @@ Cloud = true RequiresUnityCatalog = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/root/env-not-a-directory/out.test.toml b/acceptance/bundle/root/env-not-a-directory/out.test.toml index 51a5602947c..2c6699da193 100644 --- a/acceptance/bundle/root/env-not-a-directory/out.test.toml +++ b/acceptance/bundle/root/env-not-a-directory/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/root/env-not-found/out.test.toml b/acceptance/bundle/root/env-not-found/out.test.toml index 51a5602947c..2c6699da193 100644 --- a/acceptance/bundle/root/env-not-found/out.test.toml +++ b/acceptance/bundle/root/env-not-found/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/root/not-found/out.test.toml b/acceptance/bundle/root/not-found/out.test.toml index 51a5602947c..2c6699da193 100644 --- a/acceptance/bundle/root/not-found/out.test.toml +++ b/acceptance/bundle/root/not-found/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/root/real-empty-dir/out.test.toml b/acceptance/bundle/root/real-empty-dir/out.test.toml index 51a5602947c..2c6699da193 100644 --- a/acceptance/bundle/root/real-empty-dir/out.test.toml +++ b/acceptance/bundle/root/real-empty-dir/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/run/app-with-job/out.test.toml b/acceptance/bundle/run/app-with-job/out.test.toml index ef4c598cd59..d1a88727b97 100644 --- a/acceptance/bundle/run/app-with-job/out.test.toml +++ b/acceptance/bundle/run/app-with-job/out.test.toml @@ -1,4 +1,4 @@ Cloud = true CloudSlow = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/run/basic/out.test.toml b/acceptance/bundle/run/basic/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/run/basic/out.test.toml +++ b/acceptance/bundle/run/basic/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/run/diagnostics/out.test.toml b/acceptance/bundle/run/diagnostics/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/run/diagnostics/out.test.toml +++ b/acceptance/bundle/run/diagnostics/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/run/inline-script/basic/out.test.toml b/acceptance/bundle/run/inline-script/basic/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/run/inline-script/basic/out.test.toml +++ b/acceptance/bundle/run/inline-script/basic/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/run/inline-script/cwd/out.test.toml b/acceptance/bundle/run/inline-script/cwd/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/run/inline-script/cwd/out.test.toml +++ b/acceptance/bundle/run/inline-script/cwd/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/run/inline-script/databricks-cli/profile-is-passed/from_flag/out.test.toml b/acceptance/bundle/run/inline-script/databricks-cli/profile-is-passed/from_flag/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/run/inline-script/databricks-cli/profile-is-passed/from_flag/out.test.toml +++ b/acceptance/bundle/run/inline-script/databricks-cli/profile-is-passed/from_flag/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/run/inline-script/databricks-cli/target-is-passed/default/out.test.toml b/acceptance/bundle/run/inline-script/databricks-cli/target-is-passed/default/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/run/inline-script/databricks-cli/target-is-passed/default/out.test.toml +++ b/acceptance/bundle/run/inline-script/databricks-cli/target-is-passed/default/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/run/inline-script/databricks-cli/target-is-passed/from_flag/out.test.toml b/acceptance/bundle/run/inline-script/databricks-cli/target-is-passed/from_flag/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/run/inline-script/databricks-cli/target-is-passed/from_flag/out.test.toml +++ b/acceptance/bundle/run/inline-script/databricks-cli/target-is-passed/from_flag/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/run/inline-script/no-auth/out.test.toml b/acceptance/bundle/run/inline-script/no-auth/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/run/inline-script/no-auth/out.test.toml +++ b/acceptance/bundle/run/inline-script/no-auth/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/run/inline-script/no-bundle/out.test.toml b/acceptance/bundle/run/inline-script/no-bundle/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/run/inline-script/no-bundle/out.test.toml +++ b/acceptance/bundle/run/inline-script/no-bundle/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/run/inline-script/no-separator/out.test.toml b/acceptance/bundle/run/inline-script/no-separator/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/run/inline-script/no-separator/out.test.toml +++ b/acceptance/bundle/run/inline-script/no-separator/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/run/jobs/partial_run/out.test.toml b/acceptance/bundle/run/jobs/partial_run/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/run/jobs/partial_run/out.test.toml +++ b/acceptance/bundle/run/jobs/partial_run/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/run/no-state/out.test.toml b/acceptance/bundle/run/no-state/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/run/no-state/out.test.toml +++ b/acceptance/bundle/run/no-state/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/run/refresh-flags/out.test.toml b/acceptance/bundle/run/refresh-flags/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/run/refresh-flags/out.test.toml +++ b/acceptance/bundle/run/refresh-flags/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/run/scripts/basic/out.test.toml b/acceptance/bundle/run/scripts/basic/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/run/scripts/basic/out.test.toml +++ b/acceptance/bundle/run/scripts/basic/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/run/scripts/cwd/out.test.toml b/acceptance/bundle/run/scripts/cwd/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/run/scripts/cwd/out.test.toml +++ b/acceptance/bundle/run/scripts/cwd/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/run/scripts/databricks-cli/profile-is-passed/from_flag/out.test.toml b/acceptance/bundle/run/scripts/databricks-cli/profile-is-passed/from_flag/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/run/scripts/databricks-cli/profile-is-passed/from_flag/out.test.toml +++ b/acceptance/bundle/run/scripts/databricks-cli/profile-is-passed/from_flag/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/run/scripts/databricks-cli/target-is-passed/default/out.test.toml b/acceptance/bundle/run/scripts/databricks-cli/target-is-passed/default/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/run/scripts/databricks-cli/target-is-passed/default/out.test.toml +++ b/acceptance/bundle/run/scripts/databricks-cli/target-is-passed/default/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/run/scripts/databricks-cli/target-is-passed/from_flag/out.test.toml b/acceptance/bundle/run/scripts/databricks-cli/target-is-passed/from_flag/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/run/scripts/databricks-cli/target-is-passed/from_flag/out.test.toml +++ b/acceptance/bundle/run/scripts/databricks-cli/target-is-passed/from_flag/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/run/scripts/env-bad-prefix/out.test.toml b/acceptance/bundle/run/scripts/env-bad-prefix/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/run/scripts/env-bad-prefix/out.test.toml +++ b/acceptance/bundle/run/scripts/env-bad-prefix/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/run/scripts/env-precedence/out.test.toml b/acceptance/bundle/run/scripts/env-precedence/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/run/scripts/env-precedence/out.test.toml +++ b/acceptance/bundle/run/scripts/env-precedence/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/run/scripts/env-section/out.test.toml b/acceptance/bundle/run/scripts/env-section/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/run/scripts/env-section/out.test.toml +++ b/acceptance/bundle/run/scripts/env-section/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/run/scripts/exit_code/out.test.toml b/acceptance/bundle/run/scripts/exit_code/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/run/scripts/exit_code/out.test.toml +++ b/acceptance/bundle/run/scripts/exit_code/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/run/scripts/io/out.test.toml b/acceptance/bundle/run/scripts/io/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/run/scripts/io/out.test.toml +++ b/acceptance/bundle/run/scripts/io/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/run/scripts/no-auth/out.test.toml b/acceptance/bundle/run/scripts/no-auth/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/run/scripts/no-auth/out.test.toml +++ b/acceptance/bundle/run/scripts/no-auth/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/run/scripts/no-interpolation/out.test.toml b/acceptance/bundle/run/scripts/no-interpolation/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/run/scripts/no-interpolation/out.test.toml +++ b/acceptance/bundle/run/scripts/no-interpolation/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/run/scripts/no_content/out.test.toml b/acceptance/bundle/run/scripts/no_content/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/run/scripts/no_content/out.test.toml +++ b/acceptance/bundle/run/scripts/no_content/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/run/scripts/shell/envvar/out.test.toml b/acceptance/bundle/run/scripts/shell/envvar/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/run/scripts/shell/envvar/out.test.toml +++ b/acceptance/bundle/run/scripts/shell/envvar/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/run/scripts/shell/math/out.test.toml b/acceptance/bundle/run/scripts/shell/math/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/run/scripts/shell/math/out.test.toml +++ b/acceptance/bundle/run/scripts/shell/math/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/run/scripts/unique_keys/duplicate_resource_and_script_name_root/out.test.toml b/acceptance/bundle/run/scripts/unique_keys/duplicate_resource_and_script_name_root/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/run/scripts/unique_keys/duplicate_resource_and_script_name_root/out.test.toml +++ b/acceptance/bundle/run/scripts/unique_keys/duplicate_resource_and_script_name_root/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/run/scripts/unique_keys/duplicate_resource_and_script_subconfigurations/out.test.toml b/acceptance/bundle/run/scripts/unique_keys/duplicate_resource_and_script_subconfigurations/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/run/scripts/unique_keys/duplicate_resource_and_script_subconfigurations/out.test.toml +++ b/acceptance/bundle/run/scripts/unique_keys/duplicate_resource_and_script_subconfigurations/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/run/scripts/unique_keys/duplicate_script_names_in_subconfiguration/out.test.toml b/acceptance/bundle/run/scripts/unique_keys/duplicate_script_names_in_subconfiguration/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/run/scripts/unique_keys/duplicate_script_names_in_subconfiguration/out.test.toml +++ b/acceptance/bundle/run/scripts/unique_keys/duplicate_script_names_in_subconfiguration/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/run/state-wiped/out.test.toml b/acceptance/bundle/run/state-wiped/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/run/state-wiped/out.test.toml +++ b/acceptance/bundle/run/state-wiped/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/run_as/allowed/regular_user/out.test.toml b/acceptance/bundle/run_as/allowed/regular_user/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/run_as/allowed/regular_user/out.test.toml +++ b/acceptance/bundle/run_as/allowed/regular_user/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/run_as/allowed/service_principal/out.test.toml b/acceptance/bundle/run_as/allowed/service_principal/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/run_as/allowed/service_principal/out.test.toml +++ b/acceptance/bundle/run_as/allowed/service_principal/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/run_as/dashboard_embed/out.test.toml b/acceptance/bundle/run_as/dashboard_embed/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/run_as/dashboard_embed/out.test.toml +++ b/acceptance/bundle/run_as/dashboard_embed/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/run_as/empty_override/out.test.toml b/acceptance/bundle/run_as/empty_override/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/run_as/empty_override/out.test.toml +++ b/acceptance/bundle/run_as/empty_override/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/run_as/empty_run_as/out.test.toml b/acceptance/bundle/run_as/empty_run_as/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/run_as/empty_run_as/out.test.toml +++ b/acceptance/bundle/run_as/empty_run_as/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/run_as/empty_run_as_dict/out.test.toml b/acceptance/bundle/run_as/empty_run_as_dict/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/run_as/empty_run_as_dict/out.test.toml +++ b/acceptance/bundle/run_as/empty_run_as_dict/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/run_as/empty_sp/out.test.toml b/acceptance/bundle/run_as/empty_sp/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/run_as/empty_sp/out.test.toml +++ b/acceptance/bundle/run_as/empty_sp/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/run_as/empty_user/out.test.toml b/acceptance/bundle/run_as/empty_user/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/run_as/empty_user/out.test.toml +++ b/acceptance/bundle/run_as/empty_user/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/run_as/empty_user_and_sp/out.test.toml b/acceptance/bundle/run_as/empty_user_and_sp/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/run_as/empty_user_and_sp/out.test.toml +++ b/acceptance/bundle/run_as/empty_user_and_sp/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/run_as/invalid_both_sp_and_user/out.test.toml b/acceptance/bundle/run_as/invalid_both_sp_and_user/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/run_as/invalid_both_sp_and_user/out.test.toml +++ b/acceptance/bundle/run_as/invalid_both_sp_and_user/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/run_as/job_default/out.test.toml b/acceptance/bundle/run_as/job_default/out.test.toml index a40e88fc275..89d861bf4d0 100644 --- a/acceptance/bundle/run_as/job_default/out.test.toml +++ b/acceptance/bundle/run_as/job_default/out.test.toml @@ -1,3 +1,3 @@ Cloud = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/run_as/model_serving_different/out.test.toml b/acceptance/bundle/run_as/model_serving_different/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/run_as/model_serving_different/out.test.toml +++ b/acceptance/bundle/run_as/model_serving_different/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/run_as/model_serving_matching/out.test.toml b/acceptance/bundle/run_as/model_serving_matching/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/run_as/model_serving_matching/out.test.toml +++ b/acceptance/bundle/run_as/model_serving_matching/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/run_as/out.test.toml b/acceptance/bundle/run_as/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/run_as/out.test.toml +++ b/acceptance/bundle/run_as/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/run_as/pipelines/regular_user/out.test.toml b/acceptance/bundle/run_as/pipelines/regular_user/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/run_as/pipelines/regular_user/out.test.toml +++ b/acceptance/bundle/run_as/pipelines/regular_user/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/run_as/pipelines/service_principal/out.test.toml b/acceptance/bundle/run_as/pipelines/service_principal/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/run_as/pipelines/service_principal/out.test.toml +++ b/acceptance/bundle/run_as/pipelines/service_principal/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/run_as/pipelines_legacy/out.test.toml b/acceptance/bundle/run_as/pipelines_legacy/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/run_as/pipelines_legacy/out.test.toml +++ b/acceptance/bundle/run_as/pipelines_legacy/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/script.prepare b/acceptance/bundle/script.prepare index 48fbafac777..6ed8c813dff 100644 --- a/acceptance/bundle/script.prepare +++ b/acceptance/bundle/script.prepare @@ -3,7 +3,8 @@ nostamp() { # # Deployment history recording adds deployment_id and version_id to every job and # pipeline. Acceptance tests compare output byte for byte, so those two extra fields - # would fail every test in the DATABRICKS_BUNDLE_DMS=true run (see bundle/test.toml). + # would fail every test in the DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY=true run + # (see bundle/test.toml). # Pipe a plan, a state dump, or a resource payload through this and the test asserts one # golden file either way. Tests under bundle/dms assert the stamp itself and must not. # diff --git a/acceptance/bundle/scripts/no-trailing-newline/out.test.toml b/acceptance/bundle/scripts/no-trailing-newline/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/scripts/no-trailing-newline/out.test.toml +++ b/acceptance/bundle/scripts/no-trailing-newline/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/scripts/out.test.toml b/acceptance/bundle/scripts/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/scripts/out.test.toml +++ b/acceptance/bundle/scripts/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/scripts/restricted-execution/out.test.toml b/acceptance/bundle/scripts/restricted-execution/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/scripts/restricted-execution/out.test.toml +++ b/acceptance/bundle/scripts/restricted-execution/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/select/ambiguous/out.test.toml b/acceptance/bundle/select/ambiguous/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/select/ambiguous/out.test.toml +++ b/acceptance/bundle/select/ambiguous/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/select/basic/out.test.toml b/acceptance/bundle/select/basic/out.test.toml index 50c14b9ec78..336c1651c83 100644 --- a/acceptance/bundle/select/basic/out.test.toml +++ b/acceptance/bundle/select/basic/out.test.toml @@ -1,4 +1,4 @@ Cloud = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/select/grants_permissions/out.test.toml b/acceptance/bundle/select/grants_permissions/out.test.toml index 01bce62fbf2..5f30914da96 100644 --- a/acceptance/bundle/select/grants_permissions/out.test.toml +++ b/acceptance/bundle/select/grants_permissions/out.test.toml @@ -1,5 +1,5 @@ Cloud = false RequiresUnityCatalog = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/select/missing/out.test.toml b/acceptance/bundle/select/missing/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/select/missing/out.test.toml +++ b/acceptance/bundle/select/missing/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/select/rejected/out.test.toml b/acceptance/bundle/select/rejected/out.test.toml index 2563df1863f..33ed6258236 100644 --- a/acceptance/bundle/select/rejected/out.test.toml +++ b/acceptance/bundle/select/rejected/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/state/bad_env/out.test.toml b/acceptance/bundle/state/bad_env/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/state/bad_env/out.test.toml +++ b/acceptance/bundle/state/bad_env/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/state/bad_json_local/out.test.toml b/acceptance/bundle/state/bad_json_local/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/state/bad_json_local/out.test.toml +++ b/acceptance/bundle/state/bad_json_local/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/state/basic/out.test.toml b/acceptance/bundle/state/basic/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/state/basic/out.test.toml +++ b/acceptance/bundle/state/basic/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/state/engine_default/out.test.toml b/acceptance/bundle/state/engine_default/out.test.toml index 51a5602947c..2c6699da193 100644 --- a/acceptance/bundle/state/engine_default/out.test.toml +++ b/acceptance/bundle/state/engine_default/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/state/engine_mismatch/out.test.toml b/acceptance/bundle/state/engine_mismatch/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/state/engine_mismatch/out.test.toml +++ b/acceptance/bundle/state/engine_mismatch/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/state/feature_flags/out.test.toml b/acceptance/bundle/state/feature_flags/out.test.toml index 51a5602947c..2c6699da193 100644 --- a/acceptance/bundle/state/feature_flags/out.test.toml +++ b/acceptance/bundle/state/feature_flags/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/state/force_pull_commands/out.test.toml b/acceptance/bundle/state/force_pull_commands/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/state/force_pull_commands/out.test.toml +++ b/acceptance/bundle/state/force_pull_commands/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/state/future_version/out.test.toml b/acceptance/bundle/state/future_version/out.test.toml index 51a5602947c..2c6699da193 100644 --- a/acceptance/bundle/state/future_version/out.test.toml +++ b/acceptance/bundle/state/future_version/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/state/lineage_different/out.test.toml b/acceptance/bundle/state/lineage_different/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/state/lineage_different/out.test.toml +++ b/acceptance/bundle/state/lineage_different/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/state/permission_level_migration/out.test.toml b/acceptance/bundle/state/permission_level_migration/out.test.toml index 5ab60815a3b..7daaf6fd56a 100644 --- a/acceptance/bundle/state/permission_level_migration/out.test.toml +++ b/acceptance/bundle/state/permission_level_migration/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/state/permission_level_migration/test.toml b/acceptance/bundle/state/permission_level_migration/test.toml index 4adec0e8bb5..b788094ae18 100644 --- a/acceptance/bundle/state/permission_level_migration/test.toml +++ b/acceptance/bundle/state/permission_level_migration/test.toml @@ -1,7 +1,7 @@ # Recording needs a bundle it has seen from the start. This test seeds a state file, # so recording refuses it. TODO(DMS): drop this once existing state can be # handed over to the service (see the TODO in dstate.Open). -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] Ignore = [".databricks"] diff --git a/acceptance/bundle/state/same_serial/out.test.toml b/acceptance/bundle/state/same_serial/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/state/same_serial/out.test.toml +++ b/acceptance/bundle/state/same_serial/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/state/state_present/out.test.toml b/acceptance/bundle/state/state_present/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/state/state_present/out.test.toml +++ b/acceptance/bundle/state/state_present/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/summary/missing-libraries-file-path/out.test.toml b/acceptance/bundle/summary/missing-libraries-file-path/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/summary/missing-libraries-file-path/out.test.toml +++ b/acceptance/bundle/summary/missing-libraries-file-path/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/summary/modified_status/out.test.toml b/acceptance/bundle/summary/modified_status/out.test.toml index 17a5ef9eee0..7a5286dd22b 100644 --- a/acceptance/bundle/summary/modified_status/out.test.toml +++ b/acceptance/bundle/summary/modified_status/out.test.toml @@ -1,4 +1,4 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] EnvMatrix.VARIANT = ["empty_resources.yml", "no_resources.yml"] diff --git a/acceptance/bundle/sync/dryrun/out.test.toml b/acceptance/bundle/sync/dryrun/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/sync/dryrun/out.test.toml +++ b/acceptance/bundle/sync/dryrun/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/sync/out.test.toml b/acceptance/bundle/sync/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/sync/out.test.toml +++ b/acceptance/bundle/sync/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/syncroot/dotdot-git/out.test.toml b/acceptance/bundle/syncroot/dotdot-git/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/syncroot/dotdot-git/out.test.toml +++ b/acceptance/bundle/syncroot/dotdot-git/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/syncroot/dotdot-nogit/out.test.toml b/acceptance/bundle/syncroot/dotdot-nogit/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/syncroot/dotdot-nogit/out.test.toml +++ b/acceptance/bundle/syncroot/dotdot-nogit/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/telemetry/config-remote-sync-error/out.test.toml b/acceptance/bundle/telemetry/config-remote-sync-error/out.test.toml index 89404df951f..86a51691f28 100644 --- a/acceptance/bundle/telemetry/config-remote-sync-error/out.test.toml +++ b/acceptance/bundle/telemetry/config-remote-sync-error/out.test.toml @@ -1,4 +1,4 @@ Cloud = false GOOS.windows = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/telemetry/config-remote-sync-recreate/out.test.toml b/acceptance/bundle/telemetry/config-remote-sync-recreate/out.test.toml index 0007abb49a1..58b3530c41c 100644 --- a/acceptance/bundle/telemetry/config-remote-sync-recreate/out.test.toml +++ b/acceptance/bundle/telemetry/config-remote-sync-recreate/out.test.toml @@ -1,4 +1,4 @@ Cloud = false GOOS.windows = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/telemetry/config-remote-sync-save/out.test.toml b/acceptance/bundle/telemetry/config-remote-sync-save/out.test.toml index 0007abb49a1..58b3530c41c 100644 --- a/acceptance/bundle/telemetry/config-remote-sync-save/out.test.toml +++ b/acceptance/bundle/telemetry/config-remote-sync-save/out.test.toml @@ -1,4 +1,4 @@ Cloud = false GOOS.windows = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/telemetry/config-remote-sync/out.test.toml b/acceptance/bundle/telemetry/config-remote-sync/out.test.toml index 0007abb49a1..58b3530c41c 100644 --- a/acceptance/bundle/telemetry/config-remote-sync/out.test.toml +++ b/acceptance/bundle/telemetry/config-remote-sync/out.test.toml @@ -1,4 +1,4 @@ Cloud = false GOOS.windows = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/telemetry/deploy-app-lifecycle-started/out.test.toml b/acceptance/bundle/telemetry/deploy-app-lifecycle-started/out.test.toml index 5ab60815a3b..7daaf6fd56a 100644 --- a/acceptance/bundle/telemetry/deploy-app-lifecycle-started/out.test.toml +++ b/acceptance/bundle/telemetry/deploy-app-lifecycle-started/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/telemetry/deploy-artifact-path-type/out.test.toml b/acceptance/bundle/telemetry/deploy-artifact-path-type/out.test.toml index 412d37c2f86..f61c2bccd55 100644 --- a/acceptance/bundle/telemetry/deploy-artifact-path-type/out.test.toml +++ b/acceptance/bundle/telemetry/deploy-artifact-path-type/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/telemetry/deploy-artifacts-variables/out.test.toml b/acceptance/bundle/telemetry/deploy-artifacts-variables/out.test.toml index 412d37c2f86..f61c2bccd55 100644 --- a/acceptance/bundle/telemetry/deploy-artifacts-variables/out.test.toml +++ b/acceptance/bundle/telemetry/deploy-artifacts-variables/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/telemetry/deploy-compute-type/out.test.toml b/acceptance/bundle/telemetry/deploy-compute-type/out.test.toml index 412d37c2f86..f61c2bccd55 100644 --- a/acceptance/bundle/telemetry/deploy-compute-type/out.test.toml +++ b/acceptance/bundle/telemetry/deploy-compute-type/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/telemetry/deploy-config-file-count/out.test.toml b/acceptance/bundle/telemetry/deploy-config-file-count/out.test.toml index 412d37c2f86..f61c2bccd55 100644 --- a/acceptance/bundle/telemetry/deploy-config-file-count/out.test.toml +++ b/acceptance/bundle/telemetry/deploy-config-file-count/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/telemetry/deploy-error-message/out.test.toml b/acceptance/bundle/telemetry/deploy-error-message/out.test.toml index 412d37c2f86..f61c2bccd55 100644 --- a/acceptance/bundle/telemetry/deploy-error-message/out.test.toml +++ b/acceptance/bundle/telemetry/deploy-error-message/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/telemetry/deploy-error/out.test.toml b/acceptance/bundle/telemetry/deploy-error/out.test.toml index 412d37c2f86..f61c2bccd55 100644 --- a/acceptance/bundle/telemetry/deploy-error/out.test.toml +++ b/acceptance/bundle/telemetry/deploy-error/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/telemetry/deploy-experimental/out.test.toml b/acceptance/bundle/telemetry/deploy-experimental/out.test.toml index 412d37c2f86..f61c2bccd55 100644 --- a/acceptance/bundle/telemetry/deploy-experimental/out.test.toml +++ b/acceptance/bundle/telemetry/deploy-experimental/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/telemetry/deploy-mode/out.test.toml b/acceptance/bundle/telemetry/deploy-mode/out.test.toml index 412d37c2f86..f61c2bccd55 100644 --- a/acceptance/bundle/telemetry/deploy-mode/out.test.toml +++ b/acceptance/bundle/telemetry/deploy-mode/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/telemetry/deploy-name-prefix/custom/out.test.toml b/acceptance/bundle/telemetry/deploy-name-prefix/custom/out.test.toml index 412d37c2f86..f61c2bccd55 100644 --- a/acceptance/bundle/telemetry/deploy-name-prefix/custom/out.test.toml +++ b/acceptance/bundle/telemetry/deploy-name-prefix/custom/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/telemetry/deploy-name-prefix/mode-development/out.test.toml b/acceptance/bundle/telemetry/deploy-name-prefix/mode-development/out.test.toml index 412d37c2f86..f61c2bccd55 100644 --- a/acceptance/bundle/telemetry/deploy-name-prefix/mode-development/out.test.toml +++ b/acceptance/bundle/telemetry/deploy-name-prefix/mode-development/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/telemetry/deploy-no-uuid/out.test.toml b/acceptance/bundle/telemetry/deploy-no-uuid/out.test.toml index 412d37c2f86..f61c2bccd55 100644 --- a/acceptance/bundle/telemetry/deploy-no-uuid/out.test.toml +++ b/acceptance/bundle/telemetry/deploy-no-uuid/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/telemetry/deploy-run-as/out.test.toml b/acceptance/bundle/telemetry/deploy-run-as/out.test.toml index 412d37c2f86..f61c2bccd55 100644 --- a/acceptance/bundle/telemetry/deploy-run-as/out.test.toml +++ b/acceptance/bundle/telemetry/deploy-run-as/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/telemetry/deploy-target-count/out.test.toml b/acceptance/bundle/telemetry/deploy-target-count/out.test.toml index 412d37c2f86..f61c2bccd55 100644 --- a/acceptance/bundle/telemetry/deploy-target-count/out.test.toml +++ b/acceptance/bundle/telemetry/deploy-target-count/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/telemetry/deploy-variable-count/out.test.toml b/acceptance/bundle/telemetry/deploy-variable-count/out.test.toml index 412d37c2f86..f61c2bccd55 100644 --- a/acceptance/bundle/telemetry/deploy-variable-count/out.test.toml +++ b/acceptance/bundle/telemetry/deploy-variable-count/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/telemetry/deploy-whl-artifacts/out.test.toml b/acceptance/bundle/telemetry/deploy-whl-artifacts/out.test.toml index 412d37c2f86..f61c2bccd55 100644 --- a/acceptance/bundle/telemetry/deploy-whl-artifacts/out.test.toml +++ b/acceptance/bundle/telemetry/deploy-whl-artifacts/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/telemetry/deploy-workspace-folder-permissions/out.test.toml b/acceptance/bundle/telemetry/deploy-workspace-folder-permissions/out.test.toml index 412d37c2f86..f61c2bccd55 100644 --- a/acceptance/bundle/telemetry/deploy-workspace-folder-permissions/out.test.toml +++ b/acceptance/bundle/telemetry/deploy-workspace-folder-permissions/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/telemetry/deploy/out.test.toml b/acceptance/bundle/telemetry/deploy/out.test.toml index 412d37c2f86..f61c2bccd55 100644 --- a/acceptance/bundle/telemetry/deploy/out.test.toml +++ b/acceptance/bundle/telemetry/deploy/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/telemetry/test.toml b/acceptance/bundle/telemetry/test.toml index 92804cc8ed9..0c7ae57f796 100644 --- a/acceptance/bundle/telemetry/test.toml +++ b/acceptance/bundle/telemetry/test.toml @@ -3,7 +3,7 @@ IncludeRequestHeaders = ["User-Agent"] # Telemetry reports the byte size of the serialized state, which the deployment stamp # legitimately grows. The number is the assertion here, so there is nothing to normalize. -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] [Env] DATABRICKS_CACHE_ENABLED = 'false' diff --git a/acceptance/bundle/templates-machinery/helper_upper_lower/out.test.toml b/acceptance/bundle/templates-machinery/helper_upper_lower/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/templates-machinery/helper_upper_lower/out.test.toml +++ b/acceptance/bundle/templates-machinery/helper_upper_lower/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/templates-machinery/helper_username/out.test.toml b/acceptance/bundle/templates-machinery/helper_username/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/templates-machinery/helper_username/out.test.toml +++ b/acceptance/bundle/templates-machinery/helper_username/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/templates-machinery/helpers-error/out.test.toml b/acceptance/bundle/templates-machinery/helpers-error/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/templates-machinery/helpers-error/out.test.toml +++ b/acceptance/bundle/templates-machinery/helpers-error/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/templates-machinery/number-precision/out.test.toml b/acceptance/bundle/templates-machinery/number-precision/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/templates-machinery/number-precision/out.test.toml +++ b/acceptance/bundle/templates-machinery/number-precision/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/templates-machinery/supported-url/out.test.toml b/acceptance/bundle/templates-machinery/supported-url/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/templates-machinery/supported-url/out.test.toml +++ b/acceptance/bundle/templates-machinery/supported-url/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/templates-machinery/unsupported-url/out.test.toml b/acceptance/bundle/templates-machinery/unsupported-url/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/templates-machinery/unsupported-url/out.test.toml +++ b/acceptance/bundle/templates-machinery/unsupported-url/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/templates-machinery/wrong-path/out.test.toml b/acceptance/bundle/templates-machinery/wrong-path/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/templates-machinery/wrong-path/out.test.toml +++ b/acceptance/bundle/templates-machinery/wrong-path/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/templates-machinery/wrong-url/out.test.toml b/acceptance/bundle/templates-machinery/wrong-url/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/templates-machinery/wrong-url/out.test.toml +++ b/acceptance/bundle/templates-machinery/wrong-url/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/templates/dbt-sql/out.test.toml b/acceptance/bundle/templates/dbt-sql/out.test.toml index 412d37c2f86..f61c2bccd55 100644 --- a/acceptance/bundle/templates/dbt-sql/out.test.toml +++ b/acceptance/bundle/templates/dbt-sql/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/templates/default-minimal/python/out.test.toml b/acceptance/bundle/templates/default-minimal/python/out.test.toml index 412d37c2f86..f61c2bccd55 100644 --- a/acceptance/bundle/templates/default-minimal/python/out.test.toml +++ b/acceptance/bundle/templates/default-minimal/python/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/templates/default-minimal/skip/out.test.toml b/acceptance/bundle/templates/default-minimal/skip/out.test.toml index 412d37c2f86..f61c2bccd55 100644 --- a/acceptance/bundle/templates/default-minimal/skip/out.test.toml +++ b/acceptance/bundle/templates/default-minimal/skip/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/templates/default-minimal/sql/out.test.toml b/acceptance/bundle/templates/default-minimal/sql/out.test.toml index 412d37c2f86..f61c2bccd55 100644 --- a/acceptance/bundle/templates/default-minimal/sql/out.test.toml +++ b/acceptance/bundle/templates/default-minimal/sql/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/templates/default-python/azure-government/out.test.toml b/acceptance/bundle/templates/default-python/azure-government/out.test.toml index 412d37c2f86..f61c2bccd55 100644 --- a/acceptance/bundle/templates/default-python/azure-government/out.test.toml +++ b/acceptance/bundle/templates/default-python/azure-government/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/templates/default-python/classic/out.test.toml b/acceptance/bundle/templates/default-python/classic/out.test.toml index 79d4a502cca..dde56cefc7f 100644 --- a/acceptance/bundle/templates/default-python/classic/out.test.toml +++ b/acceptance/bundle/templates/default-python/classic/out.test.toml @@ -1,5 +1,5 @@ Cloud = false Phase = 1 -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/templates/default-python/combinations/classic/out.test.toml b/acceptance/bundle/templates/default-python/combinations/classic/out.test.toml index 18be9cf4d20..2dc2763dfa6 100644 --- a/acceptance/bundle/templates/default-python/combinations/classic/out.test.toml +++ b/acceptance/bundle/templates/default-python/combinations/classic/out.test.toml @@ -1,6 +1,6 @@ Cloud = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] EnvMatrix.DLT = ["yes", "no"] EnvMatrix.NBOOK = ["yes", "no"] EnvMatrix.PY = ["yes", "no"] diff --git a/acceptance/bundle/templates/default-python/combinations/serverless/out.test.toml b/acceptance/bundle/templates/default-python/combinations/serverless/out.test.toml index 18be9cf4d20..2dc2763dfa6 100644 --- a/acceptance/bundle/templates/default-python/combinations/serverless/out.test.toml +++ b/acceptance/bundle/templates/default-python/combinations/serverless/out.test.toml @@ -1,6 +1,6 @@ Cloud = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] EnvMatrix.DLT = ["yes", "no"] EnvMatrix.NBOOK = ["yes", "no"] EnvMatrix.PY = ["yes", "no"] diff --git a/acceptance/bundle/templates/default-python/fail-missing-uv/out.test.toml b/acceptance/bundle/templates/default-python/fail-missing-uv/out.test.toml index 412d37c2f86..f61c2bccd55 100644 --- a/acceptance/bundle/templates/default-python/fail-missing-uv/out.test.toml +++ b/acceptance/bundle/templates/default-python/fail-missing-uv/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/templates/default-python/integration_classic/out.test.toml b/acceptance/bundle/templates/default-python/integration_classic/out.test.toml index 46f8c644fb9..a3246c58429 100644 --- a/acceptance/bundle/templates/default-python/integration_classic/out.test.toml +++ b/acceptance/bundle/templates/default-python/integration_classic/out.test.toml @@ -1,6 +1,6 @@ Cloud = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] EnvMatrix.UV_PYTHON = [ "3.9", "3.10", diff --git a/acceptance/bundle/templates/default-python/no-uc/out.test.toml b/acceptance/bundle/templates/default-python/no-uc/out.test.toml index 412d37c2f86..f61c2bccd55 100644 --- a/acceptance/bundle/templates/default-python/no-uc/out.test.toml +++ b/acceptance/bundle/templates/default-python/no-uc/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/templates/default-python/serverless-customcatalog/out.test.toml b/acceptance/bundle/templates/default-python/serverless-customcatalog/out.test.toml index 8f26ec1671a..4cf738bfea6 100644 --- a/acceptance/bundle/templates/default-python/serverless-customcatalog/out.test.toml +++ b/acceptance/bundle/templates/default-python/serverless-customcatalog/out.test.toml @@ -1,4 +1,4 @@ Cloud = false Phase = 1 -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/templates/default-python/serverless/out.test.toml b/acceptance/bundle/templates/default-python/serverless/out.test.toml index 412d37c2f86..f61c2bccd55 100644 --- a/acceptance/bundle/templates/default-python/serverless/out.test.toml +++ b/acceptance/bundle/templates/default-python/serverless/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/templates/default-scala/out.test.toml b/acceptance/bundle/templates/default-scala/out.test.toml index 412d37c2f86..f61c2bccd55 100644 --- a/acceptance/bundle/templates/default-scala/out.test.toml +++ b/acceptance/bundle/templates/default-scala/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/templates/default-sql/out.test.toml b/acceptance/bundle/templates/default-sql/out.test.toml index 412d37c2f86..f61c2bccd55 100644 --- a/acceptance/bundle/templates/default-sql/out.test.toml +++ b/acceptance/bundle/templates/default-sql/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/templates/lakeflow-integrations/out.test.toml b/acceptance/bundle/templates/lakeflow-integrations/out.test.toml index 412d37c2f86..f61c2bccd55 100644 --- a/acceptance/bundle/templates/lakeflow-integrations/out.test.toml +++ b/acceptance/bundle/templates/lakeflow-integrations/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/templates/lakeflow-pipelines/python/out.test.toml b/acceptance/bundle/templates/lakeflow-pipelines/python/out.test.toml index 412d37c2f86..f61c2bccd55 100644 --- a/acceptance/bundle/templates/lakeflow-pipelines/python/out.test.toml +++ b/acceptance/bundle/templates/lakeflow-pipelines/python/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/templates/lakeflow-pipelines/sql/out.test.toml b/acceptance/bundle/templates/lakeflow-pipelines/sql/out.test.toml index 412d37c2f86..f61c2bccd55 100644 --- a/acceptance/bundle/templates/lakeflow-pipelines/sql/out.test.toml +++ b/acceptance/bundle/templates/lakeflow-pipelines/sql/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/templates/nested-output/out.test.toml b/acceptance/bundle/templates/nested-output/out.test.toml index 412d37c2f86..f61c2bccd55 100644 --- a/acceptance/bundle/templates/nested-output/out.test.toml +++ b/acceptance/bundle/templates/nested-output/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/templates/pydabs/check-consistency/out.test.toml b/acceptance/bundle/templates/pydabs/check-consistency/out.test.toml index de736377b08..a338b8b8a79 100644 --- a/acceptance/bundle/templates/pydabs/check-consistency/out.test.toml +++ b/acceptance/bundle/templates/pydabs/check-consistency/out.test.toml @@ -1,6 +1,6 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] EnvMatrix.INCLUDE_JOB = ["yes", "no"] EnvMatrix.INCLUDE_PIPELINE = ["yes", "no"] EnvMatrix.INCLUDE_PYTHON = ["yes", "no"] diff --git a/acceptance/bundle/templates/pydabs/check-formatting/out.test.toml b/acceptance/bundle/templates/pydabs/check-formatting/out.test.toml index de736377b08..a338b8b8a79 100644 --- a/acceptance/bundle/templates/pydabs/check-formatting/out.test.toml +++ b/acceptance/bundle/templates/pydabs/check-formatting/out.test.toml @@ -1,6 +1,6 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] EnvMatrix.INCLUDE_JOB = ["yes", "no"] EnvMatrix.INCLUDE_PIPELINE = ["yes", "no"] EnvMatrix.INCLUDE_PYTHON = ["yes", "no"] diff --git a/acceptance/bundle/templates/pydabs/deploy-classic/out.test.toml b/acceptance/bundle/templates/pydabs/deploy-classic/out.test.toml index 412d37c2f86..f61c2bccd55 100644 --- a/acceptance/bundle/templates/pydabs/deploy-classic/out.test.toml +++ b/acceptance/bundle/templates/pydabs/deploy-classic/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/templates/pydabs/init-classic/out.test.toml b/acceptance/bundle/templates/pydabs/init-classic/out.test.toml index 412d37c2f86..f61c2bccd55 100644 --- a/acceptance/bundle/templates/pydabs/init-classic/out.test.toml +++ b/acceptance/bundle/templates/pydabs/init-classic/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/templates/telemetry/custom-template/out.test.toml b/acceptance/bundle/templates/telemetry/custom-template/out.test.toml index 412d37c2f86..f61c2bccd55 100644 --- a/acceptance/bundle/templates/telemetry/custom-template/out.test.toml +++ b/acceptance/bundle/templates/telemetry/custom-template/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/templates/telemetry/dbt-sql/out.test.toml b/acceptance/bundle/templates/telemetry/dbt-sql/out.test.toml index 412d37c2f86..f61c2bccd55 100644 --- a/acceptance/bundle/templates/telemetry/dbt-sql/out.test.toml +++ b/acceptance/bundle/templates/telemetry/dbt-sql/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/templates/telemetry/default-python/out.test.toml b/acceptance/bundle/templates/telemetry/default-python/out.test.toml index 412d37c2f86..f61c2bccd55 100644 --- a/acceptance/bundle/templates/telemetry/default-python/out.test.toml +++ b/acceptance/bundle/templates/telemetry/default-python/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/templates/telemetry/default-sql/out.test.toml b/acceptance/bundle/templates/telemetry/default-sql/out.test.toml index 412d37c2f86..f61c2bccd55 100644 --- a/acceptance/bundle/templates/telemetry/default-sql/out.test.toml +++ b/acceptance/bundle/templates/telemetry/default-sql/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/templates/test.toml b/acceptance/bundle/templates/test.toml index 5d7fc7bfa7a..344203ecd58 100644 --- a/acceptance/bundle/templates/test.toml +++ b/acceptance/bundle/templates/test.toml @@ -4,7 +4,7 @@ # and some diff against a sibling test's output directory. Running all of that a second time # for deployment history recording costs minutes and adds no coverage the rest of the suite # does not already give, so these opt out. -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] [[Server]] Pattern = "POST /telemetry-ext" diff --git a/acceptance/bundle/test.toml b/acceptance/bundle/test.toml index e7355915277..e2af2bf51c9 100644 --- a/acceptance/bundle/test.toml +++ b/acceptance/bundle/test.toml @@ -4,26 +4,26 @@ EnvVaryOutput = "DATABRICKS_BUNDLE_ENGINE" # Runs the whole bundle suite a second time with deployment history recording on, so the # deployment metadata service (DMS) is exercised by every test rather than only the # handful under bundle/dms. Empty is the default pair of engine runs. -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] # DMS is only supported by the direct engine, and only against the local testserver: # the service runs in dev and staging, so a cloud run has nothing to record to. -EnvMatrixExclude.dms_needs_direct = ["DATABRICKS_BUNDLE_DMS=true", "DATABRICKS_BUNDLE_ENGINE=terraform"] -EnvMatrixExclude.dms_local_only = ["DATABRICKS_BUNDLE_DMS=true", "CONFIG_Cloud=true"] +EnvMatrixExclude.dms_needs_direct = ["DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY=true", "DATABRICKS_BUNDLE_ENGINE=terraform"] +EnvMatrixExclude.dms_local_only = ["DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY=true", "CONFIG_Cloud=true"] # A saved plan does not carry the deployment stamp. On a first deploy there is no # deployment to resolve when `bundle plan` runs, so the plan it writes leaves the field # unset; `deploy --plan` then creates the resources without it and the next plan reports # drift. Stamping at plan time would mean `bundle plan` creating the deployment record, # which is a design decision, so the saved-plan path is left out of the DMS run for now. -EnvMatrixExclude.dms_no_readplan = ["DATABRICKS_BUNDLE_DMS=true", "READPLAN=1"] +EnvMatrixExclude.dms_no_readplan = ["DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY=true", "READPLAN=1"] # Recording is gated off for users (see validate.ValidateRecordDeploymentHistory), so # force it on: the point of this run is to exercise it. Env.DATABRICKS_BUNDLE_FORCE_ALLOW_RECORD_DEPLOYMENT_HISTORY = "1" # The DMS run asserts the same golden files as the engine runs. -EnvRepl.DATABRICKS_BUNDLE_DMS = false +EnvRepl.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = false Ignore = ["databricks.yml"] diff --git a/acceptance/bundle/trampoline/warning_message/out.test.toml b/acceptance/bundle/trampoline/warning_message/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/trampoline/warning_message/out.test.toml +++ b/acceptance/bundle/trampoline/warning_message/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/trampoline/warning_message_with_new_spark/out.test.toml b/acceptance/bundle/trampoline/warning_message_with_new_spark/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/trampoline/warning_message_with_new_spark/out.test.toml +++ b/acceptance/bundle/trampoline/warning_message_with_new_spark/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/trampoline/warning_message_with_old_spark/out.test.toml b/acceptance/bundle/trampoline/warning_message_with_old_spark/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/trampoline/warning_message_with_old_spark/out.test.toml +++ b/acceptance/bundle/trampoline/warning_message_with_old_spark/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/undefined_resources/out.test.toml b/acceptance/bundle/undefined_resources/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/undefined_resources/out.test.toml +++ b/acceptance/bundle/undefined_resources/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/upload/internal_server_error/out.test.toml b/acceptance/bundle/upload/internal_server_error/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/upload/internal_server_error/out.test.toml +++ b/acceptance/bundle/upload/internal_server_error/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/upload/timeout/out.test.toml b/acceptance/bundle/upload/timeout/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/upload/timeout/out.test.toml +++ b/acceptance/bundle/upload/timeout/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/user_agent/out.test.toml b/acceptance/bundle/user_agent/out.test.toml index 8f26ec1671a..4cf738bfea6 100644 --- a/acceptance/bundle/user_agent/out.test.toml +++ b/acceptance/bundle/user_agent/out.test.toml @@ -1,4 +1,4 @@ Cloud = false Phase = 1 -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/user_agent/simple/out.test.toml b/acceptance/bundle/user_agent/simple/out.test.toml index 412d37c2f86..f61c2bccd55 100644 --- a/acceptance/bundle/user_agent/simple/out.test.toml +++ b/acceptance/bundle/user_agent/simple/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/user_agent/test.toml b/acceptance/bundle/user_agent/test.toml index 8dec56fdc86..360fc8e7711 100644 --- a/acceptance/bundle/user_agent/test.toml +++ b/acceptance/bundle/user_agent/test.toml @@ -5,7 +5,7 @@ IncludeRequestHeaders = ["User-Agent"] # This test asserts the User-Agent on every single request the CLI makes, so recording's # extra calls belong in the golden rather than being filtered out - but they are the same # header the existing requests already cover, so the DMS run only adds entries to maintain. -EnvMatrix.DATABRICKS_BUNDLE_DMS = [""] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] [Env] DATABRICKS_CACHE_ENABLED = 'false' diff --git a/acceptance/bundle/validate/anchor_containers/out.test.toml b/acceptance/bundle/validate/anchor_containers/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/validate/anchor_containers/out.test.toml +++ b/acceptance/bundle/validate/anchor_containers/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/validate/catalog_requires_direct_mode/out.test.toml b/acceptance/bundle/validate/catalog_requires_direct_mode/out.test.toml index 2563df1863f..33ed6258236 100644 --- a/acceptance/bundle/validate/catalog_requires_direct_mode/out.test.toml +++ b/acceptance/bundle/validate/catalog_requires_direct_mode/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/validate/dashboard_defaults/out.test.toml b/acceptance/bundle/validate/dashboard_defaults/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/validate/dashboard_defaults/out.test.toml +++ b/acceptance/bundle/validate/dashboard_defaults/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/validate/dashboard_required_name/out.test.toml b/acceptance/bundle/validate/dashboard_required_name/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/validate/dashboard_required_name/out.test.toml +++ b/acceptance/bundle/validate/dashboard_required_name/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/validate/dashboard_required_warehouse_id/out.test.toml b/acceptance/bundle/validate/dashboard_required_warehouse_id/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/validate/dashboard_required_warehouse_id/out.test.toml +++ b/acceptance/bundle/validate/dashboard_required_warehouse_id/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/validate/definitions_yaml_anchors/out.test.toml b/acceptance/bundle/validate/definitions_yaml_anchors/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/validate/definitions_yaml_anchors/out.test.toml +++ b/acceptance/bundle/validate/definitions_yaml_anchors/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/validate/duplicate_yaml_merge_key/out.test.toml b/acceptance/bundle/validate/duplicate_yaml_merge_key/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/validate/duplicate_yaml_merge_key/out.test.toml +++ b/acceptance/bundle/validate/duplicate_yaml_merge_key/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/validate/empty_resources/empty_def/out.test.toml b/acceptance/bundle/validate/empty_resources/empty_def/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/validate/empty_resources/empty_def/out.test.toml +++ b/acceptance/bundle/validate/empty_resources/empty_def/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/validate/empty_resources/empty_dict/out.test.toml b/acceptance/bundle/validate/empty_resources/empty_dict/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/validate/empty_resources/empty_dict/out.test.toml +++ b/acceptance/bundle/validate/empty_resources/empty_dict/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/validate/empty_resources/null/out.test.toml b/acceptance/bundle/validate/empty_resources/null/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/validate/empty_resources/null/out.test.toml +++ b/acceptance/bundle/validate/empty_resources/null/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/validate/empty_resources/with_grants/out.test.toml b/acceptance/bundle/validate/empty_resources/with_grants/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/validate/empty_resources/with_grants/out.test.toml +++ b/acceptance/bundle/validate/empty_resources/with_grants/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/validate/empty_resources/with_permissions/out.test.toml b/acceptance/bundle/validate/empty_resources/with_permissions/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/validate/empty_resources/with_permissions/out.test.toml +++ b/acceptance/bundle/validate/empty_resources/with_permissions/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/validate/empty_tasks/out.test.toml b/acceptance/bundle/validate/empty_tasks/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/validate/empty_tasks/out.test.toml +++ b/acceptance/bundle/validate/empty_tasks/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/validate/engine-config-valid/out.test.toml b/acceptance/bundle/validate/engine-config-valid/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/validate/engine-config-valid/out.test.toml +++ b/acceptance/bundle/validate/engine-config-valid/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/validate/enum/out.test.toml b/acceptance/bundle/validate/enum/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/validate/enum/out.test.toml +++ b/acceptance/bundle/validate/enum/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/validate/enum_resource_refs/out.test.toml b/acceptance/bundle/validate/enum_resource_refs/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/validate/enum_resource_refs/out.test.toml +++ b/acceptance/bundle/validate/enum_resource_refs/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/validate/genie_space_complex/out.test.toml b/acceptance/bundle/validate/genie_space_complex/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/validate/genie_space_complex/out.test.toml +++ b/acceptance/bundle/validate/genie_space_complex/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/validate/genie_space_defaults/out.test.toml b/acceptance/bundle/validate/genie_space_defaults/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/validate/genie_space_defaults/out.test.toml +++ b/acceptance/bundle/validate/genie_space_defaults/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/validate/genie_space_file_path_and_inline/out.test.toml b/acceptance/bundle/validate/genie_space_file_path_and_inline/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/validate/genie_space_file_path_and_inline/out.test.toml +++ b/acceptance/bundle/validate/genie_space_file_path_and_inline/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/validate/grants_required_principal/out.test.toml b/acceptance/bundle/validate/grants_required_principal/out.test.toml index 51a5602947c..2c6699da193 100644 --- a/acceptance/bundle/validate/grants_required_principal/out.test.toml +++ b/acceptance/bundle/validate/grants_required_principal/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/validate/immutable_workspace_paths/out.test.toml b/acceptance/bundle/validate/immutable_workspace_paths/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/validate/immutable_workspace_paths/out.test.toml +++ b/acceptance/bundle/validate/immutable_workspace_paths/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/validate/include_locations/out.test.toml b/acceptance/bundle/validate/include_locations/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/validate/include_locations/out.test.toml +++ b/acceptance/bundle/validate/include_locations/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/validate/invalid-engine-bundle/out.test.toml b/acceptance/bundle/validate/invalid-engine-bundle/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/validate/invalid-engine-bundle/out.test.toml +++ b/acceptance/bundle/validate/invalid-engine-bundle/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/validate/invalid-engine-target/out.test.toml b/acceptance/bundle/validate/invalid-engine-target/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/validate/invalid-engine-target/out.test.toml +++ b/acceptance/bundle/validate/invalid-engine-target/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/validate/job-references/out.test.toml b/acceptance/bundle/validate/job-references/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/validate/job-references/out.test.toml +++ b/acceptance/bundle/validate/job-references/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/validate/model_serving_both_fields_error/out.test.toml b/acceptance/bundle/validate/model_serving_both_fields_error/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/validate/model_serving_both_fields_error/out.test.toml +++ b/acceptance/bundle/validate/model_serving_both_fields_error/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/validate/model_serving_conversion/out.test.toml b/acceptance/bundle/validate/model_serving_conversion/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/validate/model_serving_conversion/out.test.toml +++ b/acceptance/bundle/validate/model_serving_conversion/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/validate/models/missing_name/out.test.toml b/acceptance/bundle/validate/models/missing_name/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/validate/models/missing_name/out.test.toml +++ b/acceptance/bundle/validate/models/missing_name/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/validate/models/user_id/out.test.toml b/acceptance/bundle/validate/models/user_id/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/validate/models/user_id/out.test.toml +++ b/acceptance/bundle/validate/models/user_id/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/validate/no_dashboard_etag/out.test.toml b/acceptance/bundle/validate/no_dashboard_etag/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/validate/no_dashboard_etag/out.test.toml +++ b/acceptance/bundle/validate/no_dashboard_etag/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/validate/no_genie_space_etag/out.test.toml b/acceptance/bundle/validate/no_genie_space_etag/out.test.toml index 51a5602947c..2c6699da193 100644 --- a/acceptance/bundle/validate/no_genie_space_etag/out.test.toml +++ b/acceptance/bundle/validate/no_genie_space_etag/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/validate/permissions/out.test.toml b/acceptance/bundle/validate/permissions/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/validate/permissions/out.test.toml +++ b/acceptance/bundle/validate/permissions/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/validate/permissions_overlap/out.test.toml b/acceptance/bundle/validate/permissions_overlap/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/validate/permissions_overlap/out.test.toml +++ b/acceptance/bundle/validate/permissions_overlap/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/validate/presets_max_concurrent_runs/out.test.toml b/acceptance/bundle/validate/presets_max_concurrent_runs/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/validate/presets_max_concurrent_runs/out.test.toml +++ b/acceptance/bundle/validate/presets_max_concurrent_runs/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/validate/presets_name_prefix/out.test.toml b/acceptance/bundle/validate/presets_name_prefix/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/validate/presets_name_prefix/out.test.toml +++ b/acceptance/bundle/validate/presets_name_prefix/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/validate/presets_name_prefix_dev/out.test.toml b/acceptance/bundle/validate/presets_name_prefix_dev/out.test.toml index 51a5602947c..2c6699da193 100644 --- a/acceptance/bundle/validate/presets_name_prefix_dev/out.test.toml +++ b/acceptance/bundle/validate/presets_name_prefix_dev/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/validate/presets_tags/out.test.toml b/acceptance/bundle/validate/presets_tags/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/validate/presets_tags/out.test.toml +++ b/acceptance/bundle/validate/presets_tags/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/validate/required/out.test.toml b/acceptance/bundle/validate/required/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/validate/required/out.test.toml +++ b/acceptance/bundle/validate/required/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/validate/reserved_deployment_fields/out.test.toml b/acceptance/bundle/validate/reserved_deployment_fields/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/validate/reserved_deployment_fields/out.test.toml +++ b/acceptance/bundle/validate/reserved_deployment_fields/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/validate/sql_warehouse_required_name/out.test.toml b/acceptance/bundle/validate/sql_warehouse_required_name/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/validate/sql_warehouse_required_name/out.test.toml +++ b/acceptance/bundle/validate/sql_warehouse_required_name/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/validate/strict/out.test.toml b/acceptance/bundle/validate/strict/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/validate/strict/out.test.toml +++ b/acceptance/bundle/validate/strict/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/validate/sync_patterns/out.test.toml b/acceptance/bundle/validate/sync_patterns/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/validate/sync_patterns/out.test.toml +++ b/acceptance/bundle/validate/sync_patterns/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/validate/var_in_bundle_name/out.test.toml b/acceptance/bundle/validate/var_in_bundle_name/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/validate/var_in_bundle_name/out.test.toml +++ b/acceptance/bundle/validate/var_in_bundle_name/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/validate/volume_defaults/out.test.toml b/acceptance/bundle/validate/volume_defaults/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/validate/volume_defaults/out.test.toml +++ b/acceptance/bundle/validate/volume_defaults/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/variables/arg-repeat/out.test.toml b/acceptance/bundle/variables/arg-repeat/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/variables/arg-repeat/out.test.toml +++ b/acceptance/bundle/variables/arg-repeat/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/variables/complex-cross-ref/out.test.toml b/acceptance/bundle/variables/complex-cross-ref/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/variables/complex-cross-ref/out.test.toml +++ b/acceptance/bundle/variables/complex-cross-ref/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/variables/complex-cycle-self/out.test.toml b/acceptance/bundle/variables/complex-cycle-self/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/variables/complex-cycle-self/out.test.toml +++ b/acceptance/bundle/variables/complex-cycle-self/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/variables/complex-cycle/out.test.toml b/acceptance/bundle/variables/complex-cycle/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/variables/complex-cycle/out.test.toml +++ b/acceptance/bundle/variables/complex-cycle/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/variables/complex-simple/out.test.toml b/acceptance/bundle/variables/complex-simple/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/variables/complex-simple/out.test.toml +++ b/acceptance/bundle/variables/complex-simple/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/variables/complex-transitive-deep/out.test.toml b/acceptance/bundle/variables/complex-transitive-deep/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/variables/complex-transitive-deep/out.test.toml +++ b/acceptance/bundle/variables/complex-transitive-deep/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/variables/complex-transitive-deeper/out.test.toml b/acceptance/bundle/variables/complex-transitive-deeper/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/variables/complex-transitive-deeper/out.test.toml +++ b/acceptance/bundle/variables/complex-transitive-deeper/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/variables/complex-transitive/out.test.toml b/acceptance/bundle/variables/complex-transitive/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/variables/complex-transitive/out.test.toml +++ b/acceptance/bundle/variables/complex-transitive/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/variables/complex-with-var-reference/out.test.toml b/acceptance/bundle/variables/complex-with-var-reference/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/variables/complex-with-var-reference/out.test.toml +++ b/acceptance/bundle/variables/complex-with-var-reference/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/variables/complex-within-complex/out.test.toml b/acceptance/bundle/variables/complex-within-complex/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/variables/complex-within-complex/out.test.toml +++ b/acceptance/bundle/variables/complex-within-complex/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/variables/complex/out.test.toml b/acceptance/bundle/variables/complex/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/variables/complex/out.test.toml +++ b/acceptance/bundle/variables/complex/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/variables/complex_multiple_files/out.test.toml b/acceptance/bundle/variables/complex_multiple_files/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/variables/complex_multiple_files/out.test.toml +++ b/acceptance/bundle/variables/complex_multiple_files/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/variables/cycle/out.test.toml b/acceptance/bundle/variables/cycle/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/variables/cycle/out.test.toml +++ b/acceptance/bundle/variables/cycle/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/variables/double_underscore/out.test.toml b/acceptance/bundle/variables/double_underscore/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/variables/double_underscore/out.test.toml +++ b/acceptance/bundle/variables/double_underscore/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/variables/empty/out.test.toml b/acceptance/bundle/variables/empty/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/variables/empty/out.test.toml +++ b/acceptance/bundle/variables/empty/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/variables/env_overrides/out.test.toml b/acceptance/bundle/variables/env_overrides/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/variables/env_overrides/out.test.toml +++ b/acceptance/bundle/variables/env_overrides/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/variables/file-defaults/out.test.toml b/acceptance/bundle/variables/file-defaults/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/variables/file-defaults/out.test.toml +++ b/acceptance/bundle/variables/file-defaults/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/variables/git-branch/out.test.toml b/acceptance/bundle/variables/git-branch/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/variables/git-branch/out.test.toml +++ b/acceptance/bundle/variables/git-branch/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/variables/host/out.test.toml b/acceptance/bundle/variables/host/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/variables/host/out.test.toml +++ b/acceptance/bundle/variables/host/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/variables/int/out.test.toml b/acceptance/bundle/variables/int/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/variables/int/out.test.toml +++ b/acceptance/bundle/variables/int/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/variables/issue_2436/out.test.toml b/acceptance/bundle/variables/issue_2436/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/variables/issue_2436/out.test.toml +++ b/acceptance/bundle/variables/issue_2436/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/variables/issue_3039_lookup_with_ref/out.test.toml b/acceptance/bundle/variables/issue_3039_lookup_with_ref/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/variables/issue_3039_lookup_with_ref/out.test.toml +++ b/acceptance/bundle/variables/issue_3039_lookup_with_ref/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/variables/lookup/out.test.toml b/acceptance/bundle/variables/lookup/out.test.toml index 0fc51ee009b..264de50aa96 100644 --- a/acceptance/bundle/variables/lookup/out.test.toml +++ b/acceptance/bundle/variables/lookup/out.test.toml @@ -1,3 +1,3 @@ Cloud = true -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/variables/prepend-workspace-var/out.test.toml b/acceptance/bundle/variables/prepend-workspace-var/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/variables/prepend-workspace-var/out.test.toml +++ b/acceptance/bundle/variables/prepend-workspace-var/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/variables/resolve-builtin/out.test.toml b/acceptance/bundle/variables/resolve-builtin/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/variables/resolve-builtin/out.test.toml +++ b/acceptance/bundle/variables/resolve-builtin/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/variables/resolve-empty/out.test.toml b/acceptance/bundle/variables/resolve-empty/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/variables/resolve-empty/out.test.toml +++ b/acceptance/bundle/variables/resolve-empty/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/variables/resolve-field-within-complex/out.test.toml b/acceptance/bundle/variables/resolve-field-within-complex/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/variables/resolve-field-within-complex/out.test.toml +++ b/acceptance/bundle/variables/resolve-field-within-complex/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/variables/resolve-nonstrings/out.test.toml b/acceptance/bundle/variables/resolve-nonstrings/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/variables/resolve-nonstrings/out.test.toml +++ b/acceptance/bundle/variables/resolve-nonstrings/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/variables/resolve-resources-fields/out.test.toml b/acceptance/bundle/variables/resolve-resources-fields/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/variables/resolve-resources-fields/out.test.toml +++ b/acceptance/bundle/variables/resolve-resources-fields/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/variables/resolve-vars-in-root-path/out.test.toml b/acceptance/bundle/variables/resolve-vars-in-root-path/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/variables/resolve-vars-in-root-path/out.test.toml +++ b/acceptance/bundle/variables/resolve-vars-in-root-path/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/variables/unicode_reference/out.test.toml b/acceptance/bundle/variables/unicode_reference/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/variables/unicode_reference/out.test.toml +++ b/acceptance/bundle/variables/unicode_reference/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/variables/vanilla/out.test.toml b/acceptance/bundle/variables/vanilla/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/variables/vanilla/out.test.toml +++ b/acceptance/bundle/variables/vanilla/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/variables/var_in_var/out.test.toml b/acceptance/bundle/variables/var_in_var/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/variables/var_in_var/out.test.toml +++ b/acceptance/bundle/variables/var_in_var/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/variables/variable_in_resource_key/out.test.toml b/acceptance/bundle/variables/variable_in_resource_key/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/variables/variable_in_resource_key/out.test.toml +++ b/acceptance/bundle/variables/variable_in_resource_key/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/variables/variable_overrides_in_target/out.test.toml b/acceptance/bundle/variables/variable_overrides_in_target/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/variables/variable_overrides_in_target/out.test.toml +++ b/acceptance/bundle/variables/variable_overrides_in_target/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/variables/without_definition/out.test.toml b/acceptance/bundle/variables/without_definition/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/variables/without_definition/out.test.toml +++ b/acceptance/bundle/variables/without_definition/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/volume_path/invalid_file/out.test.toml b/acceptance/bundle/volume_path/invalid_file/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/volume_path/invalid_file/out.test.toml +++ b/acceptance/bundle/volume_path/invalid_file/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/volume_path/invalid_resource/out.test.toml b/acceptance/bundle/volume_path/invalid_resource/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/volume_path/invalid_resource/out.test.toml +++ b/acceptance/bundle/volume_path/invalid_resource/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/volume_path/invalid_root/out.test.toml b/acceptance/bundle/volume_path/invalid_root/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/volume_path/invalid_root/out.test.toml +++ b/acceptance/bundle/volume_path/invalid_root/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/volume_path/invalid_state/out.test.toml b/acceptance/bundle/volume_path/invalid_state/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/volume_path/invalid_state/out.test.toml +++ b/acceptance/bundle/volume_path/invalid_state/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/volume_path/valid/out.test.toml b/acceptance/bundle/volume_path/valid/out.test.toml index 53698799aaa..c7a035e8011 100644 --- a/acceptance/bundle/volume_path/valid/out.test.toml +++ b/acceptance/bundle/volume_path/valid/out.test.toml @@ -1,3 +1,3 @@ Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_DMS = ["", "true"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/build/pybin/python b/build/pybin/python new file mode 120000 index 00000000000..40a7c693444 --- /dev/null +++ b/build/pybin/python @@ -0,0 +1 @@ +/home/shreyas.goenka/.pyenv/versions/3.11.11/bin/python3 \ No newline at end of file diff --git a/build/pybin/python3 b/build/pybin/python3 new file mode 120000 index 00000000000..40a7c693444 --- /dev/null +++ b/build/pybin/python3 @@ -0,0 +1 @@ +/home/shreyas.goenka/.pyenv/versions/3.11.11/bin/python3 \ No newline at end of file diff --git a/build/pybin/uv b/build/pybin/uv new file mode 100755 index 00000000000..7b99bf07df5 --- /dev/null +++ b/build/pybin/uv @@ -0,0 +1,9 @@ +#!/bin/bash +# Wrapper to provide uv python find functionality +if [[ "$1" == "python" && "$2" == "find" ]]; then + # Return the direct path to python3 + echo "/home/shreyas.goenka/.pyenv/versions/3.11.11/bin/python3" +else + # For other uv commands, try the real uv + /home/shreyas.goenka/.local/bin/uv "$@" +fi diff --git a/bundle/env/dms.go b/bundle/env/dms.go index 53aeaccc011..435a9ac3bcc 100644 --- a/bundle/env/dms.go +++ b/bundle/env/dms.go @@ -2,24 +2,28 @@ package env import "context" -// DMSVariable names the environment variable that turns on deployment history -// recording without setting experimental.record_deployment_history in the bundle. -// It exists for the CLI's own acceptance tests, which run the whole bundle suite -// with DMS enabled: setting it here beats adding the field to every databricks.yml. +// RecordDeploymentHistoryVariable names the environment variable that turns on +// deployment history recording without setting experimental.record_deployment_history +// in the bundle. It exists for the CLI's own acceptance tests, which run the whole +// bundle suite with recording enabled: setting it here beats adding the field to every +// databricks.yml. // // Like ForceAllowRecordDeploymentHistoryVariable it is deliberately undocumented; see // validate.ValidateRecordDeploymentHistory for why the feature is still gated off. -const DMSVariable = "DATABRICKS_BUNDLE_DMS" +const RecordDeploymentHistoryVariable = "DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY" -// DMS reports whether the environment turns on deployment history recording. -func DMS(ctx context.Context) bool { - value, ok := get(ctx, []string{DMSVariable}) - return ok && value != "" && value != "0" && value != "false" +// recordDeploymentHistoryEnv reports whether the environment turns on deployment +// history recording. Only "true" turns it on: anything else - including a typo like +// "TRUE" or "yes" - leaves a gated feature off rather than silently enabling it. +func recordDeploymentHistoryEnv(ctx context.Context) bool { + value, _ := get(ctx, []string{RecordDeploymentHistoryVariable}) + return value == "true" } // RecordsDeploymentHistory reports whether this deploy records deployment history, -// from either the bundle setting or DMSVariable. It is the single predicate the -// recording code paths branch on, so the env var and the config field cannot drift. +// from either the bundle setting or RecordDeploymentHistoryVariable. It is the single +// predicate the recording code paths branch on, so the env var and the config field +// cannot drift. func RecordsDeploymentHistory(ctx context.Context, configured bool) bool { - return configured || DMS(ctx) + return configured || recordDeploymentHistoryEnv(ctx) } diff --git a/bundle/env/dms_test.go b/bundle/env/dms_test.go index f449d5ec6b9..daa45dc00a2 100644 --- a/bundle/env/dms_test.go +++ b/bundle/env/dms_test.go @@ -7,24 +7,27 @@ import ( "github.com/stretchr/testify/assert" ) -func TestDMS(t *testing.T) { +func TestRecordDeploymentHistoryEnv(t *testing.T) { for _, tc := range []struct { value string want bool }{ {"true", true}, - {"1", true}, {"", false}, {"0", false}, {"false", false}, + // Only "true" counts, so a near miss leaves recording off. + {"1", false}, + {"TRUE", false}, + {"yes", false}, } { - ctx := env.Set(t.Context(), DMSVariable, tc.value) - assert.Equal(t, tc.want, DMS(ctx), "value %q", tc.value) + ctx := env.Set(t.Context(), RecordDeploymentHistoryVariable, tc.value) + assert.Equal(t, tc.want, recordDeploymentHistoryEnv(ctx), "value %q", tc.value) } } -func TestDMSUnset(t *testing.T) { - assert.False(t, DMS(t.Context())) +func TestRecordDeploymentHistoryEnvUnset(t *testing.T) { + assert.False(t, recordDeploymentHistoryEnv(t.Context())) } func TestRecordsDeploymentHistory(t *testing.T) { @@ -33,6 +36,6 @@ func TestRecordsDeploymentHistory(t *testing.T) { assert.True(t, RecordsDeploymentHistory(t.Context(), true)) assert.False(t, RecordsDeploymentHistory(t.Context(), false)) - ctx := env.Set(t.Context(), DMSVariable, "true") + ctx := env.Set(t.Context(), RecordDeploymentHistoryVariable, "true") assert.True(t, RecordsDeploymentHistory(ctx, false)) } diff --git a/bundle/phases/dms.go b/bundle/phases/dms.go index 8ca73fac94e..631752694fa 100644 --- a/bundle/phases/dms.go +++ b/bundle/phases/dms.go @@ -60,7 +60,8 @@ func newDeploymentRecorder(ctx context.Context, b *bundle.Bundle, eng engine.Eng } // recordsDeploymentHistory reports whether this bundle records deployment history, -// from experimental.record_deployment_history or DATABRICKS_BUNDLE_DMS. +// from experimental.record_deployment_history or +// DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY. func recordsDeploymentHistory(ctx context.Context, b *bundle.Bundle) bool { configured := b.Config.Experimental != nil && b.Config.Experimental.RecordDeploymentHistory return env.RecordsDeploymentHistory(ctx, configured) From 95c1f03c69371c2c809ffdfa6e82078329030cb9 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 11 Aug 2026 09:38:30 +0000 Subject: [PATCH 076/125] bundle: print the deployment version up front The link was printed after "Deployment complete!", which is the point it is least useful: the deploy is already over. Printing it once the version is created lets the user follow it while resources apply, and leaves them the link when a deploy fails partway. Renamed to "Current Deployment Version" - it points at the version this deploy is recording under, not at a history listing. Co-authored-by: Isaac --- acceptance/bundle/test.toml | 2 +- bundle/phases/deploy.go | 4 +++- bundle/phases/dms.go | 17 +++++++++-------- cmd/bundle/utils/process.go | 8 +++----- 4 files changed, 16 insertions(+), 15 deletions(-) diff --git a/acceptance/bundle/test.toml b/acceptance/bundle/test.toml index e2af2bf51c9..3ef7efd376c 100644 --- a/acceptance/bundle/test.toml +++ b/acceptance/bundle/test.toml @@ -34,7 +34,7 @@ Env.UV_PYTHON = "3.10" # not recording is on. The URL itself is covered by workspaceurls.TestDeploymentURL, and # the calls behind it by bundle/dms/record. [[Repls]] -Old = '(?m)^Deployment history: .*\n' +Old = '(?m)^Current Deployment Version: .*\n' New = '' # User-agent: diff --git a/bundle/phases/deploy.go b/bundle/phases/deploy.go index 006ed90f12b..af1f6331305 100644 --- a/bundle/phases/deploy.go +++ b/bundle/phases/deploy.go @@ -122,7 +122,6 @@ func deployCore(ctx context.Context, b *bundle.Bundle, plan *deployplan.Plan, st if !logdiag.HasError(ctx) { cmdio.LogString(ctx, "Deployment complete!") - logDeploymentHistory(ctx, b, recorder) } // Once the deploy is complete, dry-run the migration to the direct engine @@ -253,6 +252,9 @@ func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHand if logdiag.HasError(ctx) { return } + // Printed here rather than after the deploy so the user can follow the version + // while it runs, and still has the link if the deploy fails partway. + logDeploymentVersion(ctx, b, recorder) } planFromFile := plan != nil diff --git a/bundle/phases/dms.go b/bundle/phases/dms.go index 631752694fa..9803ebef9f6 100644 --- a/bundle/phases/dms.go +++ b/bundle/phases/dms.go @@ -84,27 +84,28 @@ func setOperationRecorder(ctx context.Context, b *bundle.Bundle, recorder *dms.R b.DeploymentBundle.OpRec = direct.NewOperationRecorder(apiClient, recorder.DeploymentID(), recorder.Version()) } -// logDeploymentHistory links to the deployment this deploy was recorded under, so -// the user can open its history without hunting for the ID. A nil recorder means -// recording is off, and a zero version means the version was never created. +// logDeploymentVersion links to the version this deploy was recorded under, so the +// user can follow it while the deploy runs rather than hunting for the ID afterwards. +// A nil recorder means recording is off, and a zero version means the version was +// never created. // // The workspace ID is left out of the URL: the page redirects correctly without it, // and omitting it keeps the line short enough to stay clickable in a terminal. -func logDeploymentHistory(ctx context.Context, b *bundle.Bundle, recorder *dms.Recorder) { +func logDeploymentVersion(ctx context.Context, b *bundle.Bundle, recorder *dms.Recorder) { if recorder == nil || recorder.Version() == 0 { return } baseURL, err := url.Parse(b.WorkspaceClient(ctx).Config.CanonicalHostName()) if err != nil { - // Only the link is lost, so report the deployment without it rather than - // failing a deploy that already succeeded. + // Only the link is lost, so report the version without it rather than failing + // a deploy over it. log.Debugf(ctx, "Not linking to the recorded deployment: %s", err) - cmdio.LogString(ctx, fmt.Sprintf("Recorded deployment %s version %d", recorder.DeploymentID(), recorder.Version())) + cmdio.LogString(ctx, fmt.Sprintf("Current Deployment Version: %s version %d", recorder.DeploymentID(), recorder.Version())) return } - cmdio.LogString(ctx, "Deployment history: "+workspaceurls.DeploymentURL(*baseURL, recorder.DeploymentID(), recorder.Version())) + cmdio.LogString(ctx, "Current Deployment Version: "+workspaceurls.DeploymentURL(*baseURL, recorder.DeploymentID(), recorder.Version())) } // deploymentMetadata describes the bundle this deploy came from and where it diff --git a/cmd/bundle/utils/process.go b/cmd/bundle/utils/process.go index 36205181115..7e937adee4c 100644 --- a/cmd/bundle/utils/process.go +++ b/cmd/bundle/utils/process.go @@ -220,11 +220,9 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle if needDirectState { _, localPath := b.StateFilenameDirect(ctx) - // When the bundle records deployment history, the deployment metadata - // service owns resource state, so hand Open a DMS source to read it from - // there instead of the file. The local identity (lineage/serial) still - // comes from the file. Reads open the state write-disabled, so no lineage - // is minted here. + // Recording makes the service the source of truth for resource state, so a + // deploy has to plan against what it holds rather than a local file that + // another machine's deploy may have left behind. var dmsSource *dstate.DMSSource if env.RecordsDeploymentHistory(ctx, b.Config.Experimental != nil && b.Config.Experimental.RecordDeploymentHistory) { w := b.WorkspaceClient(ctx) From eccac6bf28bf14af927f13ff68103289d966b112 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 11 Aug 2026 09:55:07 +0000 Subject: [PATCH 077/125] bundle: label a recreate's second write as a recreate DeploymentUnit.Create hardcoded action Create, so a recreate - which ends by calling it - reported its second write as a create. That only looked right because the write goes out as a PATCH, which cannot carry action_type, leaving the recreate the first write set. Merging the two writes exposed it, and mergeOperation existed to paper over it by carrying the older action forward. Create now takes the action it is part of, so both writes of a recreate say recreate and the merge needs no fixup. mergeOperation is gone. Note the invariant is thinly covered: acceptance tests never merge, because the two writes are separated by API calls, so a regression here would only show up in the queue's unit test. A DeploymentUnit-level test needs a fake adapter harness, which does not exist yet. Co-authored-by: Isaac --- bundle/direct/apply.go | 11 +++++++---- bundle/direct/opqueue.go | 9 +++------ bundle/direct/opqueue_test.go | 11 ++++++----- bundle/direct/oprecorder.go | 14 -------------- 4 files changed, 16 insertions(+), 29 deletions(-) diff --git a/bundle/direct/apply.go b/bundle/direct/apply.go index ec5761c4a76..5346af985e9 100644 --- a/bundle/direct/apply.go +++ b/bundle/direct/apply.go @@ -28,7 +28,7 @@ func (d *DeploymentUnit) Destroy(ctx context.Context, db *dstate.DeploymentState func (d *DeploymentUnit) Deploy(ctx context.Context, db *dstate.DeploymentState, newState any, actionType deployplan.ActionType, planEntry *deployplan.PlanEntry) error { ctx = log.WithPrefix(ctx, "deploying "+d.ResourceKey) if actionType == deployplan.Create { - return d.Create(ctx, db, newState) + return d.Create(ctx, db, newState, deployplan.Create) } oldID := db.GetResourceID(d.ResourceKey) @@ -50,7 +50,10 @@ func (d *DeploymentUnit) Deploy(ctx context.Context, db *dstate.DeploymentState, } } -func (d *DeploymentUnit) Create(ctx context.Context, db *dstate.DeploymentState, newState any) error { +// Create creates the resource and records its state. action is the operation the +// create is part of: a recreate ends by calling this, and reports the write as a +// recreate so the deployment history says how the resource got here. +func (d *DeploymentUnit) Create(ctx context.Context, db *dstate.DeploymentState, newState any, action deployplan.ActionType) error { var newID string var remoteState any _, err := retryWith(ctx, func(err error) bool { @@ -75,7 +78,7 @@ func (d *DeploymentUnit) Create(ctx context.Context, db *dstate.DeploymentState, return err } - err = db.SaveState(ctx, d.ResourceKey, newID, newState, d.DependsOn, dstate.OperationInfo{Action: deployplan.Create}) + err = db.SaveState(ctx, d.ResourceKey, newID, newState, d.DependsOn, dstate.OperationInfo{Action: action}) if err != nil { return fmt.Errorf("saving state after creating id=%s: %w", newID, err) } @@ -134,7 +137,7 @@ func (d *DeploymentUnit) Recreate(ctx context.Context, db *dstate.DeploymentStat return fmt.Errorf("waiting after deleting id=%s: %w", oldID, err) } - return d.Create(ctx, db, newState) + return d.Create(ctx, db, newState, deployplan.Recreate) } func (d *DeploymentUnit) Update(ctx context.Context, db *dstate.DeploymentState, id string, newState any, planEntry *deployplan.PlanEntry) error { diff --git a/bundle/direct/opqueue.go b/bundle/direct/opqueue.go index 4ee3712008c..b22b0a78ae0 100644 --- a/bundle/direct/opqueue.go +++ b/bundle/direct/opqueue.go @@ -45,8 +45,8 @@ type operationQueue struct { // pending holds the one operation waiting per resource key: a resource can write // state more than once in a deploy (a recreate drops the entry, then saves the new - // resource), and writes that arrive before the previous one is uploaded are merged - // by mergeOperation. No key means nothing is waiting. + // resource), and a write that arrives before the previous one is uploaded replaces + // it. No key means nothing is waiting. pending map[string]recordedOperation // queuedOrUploading means "some worker will get to this key". Recording such a @@ -118,13 +118,10 @@ func (q *operationQueue) recordFailure(ctx context.Context, resourceKey string, q.enqueue(ctx, resourceKey, op) } -// enqueue makes op the operation waiting for resourceKey, merged onto whatever was +// enqueue makes op the operation waiting for resourceKey, replacing whatever was // already waiting, and makes sure a worker will pick it up. func (q *operationQueue) enqueue(ctx context.Context, resourceKey string, op recordedOperation) { q.mu.Lock() - if waiting, ok := q.pending[resourceKey]; ok { - op = mergeOperation(waiting, op) - } q.pending[resourceKey] = op alreadyHandled := q.queuedOrUploading[resourceKey] q.queuedOrUploading[resourceKey] = true diff --git a/bundle/direct/opqueue_test.go b/bundle/direct/opqueue_test.go index 9e14fd11a1a..f4be818390a 100644 --- a/bundle/direct/opqueue_test.go +++ b/bundle/direct/opqueue_test.go @@ -160,10 +160,11 @@ func TestOperationQueueMergesWritesQueuedBehindAnUpload(t *testing.T) { } func TestOperationQueueMergedRecreateKeepsItsActionType(t *testing.T) { - // A recreate records its intermediate delete, then the create that replaces the - // resource. Merging them must upload the recreate's action: the service fixes - // action_type when the operation is created and rejects it in an update mask, so - // taking the newer create's action would report the resource as merely created. + // Both writes of a recreate report it as a recreate (see DeploymentUnit.Create), + // so the merged upload carries that action however the two are combined. It has to: + // the service fixes action_type when the operation is created and rejects it in an + // update mask, so a merged pair that reported "create" would leave the history + // saying the resource was merely created. // // Every worker is parked so both writes land in pending and merge before upload. f := &fakeUploader{block: make(chan struct{}), started: make(chan string, operationUploadWorkers+2)} @@ -175,7 +176,7 @@ func TestOperationQueueMergedRecreateKeepsItsActionType(t *testing.T) { } q.RecordOperation(t.Context(), "resources.jobs.foo", dstate.OperationInfo{Action: deployplan.Recreate, InProgress: true}, "old-id", nil) - q.RecordOperation(t.Context(), "resources.jobs.foo", dstate.OperationInfo{Action: deployplan.Create}, "new-id", envelope(t, "replacement")) + q.RecordOperation(t.Context(), "resources.jobs.foo", dstate.OperationInfo{Action: deployplan.Recreate}, "new-id", envelope(t, "replacement")) close(f.block) require.NoError(t, q.close()) diff --git a/bundle/direct/oprecorder.go b/bundle/direct/oprecorder.go index 87c6e48bb9c..775ae930998 100644 --- a/bundle/direct/oprecorder.go +++ b/bundle/direct/oprecorder.go @@ -52,20 +52,6 @@ type recordedOperation struct { state json.RawMessage } -// mergeOperation folds a newer write onto one still waiting to be uploaded, so a -// resource costs one request no matter how many times it writes state. The newer -// write describes the resource as it now stands, so its fields win. -// -// The action is the exception: it comes from the older write, because the service -// fixes action_type when the operation is created and rejects it in an update mask -// (only state, error_message, resource_id and status are updatable). Keeping the -// older one is what makes the merged upload a single create that still reports how -// the resource got here - a recreate whose second write is a create stays a recreate. -func mergeOperation(older, newer recordedOperation) recordedOperation { - newer.action = older.action - return newer -} - // newStateOperation describes a state write for upload. state is the serialized // RecordedState envelope the state DB just persisted, and nil for a delete, where // the resource is gone. It errors when the state exceeds maxOperationStateSize. From 3d0b43cdca6b865f1aaaed630804fede649c8082 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 11 Aug 2026 10:53:05 +0000 Subject: [PATCH 078/125] bundle: create the deployment version only once the deploy is approved The version was created before the plan, because the plan stamps it onto the resources it is computed from. That meant a deploy the user declined still claimed a version number, which AbortVersion then had to mark FORCE_ABORT so the history did not read like a deploy that succeeded or failed. Split the two: PrepareDeployment settles the deployment and works out the version number from last_version_id, which is all the stamp needs, and CreateVersion claims it after approval. The API takes the version id from the caller and rejects one that is not greater than the deployment's most recent, so claiming the number early is safe - a deploy that took it in between is reported rather than overwritten. AbortVersion is gone: a declined deploy now creates nothing, and the number is left for the next deploy. The declined path had no acceptance coverage, which is why removing it broke no test; bundle/dms/declined-deploy now pins it. Also drops build/pybin, local python/uv shims that a git add -A swept into the previous commit. They hardcode a home path and are not needed to run the suite. Co-authored-by: Isaac --- .../bundle/dms/declined-deploy/databricks.yml | 11 ++ .../bundle/dms/declined-deploy/out.test.toml | 3 + .../bundle/dms/declined-deploy/output.txt | 83 ++++++++++++++ acceptance/bundle/dms/declined-deploy/script | 16 +++ build/pybin/python | 1 - build/pybin/python3 | 1 - build/pybin/uv | 9 -- bundle/phases/deploy.go | 43 ++++--- libs/dms/recorder.go | 105 +++++++++++------- libs/dms/recorder_test.go | 39 +++++++ 10 files changed, 235 insertions(+), 76 deletions(-) create mode 100644 acceptance/bundle/dms/declined-deploy/databricks.yml create mode 100644 acceptance/bundle/dms/declined-deploy/out.test.toml create mode 100644 acceptance/bundle/dms/declined-deploy/output.txt create mode 100644 acceptance/bundle/dms/declined-deploy/script delete mode 120000 build/pybin/python delete mode 120000 build/pybin/python3 delete mode 100755 build/pybin/uv diff --git a/acceptance/bundle/dms/declined-deploy/databricks.yml b/acceptance/bundle/dms/declined-deploy/databricks.yml new file mode 100644 index 00000000000..dea4c2d886f --- /dev/null +++ b/acceptance/bundle/dms/declined-deploy/databricks.yml @@ -0,0 +1,11 @@ +bundle: + name: dms-declined-deploy + +experimental: + record_deployment_history: true + +resources: + schemas: + foo: + name: dms_declined_deploy_schema + catalog_name: main diff --git a/acceptance/bundle/dms/declined-deploy/out.test.toml b/acceptance/bundle/dms/declined-deploy/out.test.toml new file mode 100644 index 00000000000..7daaf6fd56a --- /dev/null +++ b/acceptance/bundle/dms/declined-deploy/out.test.toml @@ -0,0 +1,3 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/dms/declined-deploy/output.txt b/acceptance/bundle/dms/declined-deploy/output.txt new file mode 100644 index 00000000000..1cfa478e464 --- /dev/null +++ b/acceptance/bundle/dms/declined-deploy/output.txt @@ -0,0 +1,83 @@ + +=== Deploy a schema, so the deployment and its first version exist +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-declined-deploy/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> print_requests.py //api/2.0/bundle --sort +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments", + "body": { + "initial_parent_path": "/Workspace/Users/[USERNAME]/.bundle/dms-declined-deploy/default/state", + "target_name": "default" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions", + "q": { + "version_id": "1" + }, + "body": { + "cli_version": "[CLI_VERSION]", + "version_type": "VERSION_TYPE_DEPLOY", + "target_name": "default", + "display_name": "dms-declined-deploy", + "workspace_info": { + "file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-declined-deploy/default/files", + "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-declined-deploy/default" + } + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/complete", + "body": { + "completion_reason": "VERSION_COMPLETE_SUCCESS" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations", + "q": { + "resource_key": "schemas.foo" + }, + "body": { + "action_type": "OPERATION_ACTION_TYPE_CREATE", + "resource_id": "main.dms_declined_deploy_schema", + "resource_key": "schemas.foo", + "state": "{\"state\":{\"catalog_name\":\"main\",\"name\":\"dms_declined_deploy_schema\"}}", + "status": "OPERATION_STATUS_SUCCEEDED" + } +} + +=== A destructive change without --auto-approve is declined: this console cannot prompt +>>> update_file.py databricks.yml catalog_name: main catalog_name: other + +>>> musterr [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-declined-deploy/default/files... + +This action will result in the deletion or recreation of the following UC schemas. Any underlying data may be lost: + recreate resources.schemas.foo +Error: the deployment requires destructive actions, but the current console does not support prompting. +Deleting data assets such as schemas, pipelines, or volumes may cause permanent data loss and should be carefully reviewed. +To proceed, use --auto-approve after reviewing the plan above. + + +=== Nothing was recorded for the declined deploy - no version, so none to abort +>>> print_requests.py //api/2.0/bundle --sort + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.schemas.foo + +This action will result in the deletion of the following UC schemas. Any underlying data may be lost: + delete resources.schemas.foo + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/dms-declined-deploy/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/dms/declined-deploy/script b/acceptance/bundle/dms/declined-deploy/script new file mode 100644 index 00000000000..74be4c9eec6 --- /dev/null +++ b/acceptance/bundle/dms/declined-deploy/script @@ -0,0 +1,16 @@ +title "Deploy a schema, so the deployment and its first version exist" +trace $CLI bundle deploy +trace print_requests.py //api/2.0/bundle --sort + +title "A destructive change without --auto-approve is declined: this console cannot prompt" +# Changing the catalog recreates the schema, which needs approval. +trace update_file.py databricks.yml "catalog_name: main" "catalog_name: other" +trace musterr $CLI bundle deploy + +title "Nothing was recorded for the declined deploy - no version, so none to abort" +# The version number it would have used is left for the next deploy to take, so the +# history has no entry that reads like a deploy which failed or did nothing. +trace print_requests.py //api/2.0/bundle --sort + +trace $CLI bundle destroy --auto-approve +rm -f out.requests.txt diff --git a/build/pybin/python b/build/pybin/python deleted file mode 120000 index 40a7c693444..00000000000 --- a/build/pybin/python +++ /dev/null @@ -1 +0,0 @@ -/home/shreyas.goenka/.pyenv/versions/3.11.11/bin/python3 \ No newline at end of file diff --git a/build/pybin/python3 b/build/pybin/python3 deleted file mode 120000 index 40a7c693444..00000000000 --- a/build/pybin/python3 +++ /dev/null @@ -1 +0,0 @@ -/home/shreyas.goenka/.pyenv/versions/3.11.11/bin/python3 \ No newline at end of file diff --git a/build/pybin/uv b/build/pybin/uv deleted file mode 100755 index 7b99bf07df5..00000000000 --- a/build/pybin/uv +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash -# Wrapper to provide uv python find functionality -if [[ "$1" == "python" && "$2" == "find" ]]; then - # Return the direct path to python3 - echo "/home/shreyas.goenka/.pyenv/versions/3.11.11/bin/python3" -else - # For other uv commands, try the real uv - /home/shreyas.goenka/.local/bin/uv "$@" -fi diff --git a/bundle/phases/deploy.go b/bundle/phases/deploy.go index af1f6331305..17326ef24ba 100644 --- a/bundle/phases/deploy.go +++ b/bundle/phases/deploy.go @@ -167,8 +167,7 @@ func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHand // lock is acquired here // // Set up DMS recording of this deployment as a version. The version itself is - // created further down, before the plan is computed - see the comment there for - // why it cannot wait for approval. CompleteVersion is deferred before + // created once the deploy is approved. CompleteVersion is deferred before // lock.Release so it runs while the lock is still held (defers run // last-in-first-out), and is a no-op until CreateVersion has run. recorder, err := newDeploymentRecorder(ctx, b, stateEngine, dms.VersionTypeDeploy) @@ -230,15 +229,14 @@ func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHand return } - // Create the version before planning: the plan snapshots the resource config, so - // the version has to be stamped on before it is computed or the applied resources - // would not carry it. + // Settle the deployment and the version number it will use before planning: the + // plan snapshots the resource config, so both have to be stamped on before it is + // computed or the applied resources would not carry them. The version itself is + // created after approval. // - // Creating it is also what takes the deployment's lock server-side, so a deploy - // that loses a race pays for the upload above before being turned away. Moving it - // earlier does not work: on a first deploy the deployment record is registered - // under the state directory, which the upload is what creates. - if err := recorder.CreateVersion(ctx); err != nil { + // This cannot move earlier: on a first deploy the deployment is registered under + // the state directory, which the upload above is what creates. + if err := recorder.PrepareDeployment(ctx); err != nil { logdiag.LogError(ctx, err) return } @@ -252,9 +250,6 @@ func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHand if logdiag.HasError(ctx) { return } - // Printed here rather than after the deploy so the user can follow the version - // while it runs, and still has the link if the deploy fails partway. - logDeploymentVersion(ctx, b, recorder) } planFromFile := plan != nil @@ -296,17 +291,12 @@ func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHand haveApproval, approvalErr := approvalForDeploy(ctx, b, plan) if !haveApproval { - // Nothing was applied, so the version records an abort rather than a failure. - // It cannot simply be left uncreated: it is stamped onto the resources the plan - // is computed from, so it has to exist before the prompt. Aborting first also - // makes the deferred CompleteVersion a no-op. + // No version was created, so there is nothing to complete: the deferred + // CompleteVersion is a no-op until CreateVersion has run. The version number + // this deploy would have used is simply left for the next one to take. // // Both outcomes land here - the user declining, and a console that cannot // prompt at all, which returns an error instead. - if err := recorder.AbortVersion(ctx); err != nil { - logdiag.LogError(ctx, err) - return - } if approvalErr != nil { logdiag.LogError(ctx, approvalErr) return @@ -315,8 +305,15 @@ func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHand return } - // Record operations under the version created before planning, so DMS holds - // the deployed resource state. + // Create the version the plan was stamped with. Doing it here rather than before + // the prompt means a declined deploy never claims a version number. + if err := recorder.CreateVersion(ctx); err != nil { + logdiag.LogError(ctx, err) + return + } + logDeploymentVersion(ctx, b, recorder) + + // Record operations under that version, so DMS holds the deployed resource state. setOperationRecorder(ctx, b, recorder) deployCore(ctx, b, plan, stateEngine, requestedEngine, recorder) diff --git a/libs/dms/recorder.go b/libs/dms/recorder.go index ff195679c95..a6a0442156c 100644 --- a/libs/dms/recorder.go +++ b/libs/dms/recorder.go @@ -91,9 +91,16 @@ type Recorder struct { versionType VersionType metadata Metadata - // populated by CreateVersion - versionNum int64 - stopHeartbeat context.CancelFunc + // populated by PrepareDeployment: the version number this deploy intends to + // create. It is known before the version exists so it can be stamped onto the + // resources the plan is computed from. + versionNum int64 + previousVersionID string + + // populated by CreateVersion, once the version actually exists. A deploy the user + // declines never gets here, so there is nothing to complete or heartbeat. + versionCreated bool + stopHeartbeat context.CancelFunc // completed makes CompleteVersion idempotent, so a caller that completes the // version early can still defer it unconditionally. @@ -169,18 +176,37 @@ func (r *Recorder) CreateVersion(ctx context.Context) error { if r == nil { return nil } - - versionID, err := r.createDeploymentVersion(ctx) - if err != nil { - return err + // A deploy calls PrepareDeployment first, because it needs the version number to + // stamp onto the plan. A destroy has no such need, so settle it here instead. + if r.versionNum == 0 { + if err := r.PrepareDeployment(ctx); err != nil { + return err + } } - versionNum, err := strconv.ParseInt(versionID, 10, 64) + versionID := strconv.FormatInt(r.versionNum, 10) + + // The server rejects the call unless versionID is numerically greater than + // last_version_id and previous_version_id matches it. That is what makes claiming + // the number up front safe: another deploy that took it between PrepareDeployment + // and here is reported rather than overwritten. + version, err := r.versions.CreateVersion(ctx, r.deploymentID, versionID, createVersionRequest{ + CliVersion: build.GetInfo().Version, + VersionType: r.versionType, + TargetName: r.metadata.TargetName, + DisplayName: r.metadata.DisplayName, + PreviousVersionId: r.previousVersionID, + DeploymentMode: r.metadata.Mode, + GitInfo: r.metadata.Git, + WorkspaceInfo: r.metadata.Workspace, + }) if err != nil { - return fmt.Errorf("failed to parse version ID %q: %w", versionID, err) + return fmt.Errorf("failed to create deployment version: %w", err) } - r.versionNum = versionNum + + r.versionCreated = true r.stopHeartbeat = startHeartbeat(ctx, r.svc, r.deploymentID, versionID) + log.Infof(ctx, "Created deployment version: deployment=%s version=%s", r.deploymentID, version.VersionId) return nil } @@ -195,16 +221,8 @@ func (r *Recorder) CompleteVersion(ctx context.Context, success bool) error { return r.completeVersion(ctx, reason) } -// AbortVersion completes the version as aborted, for a deploy the user declined at -// the approval prompt. The version has to exist by then - it is stamped onto the -// resources the plan is computed from - so this says nothing was applied rather than -// leaving a version that reads like a deploy that failed. -func (r *Recorder) AbortVersion(ctx context.Context) error { - return r.completeVersion(ctx, bundledeployments.VersionCompleteVersionCompleteForceAbort) -} - func (r *Recorder) completeVersion(ctx context.Context, reason bundledeployments.VersionComplete) error { - if r == nil || r.versionNum == 0 || r.completed { + if r == nil || !r.versionCreated || r.completed { return nil } r.completed = true @@ -240,10 +258,31 @@ func (r *Recorder) completeVersion(ctx context.Context, reason bundledeployments // createDeploymentVersion ensures the deployment record exists, then creates a new // version under it: with no ID it creates the deployment, otherwise it reads the // existing one for the next version number. -func (r *Recorder) createDeploymentVersion(ctx context.Context) (versionID string, err error) { - // The version this one supersedes, sent as the concurrency check. Empty for a - // deployment's first version. - var previousVersionID string +// PrepareDeployment makes sure the deployment exists and works out the version number +// this deploy will create, without creating it. Both are needed before the plan, which +// stamps them onto the resources it is computed from; the version itself is not created +// until CreateVersion, so a deploy the user declines never claims one. +func (r *Recorder) PrepareDeployment(ctx context.Context) error { + if r == nil { + return nil + } + + versionID, err := r.resolveNextVersion(ctx) + if err != nil { + return err + } + + versionNum, err := strconv.ParseInt(versionID, 10, 64) + if err != nil { + return fmt.Errorf("failed to parse version ID %q: %w", versionID, err) + } + r.versionNum = versionNum + return nil +} + +// resolveNextVersion creates the deployment if this is the first deploy, and returns +// the version ID to create under it. +func (r *Recorder) resolveNextVersion(ctx context.Context) (versionID string, err error) { if r.deploymentID != "" { // The ID came from a BUNDLE_DEPLOYMENT node that get-status returned, and by // design the service has a deployment for every such node, so a not-found @@ -266,7 +305,7 @@ func (r *Recorder) createDeploymentVersion(ctx context.Context) (versionID strin return "", fmt.Errorf("failed to parse last_version_id %q: %w", dep.LastVersionId, parseErr) } versionID = strconv.FormatInt(lastVersion+1, 10) - previousVersionID = dep.LastVersionId + r.previousVersionID = dep.LastVersionId } } else { // First deploy: create the deployment so the server assigns an ID. @@ -289,24 +328,6 @@ func (r *Recorder) createDeploymentVersion(ctx context.Context) (versionID strin versionID = "1" } - // The server rejects the call unless versionID is numerically greater than - // last_version_id and previous_version_id matches it, so a deploy racing - // another is rejected rather than overwriting it. - version, versionErr := r.versions.CreateVersion(ctx, r.deploymentID, versionID, createVersionRequest{ - CliVersion: build.GetInfo().Version, - VersionType: r.versionType, - TargetName: r.metadata.TargetName, - DisplayName: r.metadata.DisplayName, - PreviousVersionId: previousVersionID, - DeploymentMode: r.metadata.Mode, - GitInfo: r.metadata.Git, - WorkspaceInfo: r.metadata.Workspace, - }) - if versionErr != nil { - return "", fmt.Errorf("failed to create deployment version: %w", versionErr) - } - - log.Infof(ctx, "Created deployment version: deployment=%s version=%s", r.deploymentID, version.VersionId) return versionID, nil } diff --git a/libs/dms/recorder_test.go b/libs/dms/recorder_test.go index 51ce63e5971..b247cf2edd8 100644 --- a/libs/dms/recorder_test.go +++ b/libs/dms/recorder_test.go @@ -237,6 +237,45 @@ func TestRecorderCompleteVersionNoOpWithoutCreateVersion(t *testing.T) { assert.Empty(t, f.completed) } +func TestRecorderPrepareDeploymentClaimsNoVersion(t *testing.T) { + // A deploy the user declines prepares but never creates: the version number is + // known, so the plan can be stamped with it, but no version exists to complete and + // the number is left for the next deploy to take. + f := &fakeDMS{ + getDeployment: func(id string) (*bundledeployments.Deployment, error) { + return &bundledeployments.Deployment{Name: "deployments/" + id, LastVersionId: "4"}, nil + }, + } + r := NewRecorder(RecorderOptions{Service: f, Versions: fakeVersions{requests: &f.versions}, DeploymentID: "stored-id", StatePath: testStatePath, Metadata: Metadata{TargetName: "dev", DisplayName: testDisplayName}, VersionType: VersionTypeDeploy}) + + require.NoError(t, r.PrepareDeployment(t.Context())) + + assert.Equal(t, int64(5), r.Version()) + assert.Empty(t, f.versions, "no version created") + + require.NoError(t, r.CompleteVersion(t.Context(), true)) + assert.Empty(t, f.completed, "nothing to complete") +} + +func TestRecorderCreateVersionUsesThePreparedNumber(t *testing.T) { + f := &fakeDMS{ + getDeployment: func(id string) (*bundledeployments.Deployment, error) { + return &bundledeployments.Deployment{Name: "deployments/" + id, LastVersionId: "4"}, nil + }, + } + r := NewRecorder(RecorderOptions{Service: f, Versions: fakeVersions{requests: &f.versions}, DeploymentID: "stored-id", StatePath: testStatePath, Metadata: Metadata{TargetName: "dev", DisplayName: testDisplayName}, VersionType: VersionTypeDeploy}) + + require.NoError(t, r.PrepareDeployment(t.Context())) + require.NoError(t, r.CreateVersion(t.Context())) + + // The version created is the one the plan was stamped with, and it reports the + // version it supersedes so the service rejects a racing deploy. + require.Len(t, f.versions, 1) + assert.Equal(t, "5", f.versions[0].versionID) + assert.Equal(t, "4", f.versions[0].body.PreviousVersionId) + assert.Equal(t, int64(5), r.Version()) +} + func TestDeploymentIDFromName(t *testing.T) { id, err := deploymentIDFromName("deployments/abc-123") require.NoError(t, err) From 3fa81b5399d9ffaf58ea54aa43716b4203fdd57d Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 11 Aug 2026 11:05:55 +0000 Subject: [PATCH 079/125] acceptance: show the emptied-resource diff, and trim depends-on emptied-resource dumped requests only after the second deploy, so the DELETE had nothing to be read against. Dump after each deploy: the first shows the grants node created with state, the second shows it deleted. depends-on asserted the delete order after wiping local state, which is really about the destroy path rather than depends_on surviving the round trip. Keep the part that records depends_on alongside the config. Co-authored-by: Isaac --- acceptance/bundle/dms/depends-on/output.txt | 11 +--- acceptance/bundle/dms/depends-on/script | 13 +---- .../bundle/dms/emptied-resource/output.txt | 57 ++++++++++--------- acceptance/bundle/dms/emptied-resource/script | 5 +- 4 files changed, 38 insertions(+), 48 deletions(-) diff --git a/acceptance/bundle/dms/depends-on/output.txt b/acceptance/bundle/dms/depends-on/output.txt index c45a040dab9..a4be8978685 100644 --- a/acceptance/bundle/dms/depends-on/output.txt +++ b/acceptance/bundle/dms/depends-on/output.txt @@ -15,9 +15,9 @@ Deployment complete! }, "body": { "action_type": "OPERATION_ACTION_TYPE_CREATE", - "resource_id": "[CHILD_ID]", + "resource_id": "[NUMID]", "resource_key": "jobs.child", - "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-depends-on/default/state/metadata.json\",\"version_id\":\"1\"},\"description\":\"depends on [PARENT_ID]\",\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"child\",\"queue\":{\"enabled\":true}},\"depends_on\":[{\"node\":\"resources.jobs.parent\",\"label\":\"${resources.jobs.parent.id}\"}]}", + "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-depends-on/default/state/metadata.json\",\"version_id\":\"1\"},\"description\":\"depends on [NUMID]\",\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"child\",\"queue\":{\"enabled\":true}},\"depends_on\":[{\"node\":\"resources.jobs.parent\",\"label\":\"${resources.jobs.parent.id}\"}]}", "status": "OPERATION_STATUS_SUCCEEDED" } } @@ -29,14 +29,13 @@ Deployment complete! }, "body": { "action_type": "OPERATION_ACTION_TYPE_CREATE", - "resource_id": "[PARENT_ID]", + "resource_id": "[NUMID]", "resource_key": "jobs.parent", "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-depends-on/default/state/metadata.json\",\"version_id\":\"1\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"parent\",\"queue\":{\"enabled\":true}}}", "status": "OPERATION_STATUS_SUCCEEDED" } } -=== Wipe the local state, then destroy: depends_on comes back from DMS, so the child is still deleted before the parent it references >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: delete resources.jobs.child @@ -46,7 +45,3 @@ All files and directories at the following location will be deleted: /Workspace/ Deleting files... Destroy complete! - ->>> print_requests.py //jobs --oneline -{"method": "POST", "path": "/api/2.2/jobs/delete", "body": {"job_id": [CHILD_ID]}} -{"method": "POST", "path": "/api/2.2/jobs/delete", "body": {"job_id": [PARENT_ID]}} diff --git a/acceptance/bundle/dms/depends-on/script b/acceptance/bundle/dms/depends-on/script index 95e6a8c2772..a7bacaf5d67 100644 --- a/acceptance/bundle/dms/depends-on/script +++ b/acceptance/bundle/dms/depends-on/script @@ -2,16 +2,5 @@ title "Deploy a job that references another: depends_on is recorded alongside th trace $CLI bundle deploy trace print_requests.py //versions/1/operations --sort -title "Wipe the local state, then destroy: depends_on comes back from DMS, so the child is still deleted before the parent it references" - -# Name both ids while state still exists, so the delete order below is readable. Without -# this both render as [NUMID] and the golden cannot show which job went first. -read_id.py parent > /dev/null -read_id.py child > /dev/null - -rm -rf .databricks trace $CLI bundle destroy --auto-approve - -# Not --sort: the order is the assertion. The child references the parent, so it has to be -# deleted first, and the golden shows [CHILD_ID] before [PARENT_ID]. -trace print_requests.py //jobs --oneline +rm -f out.requests.txt diff --git a/acceptance/bundle/dms/emptied-resource/output.txt b/acceptance/bundle/dms/emptied-resource/output.txt index cf9f021cc94..a039e52158c 100644 --- a/acceptance/bundle/dms/emptied-resource/output.txt +++ b/acceptance/bundle/dms/emptied-resource/output.txt @@ -1,13 +1,5 @@ -=== Deploy a schema with one grant, then revoke it so the grants node empties out ->>> [CLI] bundle deploy -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-emptied-resource/default/files... -Deploying resources... -Updating deployment state... -Deployment complete! - ->>> update_file.py databricks.yml grants: [{principal: someone@example.com, privileges: [USE_SCHEMA]}] grants: [] - +=== Deploy a schema with one grant: the grants node is recorded with its state >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-emptied-resource/default/files... Deploying resources... @@ -40,24 +32,6 @@ Deployment complete! } } } -{ - "method": "POST", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions", - "q": { - "version_id": "2" - }, - "body": { - "cli_version": "[CLI_VERSION]", - "version_type": "VERSION_TYPE_DEPLOY", - "target_name": "default", - "display_name": "dms-emptied-resource", - "previous_version_id": "1", - "workspace_info": { - "file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-emptied-resource/default/files", - "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-emptied-resource/default" - } - } -} { "method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/complete", @@ -93,6 +67,35 @@ Deployment complete! "status": "OPERATION_STATUS_SUCCEEDED" } } + +=== Revoke the grant, so the grants node empties out +>>> update_file.py databricks.yml grants: [{principal: someone@example.com, privileges: [USE_SCHEMA]}] grants: [] + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-emptied-resource/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> print_requests.py //api/2.0/bundle --sort +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions", + "q": { + "version_id": "2" + }, + "body": { + "cli_version": "[CLI_VERSION]", + "version_type": "VERSION_TYPE_DEPLOY", + "target_name": "default", + "display_name": "dms-emptied-resource", + "previous_version_id": "1", + "workspace_info": { + "file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-emptied-resource/default/files", + "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-emptied-resource/default" + } + } +} { "method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/complete", diff --git a/acceptance/bundle/dms/emptied-resource/script b/acceptance/bundle/dms/emptied-resource/script index 049bbc9cd80..929cf6d8cc1 100644 --- a/acceptance/bundle/dms/emptied-resource/script +++ b/acceptance/bundle/dms/emptied-resource/script @@ -1,5 +1,8 @@ -title "Deploy a schema with one grant, then revoke it so the grants node empties out" +title "Deploy a schema with one grant: the grants node is recorded with its state" trace $CLI bundle deploy +trace print_requests.py //api/2.0/bundle --sort + +title "Revoke the grant, so the grants node empties out" trace update_file.py databricks.yml 'grants: [{principal: someone@example.com, privileges: [USE_SCHEMA]}]' 'grants: []' trace $CLI bundle deploy From 926988231f24b22f94c9b4e31a3e49288271bc4d Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 11 Aug 2026 11:18:05 +0000 Subject: [PATCH 080/125] bundle: a failure must not erase the state a resource already recorded A create whose WaitAfterCreate fails has already written state for a resource that exists remotely. Recording the failure sent the whole update mask with the failure's own empty state and id, which cleared both - and a resource with no state is dropped from the deployment, so the next plan tried to create it again. That is what broke bundle/resources/job_runs/failed_run and interrupted_run under recording. An update for a failure now masks only error_message and status, so what the earlier write recorded stands. A failure that arrives before any operation exists still creates one carrying the prior state, which it has to: the service rejects state without a resource id. The fake server required update_mask and then ignored it, applying all four fields every time - so it could not have caught this. It now honours the mask, rejects a path that is not updatable, and checks its invariants against the operation the update leaves behind rather than the request. Co-authored-by: Isaac --- .../job_runs/failed_run/out.test.toml | 1 + .../job_runs/interrupted_run/out.test.toml | 1 + .../resources/job_runs/wait/out.test.toml | 1 + bundle/direct/opclient.go | 10 ++-- bundle/direct/opqueue_test.go | 7 --- bundle/direct/oprecorder.go | 25 +++++++- bundle/direct/oprecorder_test.go | 49 ++++++++++++++- libs/testserver/bundle.go | 60 ++++++++++++++++--- 8 files changed, 131 insertions(+), 23 deletions(-) diff --git a/acceptance/bundle/resources/job_runs/failed_run/out.test.toml b/acceptance/bundle/resources/job_runs/failed_run/out.test.toml index 8c52d40aa2d..6c47aa9e8d1 100644 --- a/acceptance/bundle/resources/job_runs/failed_run/out.test.toml +++ b/acceptance/bundle/resources/job_runs/failed_run/out.test.toml @@ -1,3 +1,4 @@ Cloud = true RequiresUnityCatalog = true EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/job_runs/interrupted_run/out.test.toml b/acceptance/bundle/resources/job_runs/interrupted_run/out.test.toml index 0938e678987..2c6699da193 100644 --- a/acceptance/bundle/resources/job_runs/interrupted_run/out.test.toml +++ b/acceptance/bundle/resources/job_runs/interrupted_run/out.test.toml @@ -1,2 +1,3 @@ Cloud = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/job_runs/wait/out.test.toml b/acceptance/bundle/resources/job_runs/wait/out.test.toml index 8c52d40aa2d..6c47aa9e8d1 100644 --- a/acceptance/bundle/resources/job_runs/wait/out.test.toml +++ b/acceptance/bundle/resources/job_runs/wait/out.test.toml @@ -1,3 +1,4 @@ Cloud = true RequiresUnityCatalog = true EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/bundle/direct/opclient.go b/bundle/direct/opclient.go index 8df7cd6a862..0c1e4346c10 100644 --- a/bundle/direct/opclient.go +++ b/bundle/direct/opclient.go @@ -41,10 +41,12 @@ type updateOperationRequest struct { SequenceId string `json:"sequence_id,omitempty"` } -// operationClient records operations under a deployment version. +// operationClient records operations under a deployment version. UpdateOperation +// takes the fields to update, because a failure updates fewer of them than a state +// write does. type operationClient interface { CreateOperation(ctx context.Context, parent, resourceKey string, op bundledeployments.Operation) (operationResponse, error) - UpdateOperation(ctx context.Context, parent, resourceKey string, body updateOperationRequest) (operationResponse, error) + UpdateOperation(ctx context.Context, parent, resourceKey string, fields []string, body updateOperationRequest) (operationResponse, error) } // apiOperationClient talks to the operations API through the workspace client. @@ -70,12 +72,12 @@ func (a *apiOperationClient) CreateOperation(ctx context.Context, parent, resour return result, nil } -func (a *apiOperationClient) UpdateOperation(ctx context.Context, parent, resourceKey string, body updateOperationRequest) (operationResponse, error) { +func (a *apiOperationClient) UpdateOperation(ctx context.Context, parent, resourceKey string, fields []string, body updateOperationRequest) (operationResponse, error) { var result operationResponse path := fmt.Sprintf("/api/2.0/bundle/%s/operations/%s", parent, resourceKey) err := a.client.Do(ctx, http.MethodPatch, path, auth.WorkspaceIDHeaders(a.client.Config), - map[string]any{"update_mask": strings.Join(updatableFields, ",")}, + map[string]any{"update_mask": strings.Join(fields, ",")}, body, &result) if err != nil { return operationResponse{}, err diff --git a/bundle/direct/opqueue_test.go b/bundle/direct/opqueue_test.go index f4be818390a..0a49cc1b99f 100644 --- a/bundle/direct/opqueue_test.go +++ b/bundle/direct/opqueue_test.go @@ -160,13 +160,6 @@ func TestOperationQueueMergesWritesQueuedBehindAnUpload(t *testing.T) { } func TestOperationQueueMergedRecreateKeepsItsActionType(t *testing.T) { - // Both writes of a recreate report it as a recreate (see DeploymentUnit.Create), - // so the merged upload carries that action however the two are combined. It has to: - // the service fixes action_type when the operation is created and rejects it in an - // update mask, so a merged pair that reported "create" would leave the history - // saying the resource was merely created. - // - // Every worker is parked so both writes land in pending and merge before upload. f := &fakeUploader{block: make(chan struct{}), started: make(chan string, operationUploadWorkers+2)} q := newOperationQueue(t.Context(), f) diff --git a/bundle/direct/oprecorder.go b/bundle/direct/oprecorder.go index 775ae930998..d8ed541a915 100644 --- a/bundle/direct/oprecorder.go +++ b/bundle/direct/oprecorder.go @@ -174,6 +174,14 @@ func newOperationRecorder(ops operationClient, deploymentID string, version int6 // change. resource_id is included because a recreate learns a new one. var updatableFields = []string{"state", "error_message", "resource_id", "status"} +// failureFields are the fields a failure changes on an operation that already exists. +// It deliberately leaves state and resource_id alone: the resource was written before +// the step that failed, so what is already recorded describes something that exists, +// and a failure carries no state of its own to replace it with. Including them would +// clear both - the service takes the update mask literally - and a resource with no +// state is dropped from the deployment, so the next plan would try to create it again. +var failureFields = []string{"error_message", "status"} + func (r *operationRecorder) upload(ctx context.Context, resourceKey string, op recordedOperation) error { // The read path re-adds the prefix; see dstate.ResourceKeyPrefix. dmsKey := strings.TrimPrefix(resourceKey, dstate.ResourceKeyPrefix) @@ -207,13 +215,26 @@ func (r *operationRecorder) upload(ctx context.Context, resourceKey string, op r if recorded { // Only the masked fields and sequence_id are read on an update; action_type // stays as the operation was created, so sending it would just be misleading. - result, err = r.ops.UpdateOperation(ctx, r.parent, dmsKey, updateOperationRequest{ + body := updateOperationRequest{ State: operation.State, ErrorMessage: operation.ErrorMessage, ResourceId: operation.ResourceId, Status: operation.Status, SequenceId: sequenceID, - }) + } + fields := updatableFields + if op.status == bundledeployments.OperationStatusOperationStatusFailed { + // Mark the existing record failed and leave the rest of it alone; see + // failureFields. A failure that arrives before any operation exists still + // goes through CreateOperation below, carrying the prior state. + fields = failureFields + body = updateOperationRequest{ + ErrorMessage: operation.ErrorMessage, + Status: operation.Status, + SequenceId: sequenceID, + } + } + result, err = r.ops.UpdateOperation(ctx, r.parent, dmsKey, fields, body) } else { result, err = r.ops.CreateOperation(ctx, r.parent, dmsKey, operation) } diff --git a/bundle/direct/oprecorder_test.go b/bundle/direct/oprecorder_test.go index d368fe47c9d..2f8b12c3705 100644 --- a/bundle/direct/oprecorder_test.go +++ b/bundle/direct/oprecorder_test.go @@ -22,6 +22,7 @@ type fakeOpCall struct { resourceKey string op bundledeployments.Operation update updateOperationRequest + fields []string } type fakeOpClient struct { @@ -38,10 +39,10 @@ func (f *fakeOpClient) CreateOperation(ctx context.Context, parent, resourceKey return operationResponse{SequenceId: f.sequence}, nil } -func (f *fakeOpClient) UpdateOperation(ctx context.Context, parent, resourceKey string, body updateOperationRequest) (operationResponse, error) { +func (f *fakeOpClient) UpdateOperation(ctx context.Context, parent, resourceKey string, fields []string, body updateOperationRequest) (operationResponse, error) { f.mu.Lock() defer f.mu.Unlock() - f.calls = append(f.calls, fakeOpCall{method: "update", parent: parent, resourceKey: resourceKey, update: body}) + f.calls = append(f.calls, fakeOpCall{method: "update", parent: parent, resourceKey: resourceKey, update: body, fields: fields}) return operationResponse{SequenceId: f.sequence}, nil } @@ -92,6 +93,50 @@ func TestOperationRecorderUpdatesSecondWriteForSameResource(t *testing.T) { require.NotNil(t, f.calls[1].update.State) } +func TestOperationRecorderFailureAfterAStateWriteKeepsTheState(t *testing.T) { + // A create whose WaitAfterCreate fails has already written state for a resource + // that exists remotely. Updating the operation must only mark it failed: sending + // the failure's own empty state and id would clear both, and a resource with no + // state is dropped from the deployment, so the next plan would re-create it. + f := &fakeOpClient{sequence: "3"} + r := newOperationRecorder(f, "dep-1", 2) + + uploadOne(t, r, "resources.job_runs.my_run", deployplan.Create, "run-1", envelope(t, "the run")) + + failed, err := newFailedOperation(deployplan.Create, "", nil, errors.New("run did not succeed: FAILED")) + require.NoError(t, err) + require.NoError(t, r.upload(t.Context(), "resources.job_runs.my_run", failed)) + + require.Len(t, f.calls, 2) + assert.Equal(t, "create", f.calls[0].method) + + update := f.calls[1] + assert.Equal(t, "update", update.method) + assert.Equal(t, failureFields, update.fields) + assert.Equal(t, bundledeployments.OperationStatusOperationStatusFailed, update.update.Status) + assert.Equal(t, "run did not succeed: FAILED", update.update.ErrorMessage) + // Neither is in the mask, so what the create recorded stands. + assert.Nil(t, update.update.State) + assert.Empty(t, update.update.ResourceId) +} + +func TestOperationRecorderFailureBeforeAnyWriteCarriesPriorState(t *testing.T) { + // Nothing has been recorded for the resource, so the failure creates the operation + // and has to carry the prior state itself - the resource still exists, and the + // service rejects an operation that records state without an id. + f := &fakeOpClient{sequence: "1"} + r := newOperationRecorder(f, "dep-1", 2) + + failed, err := newFailedOperation(deployplan.Update, "main.some_schema", envelope(t, "before"), errors.New("boom")) + require.NoError(t, err) + require.NoError(t, r.upload(t.Context(), "resources.schemas.foo", failed)) + + require.Len(t, f.calls, 1) + assert.Equal(t, "create", f.calls[0].method) + assert.Equal(t, "main.some_schema", f.calls[0].op.ResourceId) + require.NotNil(t, f.calls[0].op.State) +} + func TestOperationRecorderTracksSequencePerResource(t *testing.T) { // A different resource has its own operation, so its first write creates. f := &fakeOpClient{sequence: "1"} diff --git a/libs/testserver/bundle.go b/libs/testserver/bundle.go index 706fdb8efb5..7c7196026c1 100644 --- a/libs/testserver/bundle.go +++ b/libs/testserver/bundle.go @@ -6,6 +6,7 @@ import ( "path" "slices" "strconv" + "strings" "github.com/databricks/databricks-sdk-go/service/bundledeployments" "github.com/databricks/databricks-sdk-go/service/workspace" @@ -20,6 +21,11 @@ import ( // the CLI so a test would catch the CLI drifting from the service. const dmsDeploymentNodeName = "resources.deployment.json" +// dmsUpdatableOperationFields are the update_mask paths UpdateOperation accepts. Any +// other path is rejected, and action_type in particular is fixed when the operation is +// created. +var dmsUpdatableOperationFields = []string{"state", "error_message", "resource_id", "status"} + // dmsDeployment holds a deployment record together with the versions and // resources recorded under it, so the read APIs (ListVersions/ListResources) // can serve back what deploys wrote. @@ -374,6 +380,18 @@ func (s *FakeWorkspace) UpdateOperation(req Request, deploymentID, versionID, re if updateMask == "" { return dmsInvalidArgument("update_mask is required") } + // Only the paths named in the mask are written; every other field of the stored + // operation is left as it is. A caller that omits state keeps the state already + // recorded, which is how a failure marks an operation failed without erasing the + // resource it had written. + update := map[string]bool{} + for path := range strings.SplitSeq(updateMask, ",") { + path = strings.TrimSpace(path) + if !slices.Contains(dmsUpdatableOperationFields, path) { + return dmsInvalidArgument("update_mask path " + path + " is not updatable") + } + update[path] = true + } defer s.LockUnlock()() @@ -391,8 +409,26 @@ func (s *FakeWorkspace) UpdateOperation(req Request, deploymentID, versionID, re return dmsAborted("sequence_id is outdated; the operation is at " + strconv.FormatInt(existing.SequenceId, 10)) } - failed := op.Status == bundledeployments.OperationStatusOperationStatusFailed - if !failed && op.ErrorMessage != "" { + // The invariants hold over the operation the update leaves behind, not the request: + // a field the mask leaves out keeps the value it already had, so a failure that + // updates only status and error_message is checked against the state and id the + // earlier write recorded. + after := *existing + if update["state"] { + after.State = op.State + } + if update["error_message"] { + after.ErrorMessage = op.ErrorMessage + } + if update["resource_id"] { + after.ResourceId = op.ResourceId + } + if update["status"] { + after.Status = op.Status + } + + failed := after.Status == bundledeployments.OperationStatusOperationStatusFailed + if !failed && after.ErrorMessage != "" { return dmsInvalidArgument("error_message is only allowed when status is OPERATION_STATUS_FAILED") } @@ -400,15 +436,23 @@ func (s *FakeWorkspace) UpdateOperation(req Request, deploymentID, versionID, re // which resource via resource_id. This applies to both succeeded and failed // operations: a failed operation reports prior state to document what existed // before the attempt failed. - if op.State != nil && op.ResourceId == "" { + if after.State != nil && after.ResourceId == "" { return dmsInvalidArgument("resource_id is required for an operation that records state") } - // Only the mutable fields change; action_type and resource_key stay as created. - existing.State = op.State - existing.ErrorMessage = op.ErrorMessage - existing.ResourceId = op.ResourceId - existing.Status = op.Status + // Only the masked fields change; action_type and resource_key stay as created. + if update["state"] { + existing.State = op.State + } + if update["error_message"] { + existing.ErrorMessage = op.ErrorMessage + } + if update["resource_id"] { + existing.ResourceId = op.ResourceId + } + if update["status"] { + existing.Status = op.Status + } existing.SequenceId++ body, err := operationBody(existing) From 39b1a677937b0860fecfc9d2d53fd8a15c9b71ff Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 11 Aug 2026 12:13:20 +0000 Subject: [PATCH 081/125] acceptance: keep Git Bash from rewriting the DMS API paths on Windows provenance and record-failure pass a leading-'/' API path to "databricks api get". Git Bash rewrites such an argument into a Windows path before the CLI sees it, so the request went to /Program Files/Git/api/2.0/... and the testserver had no stub for it. Both already set MSYS_NO_PATHCONV for their workspace get-status; the api get needs it for the same reason. Also drops a trailing blank line from three test.toml files this branch added, which tools/validate_whitespace.py rejects. Co-authored-by: Isaac --- acceptance/bundle/dms/provenance/output.txt | 2 +- acceptance/bundle/dms/provenance/script | 4 +++- acceptance/bundle/dms/record-failure/output.txt | 2 +- acceptance/bundle/dms/record-failure/script | 4 +++- acceptance/bundle/resources/jobs/delete_task/test.toml | 1 - .../bundle/resources/jobs/remote_delete/deploy/test.toml | 1 - acceptance/bundle/resources/jobs/update/test.toml | 1 - 7 files changed, 8 insertions(+), 7 deletions(-) diff --git a/acceptance/bundle/dms/provenance/output.txt b/acceptance/bundle/dms/provenance/output.txt index c567a12d66c..6dbefc35917 100644 --- a/acceptance/bundle/dms/provenance/output.txt +++ b/acceptance/bundle/dms/provenance/output.txt @@ -53,7 +53,7 @@ Deployment complete! } === The service denormalizes them onto the deployment, so a reader gets the provenance of the latest version ->>> [CLI] api get /api/2.0/bundle/deployments/[NUMID] +>>> MSYS_NO_PATHCONV=1 [CLI] api get /api/2.0/bundle/deployments/[NUMID] { "target_name": "dev", "deployment_mode": "DEPLOYMENT_MODE_DEVELOPMENT", diff --git a/acceptance/bundle/dms/provenance/script b/acceptance/bundle/dms/provenance/script index eb36baf6067..51449ae50b9 100644 --- a/acceptance/bundle/dms/provenance/script +++ b/acceptance/bundle/dms/provenance/script @@ -8,7 +8,9 @@ trace print_requests.py //versions --sort title "The service denormalizes them onto the deployment, so a reader gets the provenance of the latest version" deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-provenance/dev/state/resources.deployment.json" -o json | python3 -c 'import sys,json; print(json.load(sys.stdin)["object_id"])') -trace $CLI api get "/api/2.0/bundle/deployments/${deployment_id}" | jq '{target_name, deployment_mode, git_info, workspace_info}' +# MSYS_NO_PATHCONV as above: Git Bash on Windows would rewrite the leading-'/' API +# path into C:/Program Files/Git/api/2.0/... before the CLI saw it. +trace MSYS_NO_PATHCONV=1 $CLI api get "/api/2.0/bundle/deployments/${deployment_id}" | jq '{target_name, deployment_mode, git_info, workspace_info}' # The deploy uploads files in a nondeterministic order; only the requests above are # asserted. diff --git a/acceptance/bundle/dms/record-failure/output.txt b/acceptance/bundle/dms/record-failure/output.txt index d1a6eaa1faf..e2ffcbc41de 100644 --- a/acceptance/bundle/dms/record-failure/output.txt +++ b/acceptance/bundle/dms/record-failure/output.txt @@ -59,7 +59,7 @@ API message: cluster spec is invalid } === The failed resource is not listed at all: state is what projects a resource, and a failed create records none, so a later deploy plans to create it rather than treating it as already deployed ->>> [CLI] api get /api/2.0/bundle/deployments/[NUMID]/resources +>>> MSYS_NO_PATHCONV=1 [CLI] api get /api/2.0/bundle/deployments/[NUMID]/resources {} === Redeploying from DMS state alone still creates the failed resource: it was never applied, so it must not be skipped as unchanged diff --git a/acceptance/bundle/dms/record-failure/script b/acceptance/bundle/dms/record-failure/script index 7e9fa3d47c2..5e0212155d9 100644 --- a/acceptance/bundle/dms/record-failure/script +++ b/acceptance/bundle/dms/record-failure/script @@ -6,7 +6,9 @@ title "The failed resource is not listed at all: state is what projects a resour # The deployment ID is the workspace node's ID; read it back the way the CLI does. # Extracted with python, not jq: the ID exceeds 2^53 and jq 1.6 rounds it. deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-record-failure/default/state/resources.deployment.json" -o json | python3 -c 'import sys,json; print(json.load(sys.stdin)["object_id"])') -trace $CLI api get "/api/2.0/bundle/deployments/${deployment_id}/resources" +# MSYS_NO_PATHCONV as above: Git Bash on Windows would rewrite the leading-'/' API +# path into C:/Program Files/Git/api/2.0/... before the CLI saw it. +trace MSYS_NO_PATHCONV=1 $CLI api get "/api/2.0/bundle/deployments/${deployment_id}/resources" title "Redeploying from DMS state alone still creates the failed resource: it was never applied, so it must not be skipped as unchanged" rm -rf .databricks diff --git a/acceptance/bundle/resources/jobs/delete_task/test.toml b/acceptance/bundle/resources/jobs/delete_task/test.toml index 57ec9bccd00..83bd492c4cf 100644 --- a/acceptance/bundle/resources/jobs/delete_task/test.toml +++ b/acceptance/bundle/resources/jobs/delete_task/test.toml @@ -5,4 +5,3 @@ EnvMatrix.READPLAN = ["", "1"] # the applied resource and the next plan reports it as a change. Recording is skipped here # until the stamp is written into the saved plan too. EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] - diff --git a/acceptance/bundle/resources/jobs/remote_delete/deploy/test.toml b/acceptance/bundle/resources/jobs/remote_delete/deploy/test.toml index 57ec9bccd00..83bd492c4cf 100644 --- a/acceptance/bundle/resources/jobs/remote_delete/deploy/test.toml +++ b/acceptance/bundle/resources/jobs/remote_delete/deploy/test.toml @@ -5,4 +5,3 @@ EnvMatrix.READPLAN = ["", "1"] # the applied resource and the next plan reports it as a change. Recording is skipped here # until the stamp is written into the saved plan too. EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] - diff --git a/acceptance/bundle/resources/jobs/update/test.toml b/acceptance/bundle/resources/jobs/update/test.toml index 56096b047d3..d8ef3c22ad0 100644 --- a/acceptance/bundle/resources/jobs/update/test.toml +++ b/acceptance/bundle/resources/jobs/update/test.toml @@ -4,4 +4,3 @@ EnvMatrix.READPLAN = ["", "1"] # the applied resource and the next plan reports it as a change. Recording is skipped here # until the stamp is written into the saved plan too. EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] - From 705269a7acbe6f2c3305b2ed49e702c9156b95a9 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 11 Aug 2026 12:38:48 +0000 Subject: [PATCH 082/125] acceptance: use nostamp instead of a hand-rolled jq del, and drop a dead parameter Fifteen scripts deleted deployment_id and version_id with their own jq expression. Six of them already piped through nostamp first, so the jq was doing nothing; the rest were reimplementing it. Both now just use nostamp. No golden changes, which is what makes it a straight simplification. deployCore no longer takes a recorder: logDeploymentVersion moved out to the caller, which left the parameter unused. destroyCore still needs its own, for CompleteVersion. Co-authored-by: Isaac --- .../bundle/resources/apps/lifecycle-started-omitted/script | 2 +- .../clusters/deploy/update-and-resize-autoscale/script | 2 +- .../resources/clusters/deploy/update-and-resize/script | 2 +- .../resources/dashboards/unpublish-out-of-band/script | 2 +- acceptance/bundle/resources/jobs/delete_job/script | 2 +- acceptance/bundle/resources/jobs/remote_add_tag/script | 2 +- .../bundle/resources/jobs/remote_matches_config/script | 2 +- acceptance/bundle/resources/jobs/update_single_node/script | 6 +++--- .../bundle/resources/jobs/webhook-reorder-remote/script | 2 +- .../permissions/genie_spaces/current_can_manage/script | 2 +- .../resources/permissions/models/current_can_manage/script | 2 +- .../bundle/resources/permissions/target_permissions/script | 2 +- .../bundle/resources/pipelines/remote_matches_config/script | 2 +- acceptance/bundle/resources/volumes/change-name/script | 2 +- .../bundle/resources/volumes/remote-change-name/script | 2 +- bundle/phases/deploy.go | 4 ++-- 16 files changed, 19 insertions(+), 19 deletions(-) diff --git a/acceptance/bundle/resources/apps/lifecycle-started-omitted/script b/acceptance/bundle/resources/apps/lifecycle-started-omitted/script index 79a1ff2e93b..dd08f33fff5 100644 --- a/acceptance/bundle/resources/apps/lifecycle-started-omitted/script +++ b/acceptance/bundle/resources/apps/lifecycle-started-omitted/script @@ -79,6 +79,6 @@ trace $CLI bundle deploy trace print_app_requests title "(started omitted, app running) -> bundle plan shows no drift" -$CLI bundle plan -o json | jq 'del(.. | objects | .deployment_id?, .version_id?)' > LOG.planjson +$CLI bundle plan -o json | nostamp > LOG.planjson verify_no_drift.py LOG.planjson echo "Plan: no drift detected" diff --git a/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/script b/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/script index eb9645721ec..68825aa8465 100755 --- a/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/script +++ b/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/script @@ -7,7 +7,7 @@ cleanup() { trap cleanup EXIT $CLI bundle plan > out.plan_.$DATABRICKS_BUNDLE_ENGINE.txt -$CLI bundle plan -o json | nostamp | jq 'del(.. | objects | .deployment_id?, .version_id?)' > out.plan_.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp > out.plan_.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy title "Cluster should exist with num_workers after bundle deployment:\n" diff --git a/acceptance/bundle/resources/clusters/deploy/update-and-resize/script b/acceptance/bundle/resources/clusters/deploy/update-and-resize/script index 0ef5d80c68b..b37aa03b62f 100644 --- a/acceptance/bundle/resources/clusters/deploy/update-and-resize/script +++ b/acceptance/bundle/resources/clusters/deploy/update-and-resize/script @@ -7,7 +7,7 @@ cleanup() { trap cleanup EXIT $CLI bundle plan > out.plan_.$DATABRICKS_BUNDLE_ENGINE.txt -$CLI bundle plan -o json | nostamp | jq 'del(.. | objects | .deployment_id?, .version_id?)' > out.plan_.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp > out.plan_.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy title "Cluster should exist after bundle deployment:\n" diff --git a/acceptance/bundle/resources/dashboards/unpublish-out-of-band/script b/acceptance/bundle/resources/dashboards/unpublish-out-of-band/script index e6267ead0a8..222e3bf521a 100644 --- a/acceptance/bundle/resources/dashboards/unpublish-out-of-band/script +++ b/acceptance/bundle/resources/dashboards/unpublish-out-of-band/script @@ -25,7 +25,7 @@ trace $CLI lakeview unpublish $DASHBOARD_ID # Direct: shows "update" because Published field changes from false to true trace $CLI bundle plan > out.plan.$DATABRICKS_BUNDLE_ENGINE.txt -$CLI bundle plan -o json | jq 'del(.. | objects | .deployment_id?, .version_id?)' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp > out.plan.$DATABRICKS_BUNDLE_ENGINE.json json_in_json_normalize.py out.plan.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy diff --git a/acceptance/bundle/resources/jobs/delete_job/script b/acceptance/bundle/resources/jobs/delete_job/script index a0910145023..1622b3e11cb 100644 --- a/acceptance/bundle/resources/jobs/delete_job/script +++ b/acceptance/bundle/resources/jobs/delete_job/script @@ -1,5 +1,5 @@ trace $CLI bundle deploy cp empty.yml databricks.yml -$CLI bundle plan -o json | jq 'del(.. | objects | .deployment_id?, .version_id?)' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp > out.plan.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy print_requests.py //jobs --nostamp diff --git a/acceptance/bundle/resources/jobs/remote_add_tag/script b/acceptance/bundle/resources/jobs/remote_add_tag/script index 79de8e89f48..3a651737979 100644 --- a/acceptance/bundle/resources/jobs/remote_add_tag/script +++ b/acceptance/bundle/resources/jobs/remote_add_tag/script @@ -8,4 +8,4 @@ r["tags"]["new_tag"] = "new_value" EOF $CLI bundle plan -$CLI bundle plan -o json | nostamp | jq 'del(.. | objects | .deployment_id?, .version_id?)' > out.plan_post_update.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp > out.plan_post_update.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/jobs/remote_matches_config/script b/acceptance/bundle/resources/jobs/remote_matches_config/script index 1402dc86189..9ca6a0655b6 100755 --- a/acceptance/bundle/resources/jobs/remote_matches_config/script +++ b/acceptance/bundle/resources/jobs/remote_matches_config/script @@ -13,7 +13,7 @@ r["max_concurrent_runs"] = 2 EOF trace $CLI bundle plan -$CLI bundle plan -o json | nostamp | jq 'del(.. | objects | .deployment_id?, .version_id?)' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp > out.plan.$DATABRICKS_BUNDLE_ENGINE.json rm out.requests.txt # XXX READPLAN diff --git a/acceptance/bundle/resources/jobs/update_single_node/script b/acceptance/bundle/resources/jobs/update_single_node/script index ada0b239589..4deefe32bbc 100644 --- a/acceptance/bundle/resources/jobs/update_single_node/script +++ b/acceptance/bundle/resources/jobs/update_single_node/script @@ -1,15 +1,15 @@ echo "*" > .gitignore trace $CLI bundle plan -$CLI bundle plan -o json | nostamp | jq 'del(.. | objects | .deployment_id?, .version_id?)' > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy -$CLI bundle plan -o json | nostamp | jq 'del(.. | objects | .deployment_id?, .version_id?)' > out.plan_skip.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp > out.plan_skip.$DATABRICKS_BUNDLE_ENGINE.json trace print_requests.py //jobs > out.create.requests.txt --nostamp title "Update trigger.periodic.unit and re-deploy" trace update_file.py databricks.yml DAYS HOURS trace $CLI bundle plan -$CLI bundle plan -o json | nostamp | jq 'del(.. | objects | .deployment_id?, .version_id?)' > out.plan_update.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp > out.plan_update.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py //jobs > out.update.requests.$DATABRICKS_BUNDLE_ENGINE.json --nostamp diff --git a/acceptance/bundle/resources/jobs/webhook-reorder-remote/script b/acceptance/bundle/resources/jobs/webhook-reorder-remote/script index e355b877ec3..c0257db6330 100644 --- a/acceptance/bundle/resources/jobs/webhook-reorder-remote/script +++ b/acceptance/bundle/resources/jobs/webhook-reorder-remote/script @@ -14,6 +14,6 @@ EOF # The reordered remote must not produce a phantom diff: on_* lists are diffed by id. trace $CLI bundle plan -$CLI bundle plan -o json | nostamp | jq '.plan."resources.jobs.my_job".changes | del(.. | objects | .deployment_id?, .version_id?)' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp | jq '.plan."resources.jobs.my_job".changes' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json trace print_requests.py //jobs --nostamp diff --git a/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/script b/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/script index 22a34cb117e..e589a40bd5b 100644 --- a/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/script +++ b/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/script @@ -1,7 +1,7 @@ trace $CLI bundle validate -o json | jq .resources.genie_spaces.foo.permissions rm out.requests.txt -$CLI bundle plan -o json | jq 'del(.. | objects | .deployment_id?, .version_id?)' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp > out.plan.$DATABRICKS_BUNDLE_ENGINE.json rm out.requests.txt trace $CLI bundle deploy diff --git a/acceptance/bundle/resources/permissions/models/current_can_manage/script b/acceptance/bundle/resources/permissions/models/current_can_manage/script index eb4446a728d..12aa59254ee 100644 --- a/acceptance/bundle/resources/permissions/models/current_can_manage/script +++ b/acceptance/bundle/resources/permissions/models/current_can_manage/script @@ -1,7 +1,7 @@ trace $CLI bundle validate -o json | jq .resources.$RESOURCE.foo.permissions rm out.requests.txt -$CLI bundle plan -o json | jq 'del(.. | objects | .deployment_id?, .version_id?)' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp > out.plan.$DATABRICKS_BUNDLE_ENGINE.json rm out.requests.txt trace errcode $CLI bundle deploy &> out.deploy.txt diff --git a/acceptance/bundle/resources/permissions/target_permissions/script b/acceptance/bundle/resources/permissions/target_permissions/script index 2c63ad82cd2..7a443d8cc2d 100644 --- a/acceptance/bundle/resources/permissions/target_permissions/script +++ b/acceptance/bundle/resources/permissions/target_permissions/script @@ -1,5 +1,5 @@ trace $CLI bundle plan -$CLI bundle plan -o json | jq 'del(.. | objects | .deployment_id?, .version_id?)' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp > out.plan.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy print_requests.py --nostamp //jobs/ > out.requests_create.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/pipelines/remote_matches_config/script b/acceptance/bundle/resources/pipelines/remote_matches_config/script index 9896fb8e4f1..6a6a334022e 100644 --- a/acceptance/bundle/resources/pipelines/remote_matches_config/script +++ b/acceptance/bundle/resources/pipelines/remote_matches_config/script @@ -15,6 +15,6 @@ r["run_as"] = {"user_name": "changed@example.test"} EOF trace $CLI bundle plan -$CLI bundle plan -o json | nostamp | jq 'del(.. | objects | .deployment_id?, .version_id?)' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp > out.plan.$DATABRICKS_BUNDLE_ENGINE.json rm out.requests.txt diff --git a/acceptance/bundle/resources/volumes/change-name/script b/acceptance/bundle/resources/volumes/change-name/script index a897616ae12..396ffb15155 100644 --- a/acceptance/bundle/resources/volumes/change-name/script +++ b/acceptance/bundle/resources/volumes/change-name/script @@ -10,7 +10,7 @@ trace update_file.py databricks.yml myvolume mynewvolume trace $CLI bundle plan # terraform marks this as "update", direct marks this as "update_with_id" -$CLI bundle plan -o json | jq 'del(.. | objects | .deployment_id?, .version_id?)' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp > out.plan.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py //unity diff --git a/acceptance/bundle/resources/volumes/remote-change-name/script b/acceptance/bundle/resources/volumes/remote-change-name/script index b4ca369abe4..536ad6d50b8 100644 --- a/acceptance/bundle/resources/volumes/remote-change-name/script +++ b/acceptance/bundle/resources/volumes/remote-change-name/script @@ -1,4 +1,4 @@ -$CLI bundle plan -o json | jq 'del(.. | objects | .deployment_id?, .version_id?)' > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace $CLI volumes update mycatalog.myschema.myname --json '{"new_name": "my_new_name"}' diff --git a/bundle/phases/deploy.go b/bundle/phases/deploy.go index 17326ef24ba..d291158cf14 100644 --- a/bundle/phases/deploy.go +++ b/bundle/phases/deploy.go @@ -76,7 +76,7 @@ func approvalForDeploy(ctx context.Context, b *bundle.Bundle, plan *deployplan.P return cmdio.AskYesOrNo(ctx, "Would you like to proceed?") } -func deployCore(ctx context.Context, b *bundle.Bundle, plan *deployplan.Plan, stateEngine engine.EngineType, requestedEngine engine.EngineSetting, recorder *dms.Recorder) { +func deployCore(ctx context.Context, b *bundle.Bundle, plan *deployplan.Plan, stateEngine engine.EngineType, requestedEngine engine.EngineSetting) { // Core mutators that CRUD resources and modify deployment state. These // mutators need informed consent if they are potentially destructive. cmdio.LogString(ctx, "Deploying resources...") @@ -315,7 +315,7 @@ func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHand // Record operations under that version, so DMS holds the deployed resource state. setOperationRecorder(ctx, b, recorder) - deployCore(ctx, b, plan, stateEngine, requestedEngine, recorder) + deployCore(ctx, b, plan, stateEngine, requestedEngine) if logdiag.HasError(ctx) { return From 7a9f4ed733defd4272b6e9760f6646c9cb1f2429 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 11 Aug 2026 13:05:19 +0000 Subject: [PATCH 083/125] acceptance: repair two tests main merged broken Both fail on main as it stands, independently of this branch: - pipelines/destroy/cascade-on-destroy-terraform-error sets Local = true, a test.toml key main removed in aebca140b ("acc: always run acceptance tests locally"), so the test aborts with "Undecoded key". The cascade PR was written before that removal and merged after it. - bundle/config-remote-sync/split/positional asserts the destroy warning text, which the same cascade PR extended with the cascade_on_destroy hint without regenerating this golden. Fixed here rather than left for a separate PR because they block this one's CI. Co-authored-by: Isaac --- .../bundle/config-remote-sync/split/positional/output.txt | 2 +- .../destroy/cascade-on-destroy-terraform-error/out.test.toml | 1 - .../destroy/cascade-on-destroy-terraform-error/test.toml | 1 - .../destroy-pipeline-cascade-state-divergence/out.test.toml | 1 - .../pipelines/destroy/destroy-pipeline-cascade/out.test.toml | 1 - 5 files changed, 1 insertion(+), 5 deletions(-) diff --git a/acceptance/bundle/config-remote-sync/split/positional/output.txt b/acceptance/bundle/config-remote-sync/split/positional/output.txt index fe7fd191521..5abcd0de88e 100644 --- a/acceptance/bundle/config-remote-sync/split/positional/output.txt +++ b/acceptance/bundle/config-remote-sync/split/positional/output.txt @@ -65,7 +65,7 @@ The following resources will be deleted: delete resources.pipelines.split_pipeline This action will result in the deletion of the following Lakeflow Spark Declarative Pipelines along with the -Streaming Tables (STs) and Materialized Views (MVs) managed by them: +Streaming Tables (STs) and Materialized Views (MVs) managed by them. Set 'cascade_on_destroy: false' on a pipeline to retain datasets on pipeline deletion: delete resources.pipelines.split_pipeline All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/dev diff --git a/acceptance/pipelines/destroy/cascade-on-destroy-terraform-error/out.test.toml b/acceptance/pipelines/destroy/cascade-on-destroy-terraform-error/out.test.toml index 65156e0457c..d2059b4b5d7 100644 --- a/acceptance/pipelines/destroy/cascade-on-destroy-terraform-error/out.test.toml +++ b/acceptance/pipelines/destroy/cascade-on-destroy-terraform-error/out.test.toml @@ -1,3 +1,2 @@ -Local = true Cloud = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform"] diff --git a/acceptance/pipelines/destroy/cascade-on-destroy-terraform-error/test.toml b/acceptance/pipelines/destroy/cascade-on-destroy-terraform-error/test.toml index ec557241780..bffe50aebe1 100644 --- a/acceptance/pipelines/destroy/cascade-on-destroy-terraform-error/test.toml +++ b/acceptance/pipelines/destroy/cascade-on-destroy-terraform-error/test.toml @@ -1,4 +1,3 @@ -Local = true RecordRequests = false # Static validation only; no need to run on cloud. diff --git a/acceptance/pipelines/destroy/destroy-pipeline-cascade-state-divergence/out.test.toml b/acceptance/pipelines/destroy/destroy-pipeline-cascade-state-divergence/out.test.toml index e90b6d5d1ba..0938e678987 100644 --- a/acceptance/pipelines/destroy/destroy-pipeline-cascade-state-divergence/out.test.toml +++ b/acceptance/pipelines/destroy/destroy-pipeline-cascade-state-divergence/out.test.toml @@ -1,3 +1,2 @@ -Local = true Cloud = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/pipelines/destroy/destroy-pipeline-cascade/out.test.toml b/acceptance/pipelines/destroy/destroy-pipeline-cascade/out.test.toml index e90b6d5d1ba..0938e678987 100644 --- a/acceptance/pipelines/destroy/destroy-pipeline-cascade/out.test.toml +++ b/acceptance/pipelines/destroy/destroy-pipeline-cascade/out.test.toml @@ -1,3 +1,2 @@ -Local = true Cloud = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] From 3f610d385fc5b6403e45df7a0a9b8d597eee4358 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Sun, 16 Aug 2026 23:54:17 +0000 Subject: [PATCH 084/125] bundle: upload recorded operations from one goroutine Eight upload workers meant a resource key could be picked up by any of them, so the queue had to track which keys were queued-or-uploading and guarantee one upload per resource at a time. One goroutine gives that for free. What goes: the worker pool, the channel of resource keys, the queuedOrUploading map, the WaitGroup, and the invariant that a key must never be left for no worker to pick up. What stays: coalescing, which matters more now - writes pile up behind a single uploader far more often than behind eight - and the sticky first error that stops the deploy mid-apply. Coalescing a failure needed one addition to stay correct. A failure carries state only when there was a pre-deploy record to carry, so a create that wrote state and then failed would, once the two writes merged, report no state at all and drop the resource from the deployment - the same bug the failure update mask fixes on the wire. coalesce now inherits the superseded write's state and id, and a test pins it. No acceptance golden changes. Co-authored-by: Isaac --- bundle/direct/bundle_apply.go | 24 +- bundle/direct/opqueue.go | 221 ----------------- bundle/direct/opqueue_test.go | 453 ---------------------------------- bundle/direct/oprecorder.go | 13 +- bundle/direct/opsink.go | 237 ++++++++++++++++++ bundle/direct/opsink_test.go | 298 ++++++++++++++++++++++ 6 files changed, 556 insertions(+), 690 deletions(-) delete mode 100644 bundle/direct/opqueue.go delete mode 100644 bundle/direct/opqueue_test.go create mode 100644 bundle/direct/opsink.go create mode 100644 bundle/direct/opsink_test.go diff --git a/bundle/direct/bundle_apply.go b/bundle/direct/bundle_apply.go index 68e009cb413..c16f728cdd2 100644 --- a/bundle/direct/bundle_apply.go +++ b/bundle/direct/bundle_apply.go @@ -35,17 +35,17 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa return } - // Operations are recorded with DMS from background workers so a resource's - // deploy is not held up by the CreateOperation round trip. The queue is - // drained below, once every apply worker has finished recording. + // Operations are recorded with DMS from one background goroutine so a resource's + // deploy is not held up by the CreateOperation round trip. It is drained below, + // once every apply worker has finished recording. // // The state DB records through it, so every state write becomes an operation and // DMS mirrors the WAL. - opQueue := newOperationQueue(ctx, b.OpRec) - if opQueue != nil { - // Assigned only when non-nil: a nil *operationQueue in an interface is not a + opSink := newOperationSink(ctx, b.OpRec) + if opSink != nil { + // Assigned only when non-nil: a nil *operationSink in an interface is not a // nil interface, so the state DB's nil check would not see it. - b.StateDB.SetOperationSink(opQueue) + b.StateDB.SetOperationSink(opSink) } g.Run(defaultParallelism, func(resourceKey string, failedDependency *string) bool { @@ -79,7 +79,7 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa } // Stop resource CRUD once uploading DMS state has failed. - if err := opQueue.firstErr(); err != nil { + if err := opSink.firstErr(); err != nil { logdiag.LogError(ctx, fmt.Errorf("%s: %w", errorPrefix, err)) return false } @@ -109,7 +109,7 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa } if err != nil { _, priorState := priorRecord(&b.StateDB, resourceKey) - opQueue.recordFailure(ctx, resourceKey, action, deletedID, priorState, err) + opSink.recordFailure(ctx, resourceKey, action, deletedID, priorState, err) logdiag.LogError(ctx, fmt.Errorf("%s: %w", errorPrefix, err)) return false } @@ -144,7 +144,7 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa // Both are empty for a create that never got an ID, which is what the // service expects for a failed create. priorID, priorState := priorRecord(&b.StateDB, resourceKey) - opQueue.recordFailure(ctx, resourceKey, action, priorID, priorState, err) + opSink.recordFailure(ctx, resourceKey, action, priorID, priorState, err) logdiag.LogError(ctx, fmt.Errorf("%s: %w", errorPrefix, err)) return false } @@ -172,10 +172,10 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa return true }) - // Wait for the queued operations before returning: the caller completes the + // Wait for the recorded operations before returning: the caller completes the // DMS version right after, and a version must not be completed with uploads // still in flight. - if err := opQueue.close(); err != nil { + if err := opSink.close(); err != nil { logdiag.LogError(ctx, err) } } diff --git a/bundle/direct/opqueue.go b/bundle/direct/opqueue.go deleted file mode 100644 index b22b0a78ae0..00000000000 --- a/bundle/direct/opqueue.go +++ /dev/null @@ -1,221 +0,0 @@ -package direct - -import ( - "context" - "encoding/json" - "fmt" - "sync" - - "github.com/databricks/cli/bundle/deployplan" - "github.com/databricks/cli/bundle/direct/dstate" - "github.com/databricks/cli/libs/log" -) - -const ( - // operationQueueSize bounds how many recorded operations wait for upload. Deep - // enough that an apply worker practically never blocks on a free slot. - operationQueueSize = 10 - - // operationUploadWorkers is how many uploads run at a time. - operationUploadWorkers = 8 -) - -// operationQueue uploads recorded operations from background workers, so a deploy -// never waits on the CreateOperation round trip. Two rules shape it: -// -// - One resource, one upload at a time. DMS keeps a single state per resource, so -// overlapping uploads could land out of order and leave the older state. -// - Newest operation wins. Each write carries the resource's full state, so writes -// that pile up behind an in-flight upload are merged into one ("coalesced") and -// the resource costs a single request instead of one per write. -// -// close reports the first upload failure, which fails the deploy: DMS becomes the -// source of truth (see dstate.readDMSState), so a missing record would make the -// next deploy create a resource that already exists. -type operationQueue struct { - uploader operationUploader - - // queue carries resource keys, not operations: a worker looks the operation up - // when it picks the key up, which is what makes coalescing work. - queue chan string - wg sync.WaitGroup - - // mu guards the fields below. - mu sync.Mutex - - // pending holds the one operation waiting per resource key: a resource can write - // state more than once in a deploy (a recreate drops the entry, then saves the new - // resource), and a write that arrives before the previous one is uploaded replaces - // it. No key means nothing is waiting. - pending map[string]recordedOperation - - // queuedOrUploading means "some worker will get to this key". Recording such a - // key writes to pending only, so two workers never upload one resource at once. - queuedOrUploading map[string]bool - - err error - closed bool -} - -// newOperationQueue starts the upload workers, returning nil when uploader is nil -// (recording off; every method is a no-op on a nil queue). ctx must outlive close. -func newOperationQueue(ctx context.Context, uploader operationUploader) *operationQueue { - if uploader == nil { - return nil - } - - q := &operationQueue{ - uploader: uploader, - queue: make(chan string, operationQueueSize), - pending: make(map[string]recordedOperation), - queuedOrUploading: make(map[string]bool), - } - - q.wg.Add(operationUploadWorkers) - for range operationUploadWorkers { - go q.work(ctx) - } - - return q -} - -// RecordOperation implements dstate.OperationSink: every state write becomes an -// operation, so DMS mirrors the WAL. state is already the serialized envelope, and -// nil for a delete. -// -// An earlier upload failure does not stop this: every write is still recorded, best -// effort, so DMS ends up as close to reality as it can get. -func (q *operationQueue) RecordOperation(ctx context.Context, resourceKey string, info dstate.OperationInfo, resourceID string, state json.RawMessage) { - if q == nil { - return - } - - op, err := newStateOperation(info, resourceID, state) - if err != nil { - // The deploy already persisted this write locally, so failing it here would - // report an error about history for a resource that deployed fine. - log.Warnf(ctx, "Not recording operation for %s: %s", resourceKey, err) - return - } - - q.enqueue(ctx, resourceKey, op) -} - -// recordFailure records that applying a resource failed, so the deployment history -// explains the failure instead of omitting the resource. It returns nothing: the -// deploy is already failing, and a second error would mask the one the user needs. -func (q *operationQueue) recordFailure(ctx context.Context, resourceKey string, action deployplan.ActionType, resourceID string, priorState json.RawMessage, cause error) { - if q == nil { - return - } - - op, err := newFailedOperation(action, resourceID, priorState, cause) - if err != nil { - log.Warnf(ctx, "Not recording failure for %s: %s", resourceKey, err) - return - } - - q.enqueue(ctx, resourceKey, op) -} - -// enqueue makes op the operation waiting for resourceKey, replacing whatever was -// already waiting, and makes sure a worker will pick it up. -func (q *operationQueue) enqueue(ctx context.Context, resourceKey string, op recordedOperation) { - q.mu.Lock() - q.pending[resourceKey] = op - alreadyHandled := q.queuedOrUploading[resourceKey] - q.queuedOrUploading[resourceKey] = true - q.mu.Unlock() - - // A worker will re-read pending before it finishes, so it picks up the operation - // stored above. Queueing again would let a second worker upload the same key. - if alreadyHandled { - return - } - - q.queue <- resourceKey -} - -// close drains the queue and returns the first upload error. Every record caller -// must have returned first (record on a closed queue panics); calling close twice -// is safe, so it can be deferred and still checked at a specific point. -// -// It takes no lock: it runs after every apply worker returned, and wg.Wait orders -// the workers' writes to err before it is read here. -func (q *operationQueue) close() error { - if q == nil { - return nil - } - - if !q.closed { - q.closed = true - close(q.queue) - q.wg.Wait() - } - - return q.err -} - -func (q *operationQueue) work(ctx context.Context) { - defer q.wg.Done() - - for resourceKey := range q.queue { - // Drain this key here instead of re-queueing it: a worker sending to the - // channel it consumes from deadlocks once the queue is full. - for { - op, ok := q.take(resourceKey) - if !ok { - break - } - - // Keep going after a failure, so one bad upload does not drop the records - // for every resource behind it. - if err := q.uploader.upload(ctx, resourceKey, op); err != nil { - q.setErr(fmt.Errorf("recording operation for %s with the deployment metadata service: %w", resourceKey, err)) - } - } - } -} - -// take claims the operation waiting for resourceKey, reporting false and clearing the -// queuedOrUploading mark when nothing is left, which lets record queue it again. Both -// happen under one lock, so a key can never be left for no worker to pick up. -func (q *operationQueue) take(resourceKey string) (recordedOperation, bool) { - q.mu.Lock() - defer q.mu.Unlock() - - op, ok := q.pending[resourceKey] - if !ok { - delete(q.queuedOrUploading, resourceKey) - return recordedOperation{}, false - } - - // The mark stays until the branch above clears it, so anything recorded during - // this upload is still picked up and no second worker takes the key meanwhile. - delete(q.pending, resourceKey) - return op, true -} - -// setErr keeps the first upload error; later ones are dropped because one failure -// is enough to fail the deploy. -func (q *operationQueue) setErr(err error) { - q.mu.Lock() - defer q.mu.Unlock() - - if q.err == nil { - q.err = err - } -} - -// firstErr returns the first upload error, or nil if every upload so far -// succeeded. A nil queue (recording disabled) never errors. -func (q *operationQueue) firstErr() error { - if q == nil { - return nil - } - - q.mu.Lock() - defer q.mu.Unlock() - - return q.err -} diff --git a/bundle/direct/opqueue_test.go b/bundle/direct/opqueue_test.go deleted file mode 100644 index 0a49cc1b99f..00000000000 --- a/bundle/direct/opqueue_test.go +++ /dev/null @@ -1,453 +0,0 @@ -package direct - -import ( - "context" - "encoding/json" - "errors" - "strconv" - "strings" - "sync" - "testing" - - "github.com/databricks/cli/bundle/deployplan" - "github.com/databricks/cli/bundle/direct/dstate" - "github.com/databricks/databricks-sdk-go/service/bundledeployments" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -// fakeUploader records the uploads it receives and optionally blocks until -// release is closed, so a test can hold operations in the queue and observe -// coalescing. -type fakeUploader struct { - block chan struct{} - started chan string - // done receives the resource key after the upload returns, for tests that need - // an upload to have completed rather than merely started. - done chan string - err error - - mu sync.Mutex - uploads []string - actions map[string]bundledeployments.OperationActionType - resourceIDs map[string]string - statuses map[string]bundledeployments.OperationStatus - errorMessages map[string]string -} - -func (f *fakeUploader) upload(ctx context.Context, resourceKey string, op recordedOperation) error { - if f.started != nil { - f.started <- resourceKey - } - if f.block != nil { - <-f.block - } - - f.mu.Lock() - f.uploads = append(f.uploads, resourceKey+"="+string(op.state)) - if f.actions == nil { - f.actions = map[string]bundledeployments.OperationActionType{} - f.resourceIDs = map[string]string{} - f.statuses = map[string]bundledeployments.OperationStatus{} - f.errorMessages = map[string]string{} - } - f.actions[resourceKey] = op.action - f.resourceIDs[resourceKey] = op.resourceID - f.statuses[resourceKey] = op.status - f.errorMessages[resourceKey] = op.errorMessage - f.mu.Unlock() - - // Sent outside the lock: a test that stops reading this channel would otherwise - // hold f.mu and deadlock every other worker. - if f.done != nil { - f.done <- resourceKey - } - return f.err -} - -func (f *fakeUploader) recorded() []string { - f.mu.Lock() - defer f.mu.Unlock() - return append([]string(nil), f.uploads...) -} - -func (f *fakeUploader) actionFor(resourceKey string) bundledeployments.OperationActionType { - f.mu.Lock() - defer f.mu.Unlock() - return f.actions[resourceKey] -} - -func (f *fakeUploader) resourceIDFor(resourceKey string) string { - f.mu.Lock() - defer f.mu.Unlock() - return f.resourceIDs[resourceKey] -} - -func (f *fakeUploader) statusFor(resourceKey string) bundledeployments.OperationStatus { - f.mu.Lock() - defer f.mu.Unlock() - return f.statuses[resourceKey] -} - -func (f *fakeUploader) errorMessageFor(resourceKey string) string { - f.mu.Lock() - defer f.mu.Unlock() - return f.errorMessages[resourceKey] -} - -// uploadsFor returns the uploads recorded for one resource key, for tests where -// other resources are uploaded alongside it. -func uploadsFor(f *fakeUploader, resourceKey string) []string { - var out []string - for _, u := range f.recorded() { - if strings.HasPrefix(u, resourceKey+"=") { - out = append(out, u) - } - } - return out -} - -// envelope builds the serialized RecordedState the state DB hands the queue. -func envelope(t *testing.T, name string) json.RawMessage { - t.Helper() - raw, err := json.Marshal(dstate.RecordedState{State: json.RawMessage(`{"name":"` + name + `"}`)}) - require.NoError(t, err) - return raw -} - -func recordState(t *testing.T, q *operationQueue, resourceKey, name string) { - t.Helper() - q.RecordOperation(t.Context(), resourceKey, dstate.OperationInfo{Action: deployplan.Update}, "id-1", envelope(t, name)) -} - -func TestOperationQueueUploadsEachOperation(t *testing.T) { - f := &fakeUploader{} - q := newOperationQueue(t.Context(), f) - - for i := range 20 { - recordState(t, q, "resources.jobs.job"+strconv.Itoa(i), "n") - } - require.NoError(t, q.close()) - - assert.Len(t, f.recorded(), 20) -} - -func TestOperationQueueMergesWritesQueuedBehindAnUpload(t *testing.T) { - // Hold the first upload so the writes behind it queue up. The two that pile up - // merge into one carrying the newest state, so the resource costs two requests - // rather than three - the in-flight one, then everything after it. - // - // started is buffered for both uploads: a worker blocking on an unread send - // would deadlock the drain below. - f := &fakeUploader{block: make(chan struct{}), started: make(chan string, 2)} - q := newOperationQueue(t.Context(), f) - - recordState(t, q, "resources.jobs.foo", "v1") - // Wait until a worker owns the key, so the operations below are queued behind - // an in-flight upload rather than racing it. - assert.Equal(t, "resources.jobs.foo", <-f.started) - - recordState(t, q, "resources.jobs.foo", "v2") - recordState(t, q, "resources.jobs.foo", "v3") - - close(f.block) - require.NoError(t, q.close()) - - assert.Equal(t, []string{ - `resources.jobs.foo={"state":{"name":"v1"}}`, - `resources.jobs.foo={"state":{"name":"v3"}}`, - }, f.recorded()) -} - -func TestOperationQueueMergedRecreateKeepsItsActionType(t *testing.T) { - f := &fakeUploader{block: make(chan struct{}), started: make(chan string, operationUploadWorkers+2)} - q := newOperationQueue(t.Context(), f) - - for i := range operationUploadWorkers { - recordState(t, q, "resources.jobs.hold"+strconv.Itoa(i), "v1") - assert.Equal(t, "resources.jobs.hold"+strconv.Itoa(i), <-f.started) - } - - q.RecordOperation(t.Context(), "resources.jobs.foo", dstate.OperationInfo{Action: deployplan.Recreate, InProgress: true}, "old-id", nil) - q.RecordOperation(t.Context(), "resources.jobs.foo", dstate.OperationInfo{Action: deployplan.Recreate}, "new-id", envelope(t, "replacement")) - - close(f.block) - require.NoError(t, q.close()) - - assert.Equal(t, - bundledeployments.OperationActionTypeOperationActionTypeRecreate, - f.actionFor("resources.jobs.foo")) - // The newer write's own fields still win: the replacement's id, state and its - // succeeded status, not the in-progress the delete asked for. - assert.Equal(t, "new-id", f.resourceIDFor("resources.jobs.foo")) - assert.Equal(t, []string{ - `resources.jobs.foo={"state":{"name":"replacement"}}`, - }, uploadsFor(f, "resources.jobs.foo")) - assert.Equal(t, - bundledeployments.OperationStatusOperationStatusSucceeded, - f.statusFor("resources.jobs.foo")) -} - -func TestOperationQueueMergesQueuedWritesWhileWorkersAreBusy(t *testing.T) { - // Every worker is parked mid-upload, so the writes below sit in pending rather - // than being picked up. They merge into one upload carrying the newest state. - // - // started is buffered for the merged foo write as well: nothing reads it after the - // loop below, and a worker blocking on the send would deadlock the drain. - f := &fakeUploader{block: make(chan struct{}), started: make(chan string, operationUploadWorkers+2)} - q := newOperationQueue(t.Context(), f) - - recordState(t, q, "resources.jobs.hold", "v1") - assert.Equal(t, "resources.jobs.hold", <-f.started) - - // Occupy the remaining workers so nothing drains the key under test. - for i := range operationUploadWorkers - 1 { - recordState(t, q, "resources.jobs.hold"+strconv.Itoa(i), "v1") - assert.Equal(t, "resources.jobs.hold"+strconv.Itoa(i), <-f.started) - } - - // A resource whose ID is only known after it was created: the first write has no - // ID, the second fills it in. - q.RecordOperation(t.Context(), "resources.jobs.foo", dstate.OperationInfo{Action: deployplan.Create}, "", envelope(t, "created")) - q.RecordOperation(t.Context(), "resources.jobs.foo", dstate.OperationInfo{Action: deployplan.Create}, "id-1", envelope(t, "updated")) - - close(f.block) - require.NoError(t, q.close()) - - assert.Equal(t, []string{ - `resources.jobs.foo={"state":{"name":"updated"}}`, - }, uploadsFor(f, "resources.jobs.foo")) - - // The ID recorded last is the one the create learned. - assert.Equal(t, "id-1", f.resourceIDFor("resources.jobs.foo")) - assert.Equal(t, - bundledeployments.OperationActionTypeOperationActionTypeCreate, - f.actionFor("resources.jobs.foo")) -} - -func TestOperationQueueRecordDuringUploadIsStillUploaded(t *testing.T) { - // Record while the key's own upload is in flight: the key is off the queue but - // still marked, so record does not queue it again. The worker that holds the key - // has to come back for it, or the operation would be silently dropped. - f := &fakeUploader{block: make(chan struct{}), started: make(chan string, 1)} - q := newOperationQueue(t.Context(), f) - - q.RecordOperation(t.Context(), "resources.jobs.foo", dstate.OperationInfo{Action: deployplan.Create}, "", envelope(t, "v1")) - assert.Equal(t, "resources.jobs.foo", <-f.started) - - // The worker has taken the key off the queue and is uploading v1 right now. - q.RecordOperation(t.Context(), "resources.jobs.foo", dstate.OperationInfo{Action: deployplan.Create}, "id-1", envelope(t, "v2")) - - close(f.block) - require.NoError(t, q.close()) - - // Two uploads, in order: an in-flight request cannot be recalled, so v2 goes up - // after v1 rather than replacing it. The service ends up with the newest state. - assert.Equal(t, []string{ - `resources.jobs.foo={"state":{"name":"v1"}}`, - `resources.jobs.foo={"state":{"name":"v2"}}`, - }, f.recorded()) - assert.Equal(t, "id-1", f.resourceIDFor("resources.jobs.foo")) - assert.Empty(t, q.pending) - assert.Empty(t, q.queuedOrUploading) -} - -func TestOperationQueueMergedFailureKeepsStatusAndMessageTogether(t *testing.T) { - // The service rejects error_message unless the status is failed, so a merge must - // not mix the newer status with the older message or vice versa. A resource that - // writes state and then fails is the sequence that would expose it. - f := &fakeUploader{block: make(chan struct{}), started: make(chan string, operationUploadWorkers+2)} - q := newOperationQueue(t.Context(), f) - - for i := range operationUploadWorkers { - recordState(t, q, "resources.jobs.hold"+strconv.Itoa(i), "v1") - assert.Equal(t, "resources.jobs.hold"+strconv.Itoa(i), <-f.started) - } - - recordState(t, q, "resources.jobs.foo", "v1") - q.recordFailure(t.Context(), "resources.jobs.foo", deployplan.Update, "id-1", envelope(t, "prior"), errors.New("boom")) - - close(f.block) - require.NoError(t, q.close()) - - assert.Equal(t, - bundledeployments.OperationStatusOperationStatusFailed, - f.statusFor("resources.jobs.foo")) - assert.Equal(t, "boom", f.errorMessageFor("resources.jobs.foo")) -} - -func TestOperationQueueReturnsUploadError(t *testing.T) { - uploadErr := errors.New("boom") - f := &fakeUploader{err: uploadErr} - q := newOperationQueue(t.Context(), f) - - recordState(t, q, "resources.jobs.foo", "v1") - - err := q.close() - require.Error(t, err) - assert.ErrorIs(t, err, uploadErr) - assert.Contains(t, err.Error(), "resources.jobs.foo") -} - -func TestOperationQueueKeepsRecordingAfterUploadError(t *testing.T) { - // A failed upload must not stop the ones behind it: every applied resource is - // recorded best effort, so DMS ends up as close to reality as it can get. - uploadErr := errors.New("boom") - f := &fakeUploader{err: uploadErr, done: make(chan string, 1)} - q := newOperationQueue(t.Context(), f) - - // Wait for the failing upload to finish, so the error is stored before the next - // record rather than racing it. - q.RecordOperation(t.Context(), "resources.jobs.foo", dstate.OperationInfo{Action: deployplan.Create}, "id-1", envelope(t, "v1")) - assert.Equal(t, "resources.jobs.foo", <-f.done) - - // The next resource is still accepted, even though the first upload failed. - q.RecordOperation(t.Context(), "resources.jobs.bar", dstate.OperationInfo{Action: deployplan.Create}, "id-2", envelope(t, "v1")) - - // Both were attempted, and close still reports the failure so the deploy fails. - require.ErrorIs(t, q.close(), uploadErr) - assert.ElementsMatch(t, []string{ - `resources.jobs.foo={"state":{"name":"v1"}}`, - `resources.jobs.bar={"state":{"name":"v1"}}`, - }, f.recorded()) - assert.Empty(t, q.pending) - assert.Empty(t, q.queuedOrUploading) -} - -func TestOperationQueueDrainsQueuedOperationsAfterUploadError(t *testing.T) { - // A failure does not discard work already recorded: the records DMS ends up with - // have to match the resources that were applied. - uploadErr := errors.New("boom") - f := &fakeUploader{err: uploadErr, block: make(chan struct{}), started: make(chan string, 1)} - q := newOperationQueue(t.Context(), f) - - // Every worker is parked mid-upload, so these stay queued. - for i := range operationUploadWorkers { - q.RecordOperation(t.Context(), "resources.jobs.hold"+strconv.Itoa(i), dstate.OperationInfo{Action: deployplan.Create}, "id-1", envelope(t, "v1")) - assert.Equal(t, "resources.jobs.hold"+strconv.Itoa(i), <-f.started) - } - q.RecordOperation(t.Context(), "resources.jobs.queued", dstate.OperationInfo{Action: deployplan.Create}, "id-2", envelope(t, "v1")) - - close(f.block) - require.ErrorIs(t, q.close(), uploadErr) - - // The queued operation was uploaded rather than dropped on the way out. - assert.Contains(t, f.recorded(), `resources.jobs.queued={"state":{"name":"v1"}}`) - assert.Len(t, f.recorded(), operationUploadWorkers+1) -} - -func TestOperationQueueRecordDropsUnsupportedAction(t *testing.T) { - f := &fakeUploader{} - q := newOperationQueue(t.Context(), f) - - // The state write already succeeded, so an operation that cannot be described is - // dropped with a warning rather than failing the deploy. - q.RecordOperation(t.Context(), "resources.jobs.foo", dstate.OperationInfo{Action: deployplan.Skip}, "id-1", nil) - - require.NoError(t, q.close()) - assert.Empty(t, f.recorded()) -} - -func TestOperationQueueRecordDropsOversizedState(t *testing.T) { - f := &fakeUploader{} - q := newOperationQueue(t.Context(), f) - - q.RecordOperation(t.Context(), "resources.jobs.foo", dstate.OperationInfo{Action: deployplan.Create}, "id-1", envelope(t, strings.Repeat("x", maxOperationStateSize))) - - require.NoError(t, q.close()) - assert.Empty(t, f.recorded()) -} - -func TestOperationQueueCloseIsIdempotent(t *testing.T) { - f := &fakeUploader{err: errors.New("boom")} - q := newOperationQueue(t.Context(), f) - - recordState(t, q, "resources.jobs.foo", "v1") - - require.Error(t, q.close()) - // A second close reports the same error instead of panicking on the already - // closed channel, so callers can both defer close and check it explicitly. - require.Error(t, q.close()) -} - -// serialUploader fails if two uploads for the same resource key ever overlap. -type serialUploader struct { - mu sync.Mutex - live map[string]bool - last map[string]string - uneven bool -} - -func (s *serialUploader) upload(ctx context.Context, resourceKey string, op recordedOperation) error { - s.mu.Lock() - if s.live[resourceKey] { - s.uneven = true - } - s.live[resourceKey] = true - s.mu.Unlock() - - s.mu.Lock() - defer s.mu.Unlock() - s.live[resourceKey] = false - s.last[resourceKey] = string(op.state) - return nil -} - -func TestOperationQueueUploadsOneResourceAtATime(t *testing.T) { - // Two workers must never upload the same resource at the same time. DMS stores - // one state per resource, so concurrent uploads can finish out of order and - // leave the older state as the final one. - // - // Lots of goroutines record a small set of keys, so the same key is recorded - // repeatedly while its earlier upload may still be running. serialUploader flags - // any overlap it sees. - // - // Whether a bug shows up depends on how the scheduler interleaves things, so one - // pass proves little - repeat it to get many chances at a bad ordering. - const ( - iterations = 200 - workers = 10 - perWorker = 5 - distinctKeyMod = 12 - ) - - for range iterations { - ctx := t.Context() - u := &serialUploader{live: map[string]bool{}, last: map[string]string{}} - q := newOperationQueue(ctx, u) - - // The envelopes are built up front: json.Marshal is fine on many goroutines, - // but the helper takes *testing.T, which is not. - states := make([]json.RawMessage, workers) - for w := range workers { - states[w] = envelope(t, strconv.Itoa(w)) - } - - var wg sync.WaitGroup - for w := range workers { - wg.Go(func() { - for i := range perWorker { - key := "resources.jobs.job" + strconv.Itoa((w*perWorker+i)%distinctKeyMod) - q.RecordOperation(ctx, key, dstate.OperationInfo{Action: deployplan.Update}, "id-1", states[w]) - } - }) - } - wg.Wait() - require.NoError(t, q.close()) - - require.False(t, u.uneven, "two uploads overlapped for the same resource key") - // Every distinct key was recorded, and close drained all of them. - require.Len(t, u.last, distinctKeyMod) - require.Empty(t, q.pending) - require.Empty(t, q.queuedOrUploading) - } -} - -func TestNilOperationQueueIsNoOp(t *testing.T) { - // Recording is disabled: newOperationQueue returns nil and every method is a - // no-op, so Apply does not have to branch. - q := newOperationQueue(t.Context(), nil) - require.Nil(t, q) - q.RecordOperation(t.Context(), "resources.jobs.foo", dstate.OperationInfo{Action: deployplan.Create}, "id-1", nil) - require.NoError(t, q.close()) -} diff --git a/bundle/direct/oprecorder.go b/bundle/direct/oprecorder.go index d8ed541a915..a718d4e85f8 100644 --- a/bundle/direct/oprecorder.go +++ b/bundle/direct/oprecorder.go @@ -34,9 +34,9 @@ const operationStatusInProgress bundledeployments.OperationStatus = "OPERATION_S // recordedOperation is an applied resource operation, serialized and waiting to be // uploaded to the deployment metadata service (DMS). // -// The payload is built on the apply worker rather than in the uploader so the -// queue does not hold on to the live resource struct, and so a malformed state -// fails the resource that produced it instead of the drain at the end of apply. +// The payload is built on the apply worker rather than in the uploader so the sink +// does not hold on to the live resource struct, and so a malformed state fails the +// resource that produced it instead of the drain at the end of apply. type recordedOperation struct { action bundledeployments.OperationActionType resourceID string @@ -52,6 +52,11 @@ type recordedOperation struct { state json.RawMessage } +// isFailure reports whether the operation records a resource that did not apply. +func (op recordedOperation) isFailure() bool { + return op.status == bundledeployments.OperationStatusOperationStatusFailed +} + // newStateOperation describes a state write for upload. state is the serialized // RecordedState envelope the state DB just persisted, and nil for a delete, where // the resource is gone. It errors when the state exceeds maxOperationStateSize. @@ -223,7 +228,7 @@ func (r *operationRecorder) upload(ctx context.Context, resourceKey string, op r SequenceId: sequenceID, } fields := updatableFields - if op.status == bundledeployments.OperationStatusOperationStatusFailed { + if op.isFailure() { // Mark the existing record failed and leave the rest of it alone; see // failureFields. A failure that arrives before any operation exists still // goes through CreateOperation below, carrying the prior state. diff --git a/bundle/direct/opsink.go b/bundle/direct/opsink.go new file mode 100644 index 00000000000..f6ad89b5748 --- /dev/null +++ b/bundle/direct/opsink.go @@ -0,0 +1,237 @@ +package direct + +import ( + "context" + "encoding/json" + "fmt" + "sync" + + "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/cli/bundle/direct/dstate" + "github.com/databricks/cli/libs/log" +) + +// operationSink uploads recorded operations from one background goroutine, so a +// deploy never waits on the CreateOperation round trip and the service sees one +// request at a time. Two rules shape it: +// +// - Newest write wins. Each carries the resource's full state, so a write waiting +// behind an upload is replaced rather than queued ("coalesced"); see coalesce for +// the one field that is carried over instead. +// - Uploads happen while apply runs. firstErr is what stops the deploy: DMS becomes +// the source of truth (see dstate.readDMSState), so a missing record would make +// the next deploy create a resource that already exists. +type operationSink struct { + uploader operationUploader + + // mu guards the fields below. + mu sync.Mutex + + // pending holds the one operation waiting per resource key. No key means nothing + // is waiting; a resource can write state more than once in a deploy (a recreate + // drops the entry, then saves the new resource) and the later write replaces the + // earlier one here. + pending map[string]recordedOperation + + // closed stops the uploader once everything recorded before close has gone up. + closed bool + + err error + + // wake reports that pending may have work. Buffered so recording never blocks on + // the uploader, and only ever holds one token: a full buffer already means "look + // again", which is all the uploader needs to know. + wake chan struct{} + + // done is closed when the uploader has drained pending and returned. + done chan struct{} + + // signalClose tells the uploader to stop once pending is empty. Wrapped so close + // can be deferred and still checked at a specific point. + signalClose func() +} + +// newOperationSink starts the uploader, returning nil when uploader is nil (recording +// off; every method is a no-op on a nil sink). ctx must outlive close. +func newOperationSink(ctx context.Context, uploader operationUploader) *operationSink { + if uploader == nil { + return nil + } + + s := &operationSink{ + uploader: uploader, + pending: make(map[string]recordedOperation), + wake: make(chan struct{}, 1), + done: make(chan struct{}), + } + s.signalClose = sync.OnceFunc(func() { + s.mu.Lock() + s.closed = true + s.mu.Unlock() + + // Wake the uploader so it notices, in case it is waiting on an empty pending. + s.notify() + }) + + go s.run(ctx) + return s +} + +// RecordOperation implements dstate.OperationSink: every state write becomes an +// operation, so DMS mirrors the WAL. state is already the serialized envelope, and +// nil for a delete. +// +// An earlier upload failure does not stop this: every write is still recorded, best +// effort, so DMS ends up as close to reality as it can get. +func (s *operationSink) RecordOperation(ctx context.Context, resourceKey string, info dstate.OperationInfo, resourceID string, state json.RawMessage) { + if s == nil { + return + } + + op, err := newStateOperation(info, resourceID, state) + if err != nil { + // The deploy already persisted this write locally, so failing it here would + // report an error about history for a resource that deployed fine. + log.Warnf(ctx, "Not recording operation for %s: %s", resourceKey, err) + return + } + + s.record(resourceKey, op) +} + +// recordFailure records that applying a resource failed, so the deployment history +// explains the failure instead of omitting the resource. It returns nothing: the +// deploy is already failing, and a second error would mask the one the user needs. +func (s *operationSink) recordFailure(ctx context.Context, resourceKey string, action deployplan.ActionType, resourceID string, priorState json.RawMessage, cause error) { + if s == nil { + return + } + + op, err := newFailedOperation(action, resourceID, priorState, cause) + if err != nil { + log.Warnf(ctx, "Not recording failure for %s: %s", resourceKey, err) + return + } + + s.record(resourceKey, op) +} + +// record makes op the operation waiting for resourceKey and wakes the uploader. +func (s *operationSink) record(resourceKey string, op recordedOperation) { + s.mu.Lock() + if waiting, ok := s.pending[resourceKey]; ok { + op = coalesce(waiting, op) + } + s.pending[resourceKey] = op + s.mu.Unlock() + + s.notify() +} + +// notify reports that there may be work, without ever blocking the caller. A dropped +// send means a token is already buffered, which the uploader has yet to consume - and +// it re-reads pending before it waits again, so it sees this operation either way. +func (s *operationSink) notify() { + select { + case s.wake <- struct{}{}: + default: + } +} + +// coalesce folds a write that never got uploaded into the one replacing it. The newer +// write describes the resource as it now stands, so its fields win. +// +// A failure is the exception: it carries the resource's state only when there was a +// pre-deploy record to carry (see newFailedOperation), so a create that wrote state +// and then failed would replace that state with nothing and drop the resource from the +// deployment. Inherit what the superseded write recorded instead, since that is the +// resource the failure is reporting on. The same reasoning applies on the wire when +// the write did get uploaded; see failureFields. +func coalesce(older, newer recordedOperation) recordedOperation { + if newer.isFailure() && newer.state == nil && older.state != nil { + newer.state = older.state + newer.resourceID = older.resourceID + } + return newer +} + +// take claims the operation waiting for one resource. Which resource comes first is +// unspecified: a resource has at most one operation waiting, so order matters only +// within a resource, and there coalesce has already settled it. +func (s *operationSink) take() (string, recordedOperation, bool) { + s.mu.Lock() + defer s.mu.Unlock() + + for resourceKey, op := range s.pending { + delete(s.pending, resourceKey) + return resourceKey, op, true + } + return "", recordedOperation{}, false +} + +func (s *operationSink) run(ctx context.Context) { + defer close(s.done) + + for { + resourceKey, op, ok := s.take() + if ok { + // Keep going after a failure, so one bad upload does not drop the records + // for every resource behind it. + if err := s.uploader.upload(ctx, resourceKey, op); err != nil { + s.setErr(fmt.Errorf("recording operation for %s with the deployment metadata service: %w", resourceKey, err)) + } + continue + } + + // Nothing waiting. Exiting only once pending is empty is what makes close a + // drain: everything recorded before it has been uploaded by the time it + // returns. + if s.isClosed() { + return + } + <-s.wake + } +} + +// close drains the pending operations and returns the first upload error. Every +// record caller must have returned first; calling close twice is safe, so it can be +// deferred and still checked at a specific point. +func (s *operationSink) close() error { + if s == nil { + return nil + } + + s.signalClose() + <-s.done + return s.firstErr() +} + +func (s *operationSink) isClosed() bool { + s.mu.Lock() + defer s.mu.Unlock() + return s.closed +} + +// setErr keeps the first upload error; later ones are dropped because one failure +// is enough to fail the deploy. +func (s *operationSink) setErr(err error) { + s.mu.Lock() + defer s.mu.Unlock() + + if s.err == nil { + s.err = err + } +} + +// firstErr returns the first upload error, or nil if every upload so far succeeded. +// A nil sink (recording disabled) never errors. +func (s *operationSink) firstErr() error { + if s == nil { + return nil + } + + s.mu.Lock() + defer s.mu.Unlock() + + return s.err +} diff --git a/bundle/direct/opsink_test.go b/bundle/direct/opsink_test.go new file mode 100644 index 00000000000..2f7cf89ba83 --- /dev/null +++ b/bundle/direct/opsink_test.go @@ -0,0 +1,298 @@ +package direct + +import ( + "context" + "encoding/json" + "errors" + "strconv" + "strings" + "sync" + "testing" + + "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/cli/bundle/direct/dstate" + "github.com/databricks/databricks-sdk-go/service/bundledeployments" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// fakeUploader records the uploads it receives and optionally blocks until block is +// closed, so a test can hold the uploader and observe what coalesces behind it. +type fakeUploader struct { + block chan struct{} + started chan string + err error + + mu sync.Mutex + uploads []string + actions map[string]bundledeployments.OperationActionType + resourceIDs map[string]string + statuses map[string]bundledeployments.OperationStatus + errorMessages map[string]string +} + +func (f *fakeUploader) upload(ctx context.Context, resourceKey string, op recordedOperation) error { + if f.started != nil { + f.started <- resourceKey + } + if f.block != nil { + <-f.block + } + + f.mu.Lock() + f.uploads = append(f.uploads, resourceKey+"="+string(op.state)) + if f.actions == nil { + f.actions = map[string]bundledeployments.OperationActionType{} + f.resourceIDs = map[string]string{} + f.statuses = map[string]bundledeployments.OperationStatus{} + f.errorMessages = map[string]string{} + } + f.actions[resourceKey] = op.action + f.resourceIDs[resourceKey] = op.resourceID + f.statuses[resourceKey] = op.status + f.errorMessages[resourceKey] = op.errorMessage + f.mu.Unlock() + + return f.err +} + +func (f *fakeUploader) recorded() []string { + f.mu.Lock() + defer f.mu.Unlock() + return append([]string(nil), f.uploads...) +} + +func (f *fakeUploader) actionFor(resourceKey string) bundledeployments.OperationActionType { + f.mu.Lock() + defer f.mu.Unlock() + return f.actions[resourceKey] +} + +func (f *fakeUploader) resourceIDFor(resourceKey string) string { + f.mu.Lock() + defer f.mu.Unlock() + return f.resourceIDs[resourceKey] +} + +func (f *fakeUploader) statusFor(resourceKey string) bundledeployments.OperationStatus { + f.mu.Lock() + defer f.mu.Unlock() + return f.statuses[resourceKey] +} + +func (f *fakeUploader) errorMessageFor(resourceKey string) string { + f.mu.Lock() + defer f.mu.Unlock() + return f.errorMessages[resourceKey] +} + +// envelope builds the serialized RecordedState the state DB hands the sink. +func envelope(t *testing.T, name string) json.RawMessage { + t.Helper() + raw, err := json.Marshal(dstate.RecordedState{State: json.RawMessage(`{"name":"` + name + `"}`)}) + require.NoError(t, err) + return raw +} + +func recordState(t *testing.T, s *operationSink, resourceKey, name string) { + t.Helper() + s.RecordOperation(t.Context(), resourceKey, dstate.OperationInfo{Action: deployplan.Update}, "id-1", envelope(t, name)) +} + +func TestOperationSinkUploadsEachOperation(t *testing.T) { + f := &fakeUploader{} + s := newOperationSink(t.Context(), f) + + for i := range 20 { + recordState(t, s, "resources.jobs.job"+strconv.Itoa(i), "n") + } + require.NoError(t, s.close()) + + assert.Len(t, f.recorded(), 20) +} + +func TestOperationSinkCoalescesWritesBehindAnUpload(t *testing.T) { + // Hold the uploader on the first write so the two behind it pile up. They carry + // the resource's full state, so only the newest needs to go: the resource costs + // two requests rather than three. + f := &fakeUploader{block: make(chan struct{}), started: make(chan string, 2)} + s := newOperationSink(t.Context(), f) + + recordState(t, s, "resources.jobs.foo", "v1") + assert.Equal(t, "resources.jobs.foo", <-f.started) + + recordState(t, s, "resources.jobs.foo", "v2") + recordState(t, s, "resources.jobs.foo", "v3") + + close(f.block) + require.NoError(t, s.close()) + + assert.Equal(t, []string{ + `resources.jobs.foo={"state":{"name":"v1"}}`, + `resources.jobs.foo={"state":{"name":"v3"}}`, + }, f.recorded()) +} + +func TestOperationSinkCoalescedFailureKeepsTheStateItReplaces(t *testing.T) { + // A create writes state and then fails waiting for the resource to come up. If + // the failure catches the write before it is uploaded, it must not replace that + // state with its own emptiness: a resource recorded without state is dropped from + // the deployment, so the next plan would create it a second time. + f := &fakeUploader{block: make(chan struct{}), started: make(chan string, 2)} + s := newOperationSink(t.Context(), f) + + // Occupy the uploader with an unrelated resource, so the two writes below both + // land in pending and coalesce. + recordState(t, s, "resources.jobs.busy", "v1") + assert.Equal(t, "resources.jobs.busy", <-f.started) + + s.RecordOperation(t.Context(), "resources.job_runs.my_run", dstate.OperationInfo{Action: deployplan.Create}, "run-1", envelope(t, "the run")) + // priorState and priorID are empty: the resource was created in this deploy, so + // there is no pre-deploy record to report. + s.recordFailure(t.Context(), "resources.job_runs.my_run", deployplan.Create, "", nil, errors.New("run did not succeed: FAILED")) + + close(f.block) + require.NoError(t, s.close()) + + assert.Equal(t, []string{ + `resources.jobs.busy={"state":{"name":"v1"}}`, + `resources.job_runs.my_run={"state":{"name":"the run"}}`, + }, f.recorded()) + assert.Equal(t, "run-1", f.resourceIDFor("resources.job_runs.my_run")) + assert.Equal(t, + bundledeployments.OperationStatusOperationStatusFailed, + f.statusFor("resources.job_runs.my_run")) + assert.Equal(t, "run did not succeed: FAILED", f.errorMessageFor("resources.job_runs.my_run")) +} + +func TestOperationSinkCoalescedDeleteStillClearsState(t *testing.T) { + // A delete legitimately carries no state, and coalescing must let it through: the + // resource is gone, and keeping the state it replaces would leave it listed. + f := &fakeUploader{block: make(chan struct{}), started: make(chan string, 2)} + s := newOperationSink(t.Context(), f) + + recordState(t, s, "resources.jobs.busy", "v1") + assert.Equal(t, "resources.jobs.busy", <-f.started) + + s.RecordOperation(t.Context(), "resources.jobs.foo", dstate.OperationInfo{Action: deployplan.Update}, "id-1", envelope(t, "before")) + s.RecordOperation(t.Context(), "resources.jobs.foo", dstate.OperationInfo{Action: deployplan.Delete}, "id-1", nil) + + close(f.block) + require.NoError(t, s.close()) + + assert.Equal(t, []string{ + `resources.jobs.busy={"state":{"name":"v1"}}`, + `resources.jobs.foo=`, + }, f.recorded()) + assert.Equal(t, + bundledeployments.OperationActionTypeOperationActionTypeDelete, + f.actionFor("resources.jobs.foo")) +} + +func TestOperationSinkRecordDuringUploadIsStillUploaded(t *testing.T) { + f := &fakeUploader{block: make(chan struct{}), started: make(chan string, 2)} + s := newOperationSink(t.Context(), f) + + recordState(t, s, "resources.jobs.foo", "v1") + assert.Equal(t, "resources.jobs.foo", <-f.started) + + // The uploader has taken this key off pending and is uploading it right now. + recordState(t, s, "resources.jobs.foo", "v2") + + close(f.block) + require.NoError(t, s.close()) + + // Two uploads, in order: an in-flight request cannot be recalled, so v2 goes up + // after v1 rather than replacing it. The service ends up with the newest state. + assert.Equal(t, []string{ + `resources.jobs.foo={"state":{"name":"v1"}}`, + `resources.jobs.foo={"state":{"name":"v2"}}`, + }, f.recorded()) + assert.Empty(t, s.pending) +} + +func TestOperationSinkReturnsUploadError(t *testing.T) { + uploadErr := errors.New("boom") + f := &fakeUploader{err: uploadErr} + s := newOperationSink(t.Context(), f) + + recordState(t, s, "resources.jobs.foo", "v1") + + err := s.close() + require.Error(t, err) + assert.ErrorIs(t, err, uploadErr) + assert.ErrorContains(t, err, "resources.jobs.foo") +} + +func TestOperationSinkKeepsRecordingAfterUploadError(t *testing.T) { + // One failed upload must not drop the records for everything behind it, so DMS + // ends up as close to reality as it can get. + f := &fakeUploader{err: errors.New("boom")} + s := newOperationSink(t.Context(), f) + + recordState(t, s, "resources.jobs.foo", "v1") + recordState(t, s, "resources.jobs.bar", "v1") + + require.Error(t, s.close()) + assert.Len(t, f.recorded(), 2) +} + +func TestOperationSinkFirstErrIsWhatStopsTheDeploy(t *testing.T) { + f := &fakeUploader{err: errors.New("boom")} + s := newOperationSink(t.Context(), f) + + assert.NoError(t, s.firstErr()) + + recordState(t, s, "resources.jobs.foo", "v1") + require.Error(t, s.close()) + + // Reported after the fact too, so the caller can check once more before it + // completes the version. + assert.Error(t, s.firstErr()) +} + +func TestOperationSinkDropsUnsupportedAction(t *testing.T) { + f := &fakeUploader{} + s := newOperationSink(t.Context(), f) + + // Skip never reaches a sink; it is dropped with a warning rather than failing a + // resource that deployed fine. + s.RecordOperation(t.Context(), "resources.jobs.foo", dstate.OperationInfo{Action: deployplan.Skip}, "id-1", nil) + require.NoError(t, s.close()) + + assert.Empty(t, f.recorded()) +} + +func TestOperationSinkDropsOversizedState(t *testing.T) { + f := &fakeUploader{} + s := newOperationSink(t.Context(), f) + + s.RecordOperation(t.Context(), "resources.jobs.foo", dstate.OperationInfo{Action: deployplan.Create}, "id-1", envelope(t, strings.Repeat("x", maxOperationStateSize))) + require.NoError(t, s.close()) + + assert.Empty(t, f.recorded()) +} + +func TestOperationSinkCloseIsIdempotent(t *testing.T) { + f := &fakeUploader{} + s := newOperationSink(t.Context(), f) + + recordState(t, s, "resources.jobs.foo", "v1") + + require.NoError(t, s.close()) + require.NoError(t, s.close()) + assert.Len(t, f.recorded(), 1) +} + +func TestNilOperationSinkIsNoOp(t *testing.T) { + var s *operationSink + s.RecordOperation(t.Context(), "resources.jobs.foo", dstate.OperationInfo{Action: deployplan.Create}, "id-1", nil) + s.recordFailure(t.Context(), "resources.jobs.foo", deployplan.Create, "id-1", nil, errors.New("boom")) + assert.NoError(t, s.firstErr()) + assert.NoError(t, s.close()) +} + +func TestNewOperationSinkNilUploaderIsNil(t *testing.T) { + // Recording off: the sink is nil so the state DB's nil check leaves it unset. + assert.Nil(t, newOperationSink(t.Context(), nil)) +} From ed5d9b15a35636f99032b0741439dbb80a1fbdbb Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 17 Aug 2026 00:22:11 +0000 Subject: [PATCH 085/125] bundle: queue resource keys, keep the operation in a map The sink kept its own doorbell channel plus a closed flag so the uploader could sleep on an empty map. Carrying the resource key in the queue and looking the operation up in a map when it comes out is the same thing with less to it: coalescing falls out of the map, and "for key := range queue" handles both work and shutdown. A key is only queued when nothing was waiting for it, so at most one token exists per resource. A queue sized for the deploy's resources therefore cannot fill, which is what lets the send stay unconditional - recording runs while the state DB holds its lock, so blocking there would stall every other resource's state write. Co-authored-by: Isaac --- bundle/direct/bundle_apply.go | 2 +- bundle/direct/opsink.go | 145 ++++++++++++++-------------------- bundle/direct/opsink_test.go | 24 +++--- 3 files changed, 74 insertions(+), 97 deletions(-) diff --git a/bundle/direct/bundle_apply.go b/bundle/direct/bundle_apply.go index c16f728cdd2..3707ee23d96 100644 --- a/bundle/direct/bundle_apply.go +++ b/bundle/direct/bundle_apply.go @@ -41,7 +41,7 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa // // The state DB records through it, so every state write becomes an operation and // DMS mirrors the WAL. - opSink := newOperationSink(ctx, b.OpRec) + opSink := newOperationSink(ctx, b.OpRec, len(plan.Plan)) if opSink != nil { // Assigned only when non-nil: a nil *operationSink in an interface is not a // nil interface, so the state DB's nil check would not see it. diff --git a/bundle/direct/opsink.go b/bundle/direct/opsink.go index f6ad89b5748..7b63a42aa31 100644 --- a/bundle/direct/opsink.go +++ b/bundle/direct/opsink.go @@ -11,67 +11,62 @@ import ( "github.com/databricks/cli/libs/log" ) -// operationSink uploads recorded operations from one background goroutine, so a -// deploy never waits on the CreateOperation round trip and the service sees one -// request at a time. Two rules shape it: +// operationSink uploads recorded operations from one background goroutine, so a deploy +// never waits on the CreateOperation round trip and the service sees one request at a +// time. // -// - Newest write wins. Each carries the resource's full state, so a write waiting -// behind an upload is replaced rather than queued ("coalesced"); see coalesce for -// the one field that is carried over instead. -// - Uploads happen while apply runs. firstErr is what stops the deploy: DMS becomes -// the source of truth (see dstate.readDMSState), so a missing record would make -// the next deploy create a resource that already exists. +// The queue carries resource keys and pending holds the operation for each, which is +// what makes coalescing fall out: a second write for a resource replaces the first in +// the map, and the key already in the queue picks up whichever operation is there when +// the uploader reaches it. +// +// close reports the first upload failure, which fails the deploy: DMS becomes the +// source of truth (see dstate.readDMSState), so a missing record would make the next +// deploy create a resource that already exists. type operationSink struct { uploader operationUploader + // queue carries the resource keys that have an operation waiting. A key is sent + // only when nothing was waiting for it, so at most one token exists per resource + // and a queue sized for the deploy's resources cannot fill. That is what keeps + // recording from blocking, which matters because it runs while the state DB holds + // its lock - a blocked send there would stall every other resource's state write. + queue chan string + + // done is closed once the uploader has drained the queue and returned. + done chan struct{} + + // stopQueue closes the queue. Wrapped so close can be deferred and still checked + // at a specific point. + stopQueue func() + // mu guards the fields below. mu sync.Mutex - // pending holds the one operation waiting per resource key. No key means nothing - // is waiting; a resource can write state more than once in a deploy (a recreate - // drops the entry, then saves the new resource) and the later write replaces the - // earlier one here. + // pending holds the operation waiting per resource key. A key is absent once the + // uploader has taken its operation. pending map[string]recordedOperation - // closed stops the uploader once everything recorded before close has gone up. - closed bool - err error - - // wake reports that pending may have work. Buffered so recording never blocks on - // the uploader, and only ever holds one token: a full buffer already means "look - // again", which is all the uploader needs to know. - wake chan struct{} - - // done is closed when the uploader has drained pending and returned. - done chan struct{} - - // signalClose tells the uploader to stop once pending is empty. Wrapped so close - // can be deferred and still checked at a specific point. - signalClose func() } // newOperationSink starts the uploader, returning nil when uploader is nil (recording -// off; every method is a no-op on a nil sink). ctx must outlive close. -func newOperationSink(ctx context.Context, uploader operationUploader) *operationSink { +// off; every method is a no-op on a nil sink). resources is how many resource keys the +// deploy can record, which is what the queue is sized for. ctx must outlive close. +func newOperationSink(ctx context.Context, uploader operationUploader, resources int) *operationSink { if uploader == nil { return nil } s := &operationSink{ uploader: uploader, - pending: make(map[string]recordedOperation), - wake: make(chan struct{}, 1), - done: make(chan struct{}), + // At least one slot: a zero-capacity queue would make the send block, which + // is the one thing the sizing above exists to prevent. + queue: make(chan string, max(resources, 1)), + done: make(chan struct{}), + pending: make(map[string]recordedOperation), } - s.signalClose = sync.OnceFunc(func() { - s.mu.Lock() - s.closed = true - s.mu.Unlock() - - // Wake the uploader so it notices, in case it is waiting on an empty pending. - s.notify() - }) + s.stopQueue = sync.OnceFunc(func() { close(s.queue) }) go s.run(ctx) return s @@ -116,25 +111,21 @@ func (s *operationSink) recordFailure(ctx context.Context, resourceKey string, a s.record(resourceKey, op) } -// record makes op the operation waiting for resourceKey and wakes the uploader. +// record makes op the operation waiting for resourceKey. Recording on a closed sink +// panics; every caller must have returned before close. func (s *operationSink) record(resourceKey string, op recordedOperation) { s.mu.Lock() - if waiting, ok := s.pending[resourceKey]; ok { + waiting, queued := s.pending[resourceKey] + if queued { op = coalesce(waiting, op) } s.pending[resourceKey] = op s.mu.Unlock() - s.notify() -} - -// notify reports that there may be work, without ever blocking the caller. A dropped -// send means a token is already buffered, which the uploader has yet to consume - and -// it re-reads pending before it waits again, so it sees this operation either way. -func (s *operationSink) notify() { - select { - case s.wake <- struct{}{}: - default: + // Already queued: the uploader reads the map when it reaches the key, so it picks + // up what was just stored without a second token. + if !queued { + s.queue <- resourceKey } } @@ -155,45 +146,37 @@ func coalesce(older, newer recordedOperation) recordedOperation { return newer } -// take claims the operation waiting for one resource. Which resource comes first is -// unspecified: a resource has at most one operation waiting, so order matters only -// within a resource, and there coalesce has already settled it. -func (s *operationSink) take() (string, recordedOperation, bool) { +// take claims the operation waiting for resourceKey. +func (s *operationSink) take(resourceKey string) (recordedOperation, bool) { s.mu.Lock() defer s.mu.Unlock() - for resourceKey, op := range s.pending { - delete(s.pending, resourceKey) - return resourceKey, op, true - } - return "", recordedOperation{}, false + op, ok := s.pending[resourceKey] + delete(s.pending, resourceKey) + return op, ok } func (s *operationSink) run(ctx context.Context) { defer close(s.done) - for { - resourceKey, op, ok := s.take() - if ok { - // Keep going after a failure, so one bad upload does not drop the records - // for every resource behind it. - if err := s.uploader.upload(ctx, resourceKey, op); err != nil { - s.setErr(fmt.Errorf("recording operation for %s with the deployment metadata service: %w", resourceKey, err)) - } + for resourceKey := range s.queue { + op, ok := s.take(resourceKey) + if !ok { + // A key is queued only when nothing is waiting for it, so this cannot + // happen; the check is here so a stray token could never upload a + // zero-valued operation. continue } - // Nothing waiting. Exiting only once pending is empty is what makes close a - // drain: everything recorded before it has been uploaded by the time it - // returns. - if s.isClosed() { - return + // Keep going after a failure, so one bad upload does not drop the records for + // every resource behind it. + if err := s.uploader.upload(ctx, resourceKey, op); err != nil { + s.setErr(fmt.Errorf("recording operation for %s with the deployment metadata service: %w", resourceKey, err)) } - <-s.wake } } -// close drains the pending operations and returns the first upload error. Every +// close drains the recorded operations and returns the first upload error. Every // record caller must have returned first; calling close twice is safe, so it can be // deferred and still checked at a specific point. func (s *operationSink) close() error { @@ -201,17 +184,11 @@ func (s *operationSink) close() error { return nil } - s.signalClose() + s.stopQueue() <-s.done return s.firstErr() } -func (s *operationSink) isClosed() bool { - s.mu.Lock() - defer s.mu.Unlock() - return s.closed -} - // setErr keeps the first upload error; later ones are dropped because one failure // is enough to fail the deploy. func (s *operationSink) setErr(err error) { diff --git a/bundle/direct/opsink_test.go b/bundle/direct/opsink_test.go index 2f7cf89ba83..b367042e7fb 100644 --- a/bundle/direct/opsink_test.go +++ b/bundle/direct/opsink_test.go @@ -101,7 +101,7 @@ func recordState(t *testing.T, s *operationSink, resourceKey, name string) { func TestOperationSinkUploadsEachOperation(t *testing.T) { f := &fakeUploader{} - s := newOperationSink(t.Context(), f) + s := newOperationSink(t.Context(), f, 32) for i := range 20 { recordState(t, s, "resources.jobs.job"+strconv.Itoa(i), "n") @@ -116,7 +116,7 @@ func TestOperationSinkCoalescesWritesBehindAnUpload(t *testing.T) { // the resource's full state, so only the newest needs to go: the resource costs // two requests rather than three. f := &fakeUploader{block: make(chan struct{}), started: make(chan string, 2)} - s := newOperationSink(t.Context(), f) + s := newOperationSink(t.Context(), f, 32) recordState(t, s, "resources.jobs.foo", "v1") assert.Equal(t, "resources.jobs.foo", <-f.started) @@ -139,7 +139,7 @@ func TestOperationSinkCoalescedFailureKeepsTheStateItReplaces(t *testing.T) { // state with its own emptiness: a resource recorded without state is dropped from // the deployment, so the next plan would create it a second time. f := &fakeUploader{block: make(chan struct{}), started: make(chan string, 2)} - s := newOperationSink(t.Context(), f) + s := newOperationSink(t.Context(), f, 32) // Occupy the uploader with an unrelated resource, so the two writes below both // land in pending and coalesce. @@ -169,7 +169,7 @@ func TestOperationSinkCoalescedDeleteStillClearsState(t *testing.T) { // A delete legitimately carries no state, and coalescing must let it through: the // resource is gone, and keeping the state it replaces would leave it listed. f := &fakeUploader{block: make(chan struct{}), started: make(chan string, 2)} - s := newOperationSink(t.Context(), f) + s := newOperationSink(t.Context(), f, 32) recordState(t, s, "resources.jobs.busy", "v1") assert.Equal(t, "resources.jobs.busy", <-f.started) @@ -191,7 +191,7 @@ func TestOperationSinkCoalescedDeleteStillClearsState(t *testing.T) { func TestOperationSinkRecordDuringUploadIsStillUploaded(t *testing.T) { f := &fakeUploader{block: make(chan struct{}), started: make(chan string, 2)} - s := newOperationSink(t.Context(), f) + s := newOperationSink(t.Context(), f, 32) recordState(t, s, "resources.jobs.foo", "v1") assert.Equal(t, "resources.jobs.foo", <-f.started) @@ -214,7 +214,7 @@ func TestOperationSinkRecordDuringUploadIsStillUploaded(t *testing.T) { func TestOperationSinkReturnsUploadError(t *testing.T) { uploadErr := errors.New("boom") f := &fakeUploader{err: uploadErr} - s := newOperationSink(t.Context(), f) + s := newOperationSink(t.Context(), f, 32) recordState(t, s, "resources.jobs.foo", "v1") @@ -228,7 +228,7 @@ func TestOperationSinkKeepsRecordingAfterUploadError(t *testing.T) { // One failed upload must not drop the records for everything behind it, so DMS // ends up as close to reality as it can get. f := &fakeUploader{err: errors.New("boom")} - s := newOperationSink(t.Context(), f) + s := newOperationSink(t.Context(), f, 32) recordState(t, s, "resources.jobs.foo", "v1") recordState(t, s, "resources.jobs.bar", "v1") @@ -239,7 +239,7 @@ func TestOperationSinkKeepsRecordingAfterUploadError(t *testing.T) { func TestOperationSinkFirstErrIsWhatStopsTheDeploy(t *testing.T) { f := &fakeUploader{err: errors.New("boom")} - s := newOperationSink(t.Context(), f) + s := newOperationSink(t.Context(), f, 32) assert.NoError(t, s.firstErr()) @@ -253,7 +253,7 @@ func TestOperationSinkFirstErrIsWhatStopsTheDeploy(t *testing.T) { func TestOperationSinkDropsUnsupportedAction(t *testing.T) { f := &fakeUploader{} - s := newOperationSink(t.Context(), f) + s := newOperationSink(t.Context(), f, 32) // Skip never reaches a sink; it is dropped with a warning rather than failing a // resource that deployed fine. @@ -265,7 +265,7 @@ func TestOperationSinkDropsUnsupportedAction(t *testing.T) { func TestOperationSinkDropsOversizedState(t *testing.T) { f := &fakeUploader{} - s := newOperationSink(t.Context(), f) + s := newOperationSink(t.Context(), f, 32) s.RecordOperation(t.Context(), "resources.jobs.foo", dstate.OperationInfo{Action: deployplan.Create}, "id-1", envelope(t, strings.Repeat("x", maxOperationStateSize))) require.NoError(t, s.close()) @@ -275,7 +275,7 @@ func TestOperationSinkDropsOversizedState(t *testing.T) { func TestOperationSinkCloseIsIdempotent(t *testing.T) { f := &fakeUploader{} - s := newOperationSink(t.Context(), f) + s := newOperationSink(t.Context(), f, 32) recordState(t, s, "resources.jobs.foo", "v1") @@ -294,5 +294,5 @@ func TestNilOperationSinkIsNoOp(t *testing.T) { func TestNewOperationSinkNilUploaderIsNil(t *testing.T) { // Recording off: the sink is nil so the state DB's nil check leaves it unset. - assert.Nil(t, newOperationSink(t.Context(), nil)) + assert.Nil(t, newOperationSink(t.Context(), nil, 32)) } From 905ff9e1db1143584395f3154316be9c78ec1f7f Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 17 Aug 2026 00:29:33 +0000 Subject: [PATCH 086/125] bundle: correct two comments left over from the operation queue operationQueue is gone, so the uploader is no longer "the operationQueue workers". Also record at the call site why the queue is sized by the plan: that is the assumption recording relies on to never block, and it is only true because every recorded key comes from the plan. Co-authored-by: Isaac --- bundle/direct/bundle_apply.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/bundle/direct/bundle_apply.go b/bundle/direct/bundle_apply.go index 3707ee23d96..4b41f2e73f0 100644 --- a/bundle/direct/bundle_apply.go +++ b/bundle/direct/bundle_apply.go @@ -41,6 +41,9 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa // // The state DB records through it, so every state write becomes an operation and // DMS mirrors the WAL. + // Sized by the plan because every recorded key comes from it: makeGraph's nodes are + // its keys, the walk below iterates those, and each resource records under its own + // key. Recording relies on that to never block; see operationSink.queue. opSink := newOperationSink(ctx, b.OpRec, len(plan.Plan)) if opSink != nil { // Assigned only when non-nil: a nil *operationSink in an interface is not a From 6eba109c738cfa9ad3630018ce34375f59cc1253 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 17 Aug 2026 09:47:31 +0000 Subject: [PATCH 087/125] bundle: hold the deploy back when recording falls behind The queue was sized so it could never fill, which meant apply could run arbitrarily far ahead of what the service had been told. A deploy that ends that way has resources applied but unrecorded, and DMS is the source of truth for the next plan. The queue is now a fixed ten slots and the send waits when they are all taken. At one waiting operation per resource that bounds the lag to about the apply parallelism. Recording moved out from under the state DB lock to make that safe: SaveState and DeleteState now persist the write, release the lock, and report afterwards. Waiting while holding db.mu would have held up every other resource's write rather than just the one that got ahead - and holding a lock across network latency is worth avoiding regardless. Co-authored-by: Isaac --- bundle/direct/bundle_apply.go | 5 +-- bundle/direct/dstate/state.go | 71 +++++++++++++++++++++++++---------- bundle/direct/opsink.go | 40 +++++++++++--------- bundle/direct/opsink_test.go | 64 +++++++++++++++++++++++++------ 4 files changed, 128 insertions(+), 52 deletions(-) diff --git a/bundle/direct/bundle_apply.go b/bundle/direct/bundle_apply.go index 4b41f2e73f0..c16f728cdd2 100644 --- a/bundle/direct/bundle_apply.go +++ b/bundle/direct/bundle_apply.go @@ -41,10 +41,7 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa // // The state DB records through it, so every state write becomes an operation and // DMS mirrors the WAL. - // Sized by the plan because every recorded key comes from it: makeGraph's nodes are - // its keys, the walk below iterates those, and each resource records under its own - // key. Recording relies on that to never block; see operationSink.queue. - opSink := newOperationSink(ctx, b.OpRec, len(plan.Plan)) + opSink := newOperationSink(ctx, b.OpRec) if opSink != nil { // Assigned only when non-nil: a nil *operationSink in an interface is not a // nil interface, so the state DB's nil check would not see it. diff --git a/bundle/direct/dstate/state.go b/bundle/direct/dstate/state.go index 77983f81b20..bfd0c578065 100644 --- a/bundle/direct/dstate/state.go +++ b/bundle/direct/dstate/state.go @@ -136,6 +136,27 @@ func NewDatabase(lineage string, serial int) Database { // the bundle does not record deployment history. func (db *DeploymentState) SaveState(ctx context.Context, key, newID string, state any, dependsOn []deployplan.DependsOnEntry, info OperationInfo) error { db.AssertOpenedForWrite() + + sink, recorded, err := db.saveStateEntry(key, newID, state, dependsOn) + if err != nil { + return err + } + + // Recorded after the WAL write, so DMS never reports a state the deploy failed to + // persist locally, and outside the lock because recording applies backpressure: + // it waits when the service is behind, and waiting under db.mu would hold up every + // other resource's write rather than just this one. + if sink != nil { + sink.RecordOperation(ctx, key, info, newID, recorded) + } + + return nil +} + +// saveStateEntry writes the resource's state and returns the sink to report it to, +// along with the serialized envelope to report, or a nil sink when the bundle does not +// record deployment history. +func (db *DeploymentState) saveStateEntry(key, newID string, state any, dependsOn []deployplan.DependsOnEntry) (OperationSink, json.RawMessage, error) { db.mu.Lock() defer db.mu.Unlock() @@ -145,7 +166,7 @@ func (db *DeploymentState) SaveState(ctx context.Context, key, newID string, sta jsonMessage, err := json.Marshal(state) if err != nil { - return err + return nil, nil, err } entry := ResourceEntry{ @@ -156,32 +177,49 @@ func (db *DeploymentState) SaveState(ctx context.Context, key, newID string, sta err = appendJSONLine(db.walFile, WALEntry{Key: key, Value: &entry}) if err != nil { - return err + return nil, nil, err } db.stateIDs[key] = newID - // Recorded after the WAL write, so DMS never reports a state the deploy failed to - // persist locally. - if db.sink != nil { - recorded, err := json.Marshal(RecordedState{State: entry.State, DependsOn: dependsOn}) - if err != nil { - return err - } - db.sink.RecordOperation(ctx, key, info, newID, recorded) + if db.sink == nil { + return nil, nil, nil } - return nil + // Serialized here, while the entry the WAL took is still to hand. + recorded, err := json.Marshal(RecordedState{State: entry.State, DependsOn: dependsOn}) + if err != nil { + return nil, nil, err + } + return db.sink, recorded, nil } // DeleteState drops the resource's state entry. info distinguishes a real delete // from the intermediate drop a recreate performs, both of which are recorded. func (db *DeploymentState) DeleteState(ctx context.Context, key string, info OperationInfo) error { db.AssertOpenedForWrite() + + sink, deletedID, err := db.deleteStateEntry(key) + if err != nil { + return err + } + + // State is nil: the resource no longer exists. Recorded outside the lock for the + // same reason as SaveState. + if sink != nil { + sink.RecordOperation(ctx, key, info, deletedID, nil) + } + + return nil +} + +// deleteStateEntry drops the resource's state entry and returns the sink to report it +// to, along with the id it had, or a nil sink when there is nothing to report. +func (db *DeploymentState) deleteStateEntry(key string) (OperationSink, string, error) { db.mu.Lock() defer db.mu.Unlock() if db.Data.State == nil { - return nil + return nil, "", nil } // Read before the delete: DMS needs the id to say which resource went away. @@ -189,16 +227,11 @@ func (db *DeploymentState) DeleteState(ctx context.Context, key string, info Ope err := appendJSONLine(db.walFile, WALEntry{Key: key}) if err != nil { - return err + return nil, "", err } delete(db.stateIDs, key) - // State is nil: the resource no longer exists. - if db.sink != nil { - db.sink.RecordOperation(ctx, key, info, deletedID, nil) - } - - return nil + return db.sink, deletedID, nil } func (db *DeploymentState) GetResourceEntry(key string) (ResourceEntry, bool) { diff --git a/bundle/direct/opsink.go b/bundle/direct/opsink.go index 7b63a42aa31..5a9a257e10f 100644 --- a/bundle/direct/opsink.go +++ b/bundle/direct/opsink.go @@ -11,9 +11,16 @@ import ( "github.com/databricks/cli/libs/log" ) +// operationSinkQueueSize is how many resources may have an unrecorded state write +// before the next one waits. Recording is deliberately not free: a deploy that ran far +// ahead of the service would leave a long tail of resources applied but unrecorded, and +// DMS is the source of truth for the next plan. At one waiting operation per resource +// this bounds the lag to roughly the apply parallelism. +const operationSinkQueueSize = 10 + // operationSink uploads recorded operations from one background goroutine, so a deploy -// never waits on the CreateOperation round trip and the service sees one request at a -// time. +// does not wait on every CreateOperation round trip and the service sees one request at +// a time. // // The queue carries resource keys and pending holds the operation for each, which is // what makes coalescing fall out: a second write for a resource replaces the first in @@ -27,10 +34,10 @@ type operationSink struct { uploader operationUploader // queue carries the resource keys that have an operation waiting. A key is sent - // only when nothing was waiting for it, so at most one token exists per resource - // and a queue sized for the deploy's resources cannot fill. That is what keeps - // recording from blocking, which matters because it runs while the state DB holds - // its lock - a blocked send there would stall every other resource's state write. + // only when nothing was waiting for it, so a resource never occupies more than one + // slot; once the queue is full, recording waits for the uploader, which is what + // holds the deploy back. Callers must therefore record outside the state DB lock - + // see dstate.SaveState. queue chan string // done is closed once the uploader has drained the queue and returned. @@ -51,20 +58,17 @@ type operationSink struct { } // newOperationSink starts the uploader, returning nil when uploader is nil (recording -// off; every method is a no-op on a nil sink). resources is how many resource keys the -// deploy can record, which is what the queue is sized for. ctx must outlive close. -func newOperationSink(ctx context.Context, uploader operationUploader, resources int) *operationSink { +// off; every method is a no-op on a nil sink). ctx must outlive close. +func newOperationSink(ctx context.Context, uploader operationUploader) *operationSink { if uploader == nil { return nil } s := &operationSink{ uploader: uploader, - // At least one slot: a zero-capacity queue would make the send block, which - // is the one thing the sizing above exists to prevent. - queue: make(chan string, max(resources, 1)), - done: make(chan struct{}), - pending: make(map[string]recordedOperation), + queue: make(chan string, operationSinkQueueSize), + done: make(chan struct{}), + pending: make(map[string]recordedOperation), } s.stopQueue = sync.OnceFunc(func() { close(s.queue) }) @@ -111,8 +115,9 @@ func (s *operationSink) recordFailure(ctx context.Context, resourceKey string, a s.record(resourceKey, op) } -// record makes op the operation waiting for resourceKey. Recording on a closed sink -// panics; every caller must have returned before close. +// record makes op the operation waiting for resourceKey, waiting for the uploader when +// the queue is full. Recording on a closed sink panics; every caller must have returned +// before close. func (s *operationSink) record(resourceKey string, op recordedOperation) { s.mu.Lock() waiting, queued := s.pending[resourceKey] @@ -123,7 +128,8 @@ func (s *operationSink) record(resourceKey string, op recordedOperation) { s.mu.Unlock() // Already queued: the uploader reads the map when it reaches the key, so it picks - // up what was just stored without a second token. + // up what was just stored without a second token - and without waiting, since a + // resource that is already represented in the queue is not running ahead. if !queued { s.queue <- resourceKey } diff --git a/bundle/direct/opsink_test.go b/bundle/direct/opsink_test.go index b367042e7fb..a347e9e1f7f 100644 --- a/bundle/direct/opsink_test.go +++ b/bundle/direct/opsink_test.go @@ -8,6 +8,7 @@ import ( "strings" "sync" "testing" + "time" "github.com/databricks/cli/bundle/deployplan" "github.com/databricks/cli/bundle/direct/dstate" @@ -101,7 +102,7 @@ func recordState(t *testing.T, s *operationSink, resourceKey, name string) { func TestOperationSinkUploadsEachOperation(t *testing.T) { f := &fakeUploader{} - s := newOperationSink(t.Context(), f, 32) + s := newOperationSink(t.Context(), f) for i := range 20 { recordState(t, s, "resources.jobs.job"+strconv.Itoa(i), "n") @@ -116,7 +117,7 @@ func TestOperationSinkCoalescesWritesBehindAnUpload(t *testing.T) { // the resource's full state, so only the newest needs to go: the resource costs // two requests rather than three. f := &fakeUploader{block: make(chan struct{}), started: make(chan string, 2)} - s := newOperationSink(t.Context(), f, 32) + s := newOperationSink(t.Context(), f) recordState(t, s, "resources.jobs.foo", "v1") assert.Equal(t, "resources.jobs.foo", <-f.started) @@ -139,7 +140,7 @@ func TestOperationSinkCoalescedFailureKeepsTheStateItReplaces(t *testing.T) { // state with its own emptiness: a resource recorded without state is dropped from // the deployment, so the next plan would create it a second time. f := &fakeUploader{block: make(chan struct{}), started: make(chan string, 2)} - s := newOperationSink(t.Context(), f, 32) + s := newOperationSink(t.Context(), f) // Occupy the uploader with an unrelated resource, so the two writes below both // land in pending and coalesce. @@ -169,7 +170,7 @@ func TestOperationSinkCoalescedDeleteStillClearsState(t *testing.T) { // A delete legitimately carries no state, and coalescing must let it through: the // resource is gone, and keeping the state it replaces would leave it listed. f := &fakeUploader{block: make(chan struct{}), started: make(chan string, 2)} - s := newOperationSink(t.Context(), f, 32) + s := newOperationSink(t.Context(), f) recordState(t, s, "resources.jobs.busy", "v1") assert.Equal(t, "resources.jobs.busy", <-f.started) @@ -191,7 +192,7 @@ func TestOperationSinkCoalescedDeleteStillClearsState(t *testing.T) { func TestOperationSinkRecordDuringUploadIsStillUploaded(t *testing.T) { f := &fakeUploader{block: make(chan struct{}), started: make(chan string, 2)} - s := newOperationSink(t.Context(), f, 32) + s := newOperationSink(t.Context(), f) recordState(t, s, "resources.jobs.foo", "v1") assert.Equal(t, "resources.jobs.foo", <-f.started) @@ -211,10 +212,49 @@ func TestOperationSinkRecordDuringUploadIsStillUploaded(t *testing.T) { assert.Empty(t, s.pending) } +func TestOperationSinkRecordWaitsWhenTheQueueIsFull(t *testing.T) { + // Recording holds the deploy back rather than letting it run arbitrarily far ahead + // of what the service has been told: once every slot holds a resource, the next + // write waits for the uploader. + // started is buffered for every upload: nothing reads it after the first, and an + // uploader blocked sending to it would never drain the queue. + f := &fakeUploader{block: make(chan struct{}), started: make(chan string, operationSinkQueueSize+4)} + s := newOperationSink(t.Context(), f) + + // One key is taken off the queue and stuck in the uploader; the rest fill it. + recordState(t, s, "resources.jobs.busy", "v1") + assert.Equal(t, "resources.jobs.busy", <-f.started) + for i := range operationSinkQueueSize { + recordState(t, s, "resources.jobs.job"+strconv.Itoa(i), "v1") + } + + // The next distinct resource has nowhere to go until the uploader moves on. Called + // directly rather than through recordState: its assertions may only run on the + // test's own goroutine. + late := envelope(t, "v1") + blocked := make(chan struct{}) + go func() { + s.RecordOperation(t.Context(), "resources.jobs.late", dstate.OperationInfo{Action: deployplan.Update}, "id-1", late) + close(blocked) + }() + + select { + case <-blocked: + t.Fatal("recording did not wait for a full queue, so the deploy can outrun the service") + case <-time.After(50 * time.Millisecond): + } + + close(f.block) + <-blocked + require.NoError(t, s.close()) + + assert.Len(t, f.recorded(), operationSinkQueueSize+2) +} + func TestOperationSinkReturnsUploadError(t *testing.T) { uploadErr := errors.New("boom") f := &fakeUploader{err: uploadErr} - s := newOperationSink(t.Context(), f, 32) + s := newOperationSink(t.Context(), f) recordState(t, s, "resources.jobs.foo", "v1") @@ -228,7 +268,7 @@ func TestOperationSinkKeepsRecordingAfterUploadError(t *testing.T) { // One failed upload must not drop the records for everything behind it, so DMS // ends up as close to reality as it can get. f := &fakeUploader{err: errors.New("boom")} - s := newOperationSink(t.Context(), f, 32) + s := newOperationSink(t.Context(), f) recordState(t, s, "resources.jobs.foo", "v1") recordState(t, s, "resources.jobs.bar", "v1") @@ -239,7 +279,7 @@ func TestOperationSinkKeepsRecordingAfterUploadError(t *testing.T) { func TestOperationSinkFirstErrIsWhatStopsTheDeploy(t *testing.T) { f := &fakeUploader{err: errors.New("boom")} - s := newOperationSink(t.Context(), f, 32) + s := newOperationSink(t.Context(), f) assert.NoError(t, s.firstErr()) @@ -253,7 +293,7 @@ func TestOperationSinkFirstErrIsWhatStopsTheDeploy(t *testing.T) { func TestOperationSinkDropsUnsupportedAction(t *testing.T) { f := &fakeUploader{} - s := newOperationSink(t.Context(), f, 32) + s := newOperationSink(t.Context(), f) // Skip never reaches a sink; it is dropped with a warning rather than failing a // resource that deployed fine. @@ -265,7 +305,7 @@ func TestOperationSinkDropsUnsupportedAction(t *testing.T) { func TestOperationSinkDropsOversizedState(t *testing.T) { f := &fakeUploader{} - s := newOperationSink(t.Context(), f, 32) + s := newOperationSink(t.Context(), f) s.RecordOperation(t.Context(), "resources.jobs.foo", dstate.OperationInfo{Action: deployplan.Create}, "id-1", envelope(t, strings.Repeat("x", maxOperationStateSize))) require.NoError(t, s.close()) @@ -275,7 +315,7 @@ func TestOperationSinkDropsOversizedState(t *testing.T) { func TestOperationSinkCloseIsIdempotent(t *testing.T) { f := &fakeUploader{} - s := newOperationSink(t.Context(), f, 32) + s := newOperationSink(t.Context(), f) recordState(t, s, "resources.jobs.foo", "v1") @@ -294,5 +334,5 @@ func TestNilOperationSinkIsNoOp(t *testing.T) { func TestNewOperationSinkNilUploaderIsNil(t *testing.T) { // Recording off: the sink is nil so the state DB's nil check leaves it unset. - assert.Nil(t, newOperationSink(t.Context(), nil, 32)) + assert.Nil(t, newOperationSink(t.Context(), nil)) } From 4e331340b80fe53c34c01ad12f889ae1ac0fe87a Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 17 Aug 2026 10:17:05 +0000 Subject: [PATCH 088/125] bundle: fail the deploy when an operation cannot be recorded Building an operation can fail two ways: an action the service has no enum for, which is a programming error, and a serialized state over the 64KB the service accepts, which a large enough resource reaches. Both were logged as a warning and skipped, on the grounds that the resource itself had deployed fine. That reasoning does not hold. DMS owns the resource set the next plan reads (see dstate.readDMSState), so a resource left unrecorded is a resource the next deploy creates a second time - the same failure mode that makes an upload error fail the deploy. Skipping quietly turned a clear error into a duplicate resource later. Both now set the sink's first error, so they stop the deploy exactly as an upload failure does, and the message names the resource and the limit. Co-authored-by: Isaac --- bundle/direct/opsink.go | 10 +++++----- bundle/direct/opsink_test.go | 20 ++++++++++++++------ 2 files changed, 19 insertions(+), 11 deletions(-) diff --git a/bundle/direct/opsink.go b/bundle/direct/opsink.go index 5a9a257e10f..8d21c2048dc 100644 --- a/bundle/direct/opsink.go +++ b/bundle/direct/opsink.go @@ -8,7 +8,6 @@ import ( "github.com/databricks/cli/bundle/deployplan" "github.com/databricks/cli/bundle/direct/dstate" - "github.com/databricks/cli/libs/log" ) // operationSinkQueueSize is how many resources may have an unrecorded state write @@ -89,9 +88,10 @@ func (s *operationSink) RecordOperation(ctx context.Context, resourceKey string, op, err := newStateOperation(info, resourceID, state) if err != nil { - // The deploy already persisted this write locally, so failing it here would - // report an error about history for a resource that deployed fine. - log.Warnf(ctx, "Not recording operation for %s: %s", resourceKey, err) + // Fails the deploy, like an upload failure: the write is on disk locally, but + // DMS owns the resource set the next plan reads (see dstate.readDMSState), so + // leaving a resource unrecorded would have that plan create it a second time. + s.setErr(fmt.Errorf("recording operation for %s: %w", resourceKey, err)) return } @@ -108,7 +108,7 @@ func (s *operationSink) recordFailure(ctx context.Context, resourceKey string, a op, err := newFailedOperation(action, resourceID, priorState, cause) if err != nil { - log.Warnf(ctx, "Not recording failure for %s: %s", resourceKey, err) + s.setErr(fmt.Errorf("recording failure for %s: %w", resourceKey, err)) return } diff --git a/bundle/direct/opsink_test.go b/bundle/direct/opsink_test.go index a347e9e1f7f..28974b03f6d 100644 --- a/bundle/direct/opsink_test.go +++ b/bundle/direct/opsink_test.go @@ -291,25 +291,33 @@ func TestOperationSinkFirstErrIsWhatStopsTheDeploy(t *testing.T) { assert.Error(t, s.firstErr()) } -func TestOperationSinkDropsUnsupportedAction(t *testing.T) { +func TestOperationSinkFailsOnUnsupportedAction(t *testing.T) { f := &fakeUploader{} s := newOperationSink(t.Context(), f) - // Skip never reaches a sink; it is dropped with a warning rather than failing a - // resource that deployed fine. + // Skip never reaches a sink, so this is a programming error rather than anything a + // user did - but it still has to fail the deploy rather than pass silently, because + // the resource would be left out of the deployment. s.RecordOperation(t.Context(), "resources.jobs.foo", dstate.OperationInfo{Action: deployplan.Skip}, "id-1", nil) - require.NoError(t, s.close()) + err := s.close() + require.Error(t, err) + assert.ErrorContains(t, err, "resources.jobs.foo") assert.Empty(t, f.recorded()) } -func TestOperationSinkDropsOversizedState(t *testing.T) { +func TestOperationSinkFailsOnOversizedState(t *testing.T) { + // The service will not take a state this large, so the resource cannot be recorded. + // Failing here says so, where reporting nothing would leave DMS without the resource + // and the next plan would create it again. f := &fakeUploader{} s := newOperationSink(t.Context(), f) s.RecordOperation(t.Context(), "resources.jobs.foo", dstate.OperationInfo{Action: deployplan.Create}, "id-1", envelope(t, strings.Repeat("x", maxOperationStateSize))) - require.NoError(t, s.close()) + err := s.close() + require.Error(t, err) + assert.ErrorContains(t, err, "exceeds the 65536 byte limit") assert.Empty(t, f.recorded()) } From 4deaf422748025b8efaed32104d72eed1c3b5860 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 17 Aug 2026 11:24:20 +0000 Subject: [PATCH 089/125] bundle: trim comments that restate the code The setErr call needs no note explaining that it fails the deploy, and recordFailure needs none explaining that it returns nothing. Narrow the two error helpers' docs while here: they carry a failure to build an operation now, not only a failed upload. Co-authored-by: Isaac --- bundle/direct/opsink.go | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/bundle/direct/opsink.go b/bundle/direct/opsink.go index 8d21c2048dc..1217e22512c 100644 --- a/bundle/direct/opsink.go +++ b/bundle/direct/opsink.go @@ -88,9 +88,6 @@ func (s *operationSink) RecordOperation(ctx context.Context, resourceKey string, op, err := newStateOperation(info, resourceID, state) if err != nil { - // Fails the deploy, like an upload failure: the write is on disk locally, but - // DMS owns the resource set the next plan reads (see dstate.readDMSState), so - // leaving a resource unrecorded would have that plan create it a second time. s.setErr(fmt.Errorf("recording operation for %s: %w", resourceKey, err)) return } @@ -99,8 +96,7 @@ func (s *operationSink) RecordOperation(ctx context.Context, resourceKey string, } // recordFailure records that applying a resource failed, so the deployment history -// explains the failure instead of omitting the resource. It returns nothing: the -// deploy is already failing, and a second error would mask the one the user needs. +// explains the failure instead of omitting the resource. func (s *operationSink) recordFailure(ctx context.Context, resourceKey string, action deployplan.ActionType, resourceID string, priorState json.RawMessage, cause error) { if s == nil { return @@ -195,7 +191,7 @@ func (s *operationSink) close() error { return s.firstErr() } -// setErr keeps the first upload error; later ones are dropped because one failure +// setErr keeps the first recording error; later ones are dropped because one failure // is enough to fail the deploy. func (s *operationSink) setErr(err error) { s.mu.Lock() @@ -206,8 +202,8 @@ func (s *operationSink) setErr(err error) { } } -// firstErr returns the first upload error, or nil if every upload so far succeeded. -// A nil sink (recording disabled) never errors. +// firstErr returns the first recording error, or nil if everything so far was +// recorded. A nil sink (recording disabled) never errors. func (s *operationSink) firstErr() error { if s == nil { return nil From fc5465fba80704b4386e299280bafe56239b28d7 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 17 Aug 2026 11:47:33 +0000 Subject: [PATCH 090/125] bundle: a coalesced failure must not revert a resource to its pre-deploy state A failure carries the pre-deploy record, which for an update is a state the write it supersedes has already moved past. coalesce only kept the superseded state when the failure had none of its own, so an update that succeeded and then failed waiting recorded the resource as it was before the deploy - and DMS is what the next plan reads back as current. Which state won depended on timing, too: once the write was uploaded the wire mask kept it, because a failure updates only status and error_message there. A failure now keeps the superseded state whenever there is one, so both paths follow the same rule. Co-authored-by: Isaac --- bundle/direct/opsink.go | 14 +++++++------- bundle/direct/opsink_test.go | 28 ++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 7 deletions(-) diff --git a/bundle/direct/opsink.go b/bundle/direct/opsink.go index 1217e22512c..20ddf1e5ca1 100644 --- a/bundle/direct/opsink.go +++ b/bundle/direct/opsink.go @@ -134,14 +134,14 @@ func (s *operationSink) record(resourceKey string, op recordedOperation) { // coalesce folds a write that never got uploaded into the one replacing it. The newer // write describes the resource as it now stands, so its fields win. // -// A failure is the exception: it carries the resource's state only when there was a -// pre-deploy record to carry (see newFailedOperation), so a create that wrote state -// and then failed would replace that state with nothing and drop the resource from the -// deployment. Inherit what the superseded write recorded instead, since that is the -// resource the failure is reporting on. The same reasoning applies on the wire when -// the write did get uploaded; see failureFields. +// A failure is the exception: it says why a resource stopped, not what it looks like, +// so it keeps the state of the write it supersedes. Its own state comes from the +// pre-deploy record (see newFailedOperation), which is either nothing - dropping the +// resource from the deployment - or a state the write it replaces has already moved +// past. This is the same rule the wire applies once the write is uploaded, where a +// failure updates only status and error_message; see failureFields. func coalesce(older, newer recordedOperation) recordedOperation { - if newer.isFailure() && newer.state == nil && older.state != nil { + if newer.isFailure() && older.state != nil { newer.state = older.state newer.resourceID = older.resourceID } diff --git a/bundle/direct/opsink_test.go b/bundle/direct/opsink_test.go index 28974b03f6d..631bf2af6fd 100644 --- a/bundle/direct/opsink_test.go +++ b/bundle/direct/opsink_test.go @@ -166,6 +166,34 @@ func TestOperationSinkCoalescedFailureKeepsTheStateItReplaces(t *testing.T) { assert.Equal(t, "run did not succeed: FAILED", f.errorMessageFor("resources.job_runs.my_run")) } +func TestOperationSinkCoalescedFailureDoesNotRevertToPriorState(t *testing.T) { + // An update that succeeded and then failed waiting carries the pre-deploy state, + // which the write it supersedes has already moved past. Sending that would record + // the resource as it was before the deploy, and the next plan would read it back as + // current. Once the write is uploaded the wire mask keeps it out; before that, this + // does. + f := &fakeUploader{block: make(chan struct{}), started: make(chan string, 2)} + s := newOperationSink(t.Context(), f) + + recordState(t, s, "resources.jobs.busy", "v1") + assert.Equal(t, "resources.jobs.busy", <-f.started) + + s.RecordOperation(t.Context(), "resources.jobs.foo", dstate.OperationInfo{Action: deployplan.Update}, "id-new", envelope(t, "after the update")) + s.recordFailure(t.Context(), "resources.jobs.foo", deployplan.Update, "id-old", envelope(t, "before the deploy"), errors.New("waiting after updating: timed out")) + + close(f.block) + require.NoError(t, s.close()) + + assert.Equal(t, []string{ + `resources.jobs.busy={"state":{"name":"v1"}}`, + `resources.jobs.foo={"state":{"name":"after the update"}}`, + }, f.recorded()) + assert.Equal(t, "id-new", f.resourceIDFor("resources.jobs.foo")) + assert.Equal(t, + bundledeployments.OperationStatusOperationStatusFailed, + f.statusFor("resources.jobs.foo")) +} + func TestOperationSinkCoalescedDeleteStillClearsState(t *testing.T) { // A delete legitimately carries no state, and coalescing must let it through: the // resource is gone, and keeping the state it replaces would leave it listed. From 162f619508809e8eb2120920213a4abbbc99917b Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 17 Aug 2026 12:10:21 +0000 Subject: [PATCH 091/125] bundle: build the update mask from what the service already holds The mask was two fixed lists picked by whether the operation was a failure. That dropped state from every failure update, including the one case where the service has none to keep: a recreate records an in-progress delete with no state, and if the create that follows fails before its write is uploaded, the failure is carrying that write's state - which the mask then threw away, leaving the resource out of the deployment. The recorder now remembers whether what it uploaded for a resource describes it, and masks state out only then. A failure with nothing recorded yet supplies the state it carries instead. Keying on the operation alone does not work: a second deploy of a failed resource has a pre-deploy record, so "carries state" is true there while the state is older than the write it would overwrite. Only the service's side of it distinguishes the two. Co-authored-by: Isaac --- bundle/direct/oprecorder.go | 91 ++++++++++++++++++-------------- bundle/direct/oprecorder_test.go | 25 ++++++++- bundle/direct/opsink.go | 21 +++++--- 3 files changed, 88 insertions(+), 49 deletions(-) diff --git a/bundle/direct/oprecorder.go b/bundle/direct/oprecorder.go index a718d4e85f8..bf84ef1dade 100644 --- a/bundle/direct/oprecorder.go +++ b/bundle/direct/oprecorder.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "fmt" + "slices" "strings" "sync" @@ -147,15 +148,14 @@ type operationRecorder struct { // "deployments/{deployment_id}/versions/{version_id}". parent string - // mu guards sequenceIDs. + // mu guards recorded. mu sync.Mutex - // sequenceIDs holds the last sequence_id the service returned per resource key, - // which is how a resource already recorded in this version is recognised. The - // service names operations "operations/{resource_key}", so it keeps one per - // resource per version: the second write for a resource has to update that - // operation, and echo this value as the concurrency precondition. - sequenceIDs map[string]string + // recorded holds what the service has per resource key, which is how a resource + // already recorded in this version is recognised. The service names operations + // "operations/{resource_key}", so it keeps one per resource per version: the second + // write for a resource has to update that operation. + recorded map[string]recordedState } // NewOperationRecorder returns an operationUploader backed by the DMS operations @@ -169,23 +169,40 @@ func NewOperationRecorder(apiClient *client.DatabricksClient, deploymentID strin // operationClient. func newOperationRecorder(ops operationClient, deploymentID string, version int64) operationUploader { return &operationRecorder{ - ops: ops, - parent: fmt.Sprintf("deployments/%s/versions/%d", deploymentID, version), - sequenceIDs: make(map[string]string), + ops: ops, + parent: fmt.Sprintf("deployments/%s/versions/%d", deploymentID, version), + recorded: make(map[string]recordedState), } } -// updatableFields are the operation fields a later write for the same resource can -// change. resource_id is included because a recreate learns a new one. -var updatableFields = []string{"state", "error_message", "resource_id", "status"} +// recordedState is what the service holds for a resource in this version. +type recordedState struct { + // sequenceID is echoed as the concurrency precondition on the next update. + sequenceID string -// failureFields are the fields a failure changes on an operation that already exists. -// It deliberately leaves state and resource_id alone: the resource was written before -// the step that failed, so what is already recorded describes something that exists, -// and a failure carries no state of its own to replace it with. Including them would -// clear both - the service takes the update mask literally - and a resource with no -// state is dropped from the deployment, so the next plan would try to create it again. -var failureFields = []string{"error_message", "status"} + // hasState says whether the recorded operation describes the resource. A failure + // must not disturb that description, and must supply one when it is missing. + hasState bool +} + +// updateMask lists the fields an update changes. The service takes it literally: a +// field named in the mask is written, a field left out keeps the value it had. +// +// A failure is the only operation that leaves anything out, and only when the service +// already describes the resource. It says why the resource stopped, not how it looks, +// and the state it carries is from before the deploy - older than whatever the write +// recorded. Naming state would replace the newer description with that, or clear it +// outright, and a resource with no state is dropped from the deployment, which has the +// next plan create it again. +// +// When the service has no state - a recreate whose in-progress delete recorded none - +// the failure is the only thing that can supply one, so it does. +func updateMask(op recordedOperation, has recordedState) []string { + if op.isFailure() && has.hasState { + return []string{"error_message", "status"} + } + return []string{"state", "error_message", "resource_id", "status"} +} func (r *operationRecorder) upload(ctx context.Context, resourceKey string, op recordedOperation) error { // The read path re-adds the prefix; see dstate.ResourceKeyPrefix. @@ -212,34 +229,22 @@ func (r *operationRecorder) upload(ctx context.Context, resourceKey string, op r } r.mu.Lock() - sequenceID, recorded := r.sequenceIDs[dmsKey] + has, recorded := r.recorded[dmsKey] r.mu.Unlock() var result operationResponse var err error + mask := updateMask(op, has) if recorded { // Only the masked fields and sequence_id are read on an update; action_type // stays as the operation was created, so sending it would just be misleading. - body := updateOperationRequest{ + result, err = r.ops.UpdateOperation(ctx, r.parent, dmsKey, mask, updateOperationRequest{ State: operation.State, ErrorMessage: operation.ErrorMessage, ResourceId: operation.ResourceId, Status: operation.Status, - SequenceId: sequenceID, - } - fields := updatableFields - if op.isFailure() { - // Mark the existing record failed and leave the rest of it alone; see - // failureFields. A failure that arrives before any operation exists still - // goes through CreateOperation below, carrying the prior state. - fields = failureFields - body = updateOperationRequest{ - ErrorMessage: operation.ErrorMessage, - Status: operation.Status, - SequenceId: sequenceID, - } - } - result, err = r.ops.UpdateOperation(ctx, r.parent, dmsKey, fields, body) + SequenceId: has.sequenceID, + }) } else { result, err = r.ops.CreateOperation(ctx, r.parent, dmsKey, operation) } @@ -247,10 +252,16 @@ func (r *operationRecorder) upload(ctx context.Context, resourceKey string, op r return err } - // Remember the sequence the service assigned, so the next write for this - // resource updates rather than re-creates. + // Remember what the service now holds, so the next write for this resource updates + // rather than re-creates, and a failure can tell whether the resource is already + // described. A mask that left state out leaves whatever was there. + hasState := op.state != nil + if recorded && !slices.Contains(mask, "state") { + hasState = has.hasState + } + r.mu.Lock() - r.sequenceIDs[dmsKey] = result.SequenceId + r.recorded[dmsKey] = recordedState{sequenceID: result.SequenceId, hasState: hasState} r.mu.Unlock() return nil diff --git a/bundle/direct/oprecorder_test.go b/bundle/direct/oprecorder_test.go index 2f8b12c3705..7f69e62ec9f 100644 --- a/bundle/direct/oprecorder_test.go +++ b/bundle/direct/oprecorder_test.go @@ -112,7 +112,7 @@ func TestOperationRecorderFailureAfterAStateWriteKeepsTheState(t *testing.T) { update := f.calls[1] assert.Equal(t, "update", update.method) - assert.Equal(t, failureFields, update.fields) + assert.Equal(t, []string{"error_message", "status"}, update.fields) assert.Equal(t, bundledeployments.OperationStatusOperationStatusFailed, update.update.Status) assert.Equal(t, "run did not succeed: FAILED", update.update.ErrorMessage) // Neither is in the mask, so what the create recorded stands. @@ -120,6 +120,29 @@ func TestOperationRecorderFailureAfterAStateWriteKeepsTheState(t *testing.T) { assert.Empty(t, update.update.ResourceId) } +func TestOperationRecorderFailureCarryingStateSendsIt(t *testing.T) { + // A recreate records its in-progress delete, which has no state, and then the create + // that replaces the resource. If the create's write is still waiting when the create + // fails, the failure carries that write's state (see coalesce) - and the update has + // to name state in the mask, or the service keeps the nothing the delete recorded and + // drops the resource from the deployment. + f := &fakeOpClient{sequence: "2"} + r := newOperationRecorder(f, "dep-1", 2) + + uploadOne(t, r, "resources.jobs.foo", deployplan.Recreate, "old-id", nil) + + failed, err := newFailedOperation(deployplan.Recreate, "new-id", envelope(t, "the replacement"), errors.New("boom")) + require.NoError(t, err) + require.NoError(t, r.upload(t.Context(), "resources.jobs.foo", failed)) + + require.Len(t, f.calls, 2) + update := f.calls[1] + assert.Equal(t, "update", update.method) + assert.Equal(t, []string{"state", "error_message", "resource_id", "status"}, update.fields) + require.NotNil(t, update.update.State) + assert.Equal(t, "new-id", update.update.ResourceId) +} + func TestOperationRecorderFailureBeforeAnyWriteCarriesPriorState(t *testing.T) { // Nothing has been recorded for the resource, so the failure creates the operation // and has to carry the prior state itself - the resource still exists, and the diff --git a/bundle/direct/opsink.go b/bundle/direct/opsink.go index 20ddf1e5ca1..c67c2a0474e 100644 --- a/bundle/direct/opsink.go +++ b/bundle/direct/opsink.go @@ -131,15 +131,20 @@ func (s *operationSink) record(resourceKey string, op recordedOperation) { } } -// coalesce folds a write that never got uploaded into the one replacing it. The newer -// write describes the resource as it now stands, so its fields win. +// coalesce replaces a waiting operation with the one that supersedes it. The newer one +// says how the resource looks now, so its fields win. // -// A failure is the exception: it says why a resource stopped, not what it looks like, -// so it keeps the state of the write it supersedes. Its own state comes from the -// pre-deploy record (see newFailedOperation), which is either nothing - dropping the -// resource from the deployment - or a state the write it replaces has already moved -// past. This is the same rule the wire applies once the write is uploaded, where a -// failure updates only status and error_message; see failureFields. +// A failure is different, because it says why the resource stopped rather than how it +// looks, and the state it carries is from before the deploy. Which state should reach +// the service depends on whether the write it replaces got uploaded: +// +// - Uploaded: the service already has the right state. The failure sends none, and +// updateMask leaves it out so it stays. +// - Not uploaded: nobody has told the service anything yet, so this failure is the +// only chance to. It takes the state of the write it replaces, and updateMask +// includes it. +// +// Either way the pre-deploy state is not what goes: it is older than the write. func coalesce(older, newer recordedOperation) recordedOperation { if newer.isFailure() && older.state != nil { newer.state = older.state From d3fca3b1ac5efee6624d4196cdc8e33ef6c45c06 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 17 Aug 2026 12:34:18 +0000 Subject: [PATCH 092/125] bundle: let each operation carry its own update mask The mask was derived at upload time from isFailure() crossed with whether the service already held state. Each operation now states its mask when it is built - a write describes the resource, a failure only says it stopped - and coalesce carries the mask along with the state it adopts, so a failure that takes over a pending write's state also says it is writing state. That closes a hole the derived form had: a failure coalescing a second state write for a resource whose first write was already uploaded would have dropped the newer state, leaving DMS with the older one. No action reaches that today, since the only two-write sequence is a recreate whose delete step records no state, but the derived condition could not express it at all. Also validate the prior state's size in newFailedOperation, and cover that the uploader keeps running while the queue is momentarily empty. Co-authored-by: Isaac --- bundle/direct/oprecorder.go | 80 +++++++++++++++++++------------- bundle/direct/oprecorder_test.go | 32 +++++++++++++ bundle/direct/opsink.go | 3 ++ bundle/direct/opsink_test.go | 17 +++++++ 4 files changed, 101 insertions(+), 31 deletions(-) diff --git a/bundle/direct/oprecorder.go b/bundle/direct/oprecorder.go index bf84ef1dade..cfa499c4031 100644 --- a/bundle/direct/oprecorder.go +++ b/bundle/direct/oprecorder.go @@ -51,13 +51,35 @@ type recordedOperation struct { // delete, where the resource no longer exists, and for a failure, where the // resource was not written. state json.RawMessage + + // updateFields is the update mask to send if this operation updates one the service + // already has. The service takes it literally: a field named here is written, a field + // left out keeps the value it had. + updateFields []string } +// describesResource is the update mask for an operation that says how the resource +// looks: everything an update is allowed to change. +var describesResource = []string{"state", "error_message", "resource_id", "status"} + +// failedKeepingState is the update mask for a failure. A failure says why the +// resource stopped rather than how it looks, and the state it reports is the pre-deploy +// one - older than whatever a write already recorded - so it leaves state alone. +var failedKeepingState = []string{"error_message", "status"} + // isFailure reports whether the operation records a resource that did not apply. func (op recordedOperation) isFailure() bool { return op.status == bundledeployments.OperationStatusOperationStatusFailed } +// checkStateSize rejects a state the service will not accept. +func checkStateSize(state json.RawMessage) error { + if len(state) > maxOperationStateSize { + return fmt.Errorf("serialized state is %d bytes, which exceeds the %d byte limit for recording deployment history", len(state), maxOperationStateSize) + } + return nil +} + // newStateOperation describes a state write for upload. state is the serialized // RecordedState envelope the state DB just persisted, and nil for a delete, where // the resource is gone. It errors when the state exceeds maxOperationStateSize. @@ -67,8 +89,8 @@ func newStateOperation(info dstate.OperationInfo, resourceID string, state json. return recordedOperation{}, err } - if len(state) > maxOperationStateSize { - return recordedOperation{}, fmt.Errorf("serialized state is %d bytes, which exceeds the %d byte limit for recording deployment history", len(state), maxOperationStateSize) + if err := checkStateSize(state); err != nil { + return recordedOperation{}, err } status := bundledeployments.OperationStatusOperationStatusSucceeded @@ -77,10 +99,11 @@ func newStateOperation(info dstate.OperationInfo, resourceID string, state json. } return recordedOperation{ - action: actionType, - resourceID: resourceID, - status: status, - state: state, + action: actionType, + resourceID: resourceID, + status: status, + state: state, + updateFields: describesResource, }, nil } @@ -99,6 +122,12 @@ func newFailedOperation(action deployplan.ActionType, resourceID string, priorSt return recordedOperation{}, err } + // A guard: this state came back from the state DB, so it was within the limit when + // it was written. + if err := checkStateSize(priorState); err != nil { + return recordedOperation{}, err + } + message := cause.Error() if len(message) > maxOperationErrorMessageSize { message = message[:maxOperationErrorMessageSize] @@ -110,6 +139,7 @@ func newFailedOperation(action deployplan.ActionType, resourceID string, priorSt status: bundledeployments.OperationStatusOperationStatusFailed, errorMessage: message, state: priorState, + updateFields: failedKeepingState, }, nil } @@ -135,8 +165,8 @@ func priorRecord(db *dstate.DeploymentState, resourceKey string) (string, json.R return entry.ID, raw } -// operationUploader records an applied resource operation with DMS. Uploads run -// on the operationQueue workers, off the apply path. +// operationUploader records an applied resource operation with DMS. Uploads run on +// the operationSink goroutine, off the apply path. type operationUploader interface { upload(ctx context.Context, resourceKey string, op recordedOperation) error } @@ -185,25 +215,6 @@ type recordedState struct { hasState bool } -// updateMask lists the fields an update changes. The service takes it literally: a -// field named in the mask is written, a field left out keeps the value it had. -// -// A failure is the only operation that leaves anything out, and only when the service -// already describes the resource. It says why the resource stopped, not how it looks, -// and the state it carries is from before the deploy - older than whatever the write -// recorded. Naming state would replace the newer description with that, or clear it -// outright, and a resource with no state is dropped from the deployment, which has the -// next plan create it again. -// -// When the service has no state - a recreate whose in-progress delete recorded none - -// the failure is the only thing that can supply one, so it does. -func updateMask(op recordedOperation, has recordedState) []string { - if op.isFailure() && has.hasState { - return []string{"error_message", "status"} - } - return []string{"state", "error_message", "resource_id", "status"} -} - func (r *operationRecorder) upload(ctx context.Context, resourceKey string, op recordedOperation) error { // The read path re-adds the prefix; see dstate.ResourceKeyPrefix. dmsKey := strings.TrimPrefix(resourceKey, dstate.ResourceKeyPrefix) @@ -232,9 +243,16 @@ func (r *operationRecorder) upload(ctx context.Context, resourceKey string, op r has, recorded := r.recorded[dmsKey] r.mu.Unlock() + mask := op.updateFields + if !has.hasState { + // Nothing recorded describes the resource yet, so this operation has to, even a + // failure that would rather leave state alone: a resource recorded without state + // is dropped from the deployment, and the next plan creates it again. + mask = describesResource + } + var result operationResponse var err error - mask := updateMask(op, has) if recorded { // Only the masked fields and sequence_id are read on an update; action_type // stays as the operation was created, so sending it would just be misleading. @@ -255,9 +273,9 @@ func (r *operationRecorder) upload(ctx context.Context, resourceKey string, op r // Remember what the service now holds, so the next write for this resource updates // rather than re-creates, and a failure can tell whether the resource is already // described. A mask that left state out leaves whatever was there. - hasState := op.state != nil - if recorded && !slices.Contains(mask, "state") { - hasState = has.hasState + hasState := has.hasState + if slices.Contains(mask, "state") { + hasState = op.state != nil } r.mu.Lock() diff --git a/bundle/direct/oprecorder_test.go b/bundle/direct/oprecorder_test.go index 7f69e62ec9f..9fa74c7b2bd 100644 --- a/bundle/direct/oprecorder_test.go +++ b/bundle/direct/oprecorder_test.go @@ -143,6 +143,31 @@ func TestOperationRecorderFailureCarryingStateSendsIt(t *testing.T) { assert.Equal(t, "new-id", update.update.ResourceId) } +func TestOperationRecorderFailureCarryingALaterWriteSendsIt(t *testing.T) { + // Two writes for one resource: the first is uploaded, the second is still waiting + // when the resource fails, so the failure carries it (see coalesce). That state is + // newer than what the service holds, so the update has to name state - leaving it out + // would keep the first write's state and the next plan would read back something this + // deploy has already moved past. + f := &fakeOpClient{sequence: "4"} + r := newOperationRecorder(f, "dep-1", 2) + + uploadOne(t, r, "resources.jobs.foo", deployplan.Update, "id-1", envelope(t, "first write")) + + second, err := newStateOperation(dstate.OperationInfo{Action: deployplan.Update}, "id-1", envelope(t, "second write")) + require.NoError(t, err) + failed, err := newFailedOperation(deployplan.Update, "id-old", envelope(t, "before the deploy"), errors.New("boom")) + require.NoError(t, err) + require.NoError(t, r.upload(t.Context(), "resources.jobs.foo", coalesce(second, failed))) + + require.Len(t, f.calls, 2) + update := f.calls[1] + assert.Equal(t, []string{"state", "error_message", "resource_id", "status"}, update.fields) + require.NotNil(t, update.update.State) + assert.Contains(t, string(*update.update.State), "second write") + assert.Equal(t, "id-1", update.update.ResourceId) +} + func TestOperationRecorderFailureBeforeAnyWriteCarriesPriorState(t *testing.T) { // Nothing has been recorded for the resource, so the failure creates the operation // and has to carry the prior state itself - the resource still exists, and the @@ -219,6 +244,13 @@ func TestNewFailedOperationRecordsPriorStateWithID(t *testing.T) { assert.JSONEq(t, `{"state":{"catalog_name":"main"}}`, string(op.state)) } +func TestNewFailedOperationRejectsOversizedPriorState(t *testing.T) { + big := json.RawMessage(strings.Repeat("x", maxOperationStateSize+1)) + + _, err := newFailedOperation(deployplan.Update, "job-123", big, errors.New("boom")) + assert.ErrorContains(t, err, "exceeds the 65536 byte limit") +} + func TestNewFailedOperationTruncatesLongError(t *testing.T) { // Truncated rather than rejected: a message over the limit would make recording // fail and hide the error it is reporting. diff --git a/bundle/direct/opsink.go b/bundle/direct/opsink.go index c67c2a0474e..265d926bff8 100644 --- a/bundle/direct/opsink.go +++ b/bundle/direct/opsink.go @@ -149,6 +149,9 @@ func coalesce(older, newer recordedOperation) recordedOperation { if newer.isFailure() && older.state != nil { newer.state = older.state newer.resourceID = older.resourceID + // The mask comes with the state: the failure is now reporting how the resource + // looks, so it has to say so rather than leave state alone. + newer.updateFields = older.updateFields } return newer } diff --git a/bundle/direct/opsink_test.go b/bundle/direct/opsink_test.go index 631bf2af6fd..960213c8e7a 100644 --- a/bundle/direct/opsink_test.go +++ b/bundle/direct/opsink_test.go @@ -112,6 +112,23 @@ func TestOperationSinkUploadsEachOperation(t *testing.T) { assert.Len(t, f.recorded(), 20) } +func TestOperationSinkKeepsUploadingAfterGoingIdle(t *testing.T) { + // The uploader parks on an empty queue instead of returning. Apply spends most of a + // deploy inside resource CRUD, so the queue is empty far more often than not, and an + // uploader that exited while idle would silently drop everything recorded after it. + f := &fakeUploader{} + s := newOperationSink(t.Context(), f) + + recordState(t, s, "resources.jobs.foo", "v1") + require.Eventually(t, func() bool { return len(f.recorded()) == 1 }, 5*time.Second, time.Millisecond) + + // The queue is drained and the uploader idle; what is recorded now still has to go. + recordState(t, s, "resources.jobs.bar", "v1") + require.NoError(t, s.close()) + + assert.Len(t, f.recorded(), 2) +} + func TestOperationSinkCoalescesWritesBehindAnUpload(t *testing.T) { // Hold the uploader on the first write so the two behind it pile up. They carry // the resource's full state, so only the newest needs to go: the resource costs From 9f65159a5b4e5ac6447a56918598d321ff6dbaac Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 17 Aug 2026 13:10:04 +0000 Subject: [PATCH 093/125] bundle: record a failed recreate as the resource being gone The update mask was derived at upload time from isFailure() crossed with whether the service already held state, and widened to write state whenever it held none. That widening was wrong: the one thing that reaches it is a recreate whose delete went through and whose create did not, so the resource really is gone, and filling the gap with the pre-deploy state described a resource that no longer exists. The next plan would then update it and get a 404 instead of creating it. Each operation now carries the mask it wants, set where its meaning is decided - a write describes the resource, a failure only marks it failed - so nothing has to be derived and the recorder no longer tracks whether state was recorded. A failure still supplies the pre-deploy state when it is the first thing recorded for the resource in this version, because nothing touched the resource and dropping it would have the next plan create a second one. coalesce takes a superseded write's state whether it is present or absent, along with the mask that writes it. Absent is what a recreate's delete step leaves, so DMS ends up the same whether that delete was uploaded or still waiting; before, the two orderings recorded different things. Update requests now send only the fields the mask names, and validate the prior state's size. Adds an acceptance test covering the failed recreate end to end. Co-authored-by: Isaac --- .../bundle/dms/failed-recreate/databricks.yml | 12 ++ .../bundle/dms/failed-recreate/out.test.toml | 3 + .../bundle/dms/failed-recreate/output.txt | 89 ++++++++++++++ acceptance/bundle/dms/failed-recreate/script | 25 ++++ bundle/direct/oprecorder.go | 111 +++++++----------- bundle/direct/oprecorder_test.go | 37 +++--- bundle/direct/opsink.go | 20 +--- bundle/direct/opsink_test.go | 26 ++++ 8 files changed, 224 insertions(+), 99 deletions(-) create mode 100644 acceptance/bundle/dms/failed-recreate/databricks.yml create mode 100644 acceptance/bundle/dms/failed-recreate/out.test.toml create mode 100644 acceptance/bundle/dms/failed-recreate/output.txt create mode 100644 acceptance/bundle/dms/failed-recreate/script diff --git a/acceptance/bundle/dms/failed-recreate/databricks.yml b/acceptance/bundle/dms/failed-recreate/databricks.yml new file mode 100644 index 00000000000..ee735a2fd2c --- /dev/null +++ b/acceptance/bundle/dms/failed-recreate/databricks.yml @@ -0,0 +1,12 @@ +bundle: + name: dms-failed-recreate + +experimental: + record_deployment_history: true + +resources: + schemas: + foo: + name: dms_failed_recreate_schema + catalog_name: main + comment: v1 diff --git a/acceptance/bundle/dms/failed-recreate/out.test.toml b/acceptance/bundle/dms/failed-recreate/out.test.toml new file mode 100644 index 00000000000..7daaf6fd56a --- /dev/null +++ b/acceptance/bundle/dms/failed-recreate/out.test.toml @@ -0,0 +1,3 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/dms/failed-recreate/output.txt b/acceptance/bundle/dms/failed-recreate/output.txt new file mode 100644 index 00000000000..c573aa16e85 --- /dev/null +++ b/acceptance/bundle/dms/failed-recreate/output.txt @@ -0,0 +1,89 @@ + +=== Deploy the schema, so there is a resource to recreate +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-failed-recreate/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +=== A recreate deletes and then fails to create. The delete went through, so the resource really is gone: the operation records no state, rather than describing the resource as it was before the deploy +>>> update_file.py databricks.yml catalog_name: main catalog_name: other + +>>> fault.py POST /api/2.1/unity-catalog/schemas 400 0 1 INVALID_PARAMETER_VALUE + +>>> musterr [CLI] bundle deploy --auto-approve +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-failed-recreate/default/files... + +This action will result in the deletion or recreation of the following UC schemas. Any underlying data may be lost: + recreate resources.schemas.foo +Deploying resources... +Error: cannot recreate resources.schemas.foo: Fault injected by test. (400 INVALID_PARAMETER_VALUE) + +Endpoint: POST [DATABRICKS_URL]/api/2.1/unity-catalog/schemas +HTTP Status: 400 Bad Request +API error_code: INVALID_PARAMETER_VALUE +API message: Fault injected by test. + +Updating deployment state... + +>>> print_requests.py //api/2.0/bundle --sort +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations/schemas.foo", + "q": { + "update_mask": "error_message,status" + }, + "body": { + "error_message": "Fault injected by test.", + "status": "OPERATION_STATUS_FAILED", + "sequence_id": "1" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions", + "q": { + "version_id": "2" + }, + "body": { + "cli_version": "[CLI_VERSION]", + "version_type": "VERSION_TYPE_DEPLOY", + "target_name": "default", + "display_name": "dms-failed-recreate", + "previous_version_id": "1", + "workspace_info": { + "file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-failed-recreate/default/files", + "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-failed-recreate/default" + } + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/complete", + "body": { + "completion_reason": "VERSION_COMPLETE_FAILURE" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations", + "q": { + "resource_key": "schemas.foo" + }, + "body": { + "action_type": "OPERATION_ACTION_TYPE_RECREATE", + "resource_id": "main.dms_failed_recreate_schema", + "resource_key": "schemas.foo", + "status": "OPERATION_STATUS_IN_PROGRESS" + } +} + +=== The resource is not listed: state is what projects a resource, and the failed recreate left none +>>> MSYS_NO_PATHCONV=1 [CLI] api get /api/2.0/bundle/deployments/[NUMID]/resources +{} + +=== Planning from DMS state alone creates the schema, which is what has to happen: it no longer exists +>>> [CLI] bundle plan +create schemas.foo + +Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged diff --git a/acceptance/bundle/dms/failed-recreate/script b/acceptance/bundle/dms/failed-recreate/script new file mode 100644 index 00000000000..969990a931c --- /dev/null +++ b/acceptance/bundle/dms/failed-recreate/script @@ -0,0 +1,25 @@ +title "Deploy the schema, so there is a resource to recreate" +trace $CLI bundle deploy +rm -f out.requests.txt + +title "A recreate deletes and then fails to create. The delete went through, so the resource really is gone: the operation records no state, rather than describing the resource as it was before the deploy" +trace update_file.py databricks.yml "catalog_name: main" "catalog_name: other" +# Fail only the create that follows the delete; the delete itself still goes through. +trace fault.py "POST /api/2.1/unity-catalog/schemas" 400 0 1 INVALID_PARAMETER_VALUE +trace musterr $CLI bundle deploy --auto-approve +trace print_requests.py //api/2.0/bundle --sort + +title "The resource is not listed: state is what projects a resource, and the failed recreate left none" +# The deployment ID is the workspace node's ID; read it back the way the CLI does. +# Extracted with python, not jq: the ID exceeds 2^53 and jq 1.6 rounds it. +deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-failed-recreate/default/state/resources.deployment.json" -o json | python3 -c 'import sys,json; print(json.load(sys.stdin)["object_id"])') +# MSYS_NO_PATHCONV as above: Git Bash on Windows would rewrite the leading-'/' API +# path into C:/Program Files/Git/api/2.0/... before the CLI saw it. +trace MSYS_NO_PATHCONV=1 $CLI api get "/api/2.0/bundle/deployments/${deployment_id}/resources" + +title "Planning from DMS state alone creates the schema, which is what has to happen: it no longer exists" +rm -rf .databricks +trace $CLI bundle plan +# plan probes the state files concurrently, so the recorded order varies. Only the +# operations above are asserted; drop the rest rather than diff a racy order. +rm -f out.requests.txt diff --git a/bundle/direct/oprecorder.go b/bundle/direct/oprecorder.go index cfa499c4031..a89ed00eac3 100644 --- a/bundle/direct/oprecorder.go +++ b/bundle/direct/oprecorder.go @@ -47,9 +47,9 @@ type recordedOperation struct { // failed, which the service enforces. errorMessage string - // state is the serialized local config after the operation. It is nil for a - // delete, where the resource no longer exists, and for a failure, where the - // resource was not written. + // state is the serialized local config after the operation. It is nil for a delete, + // where the resource no longer exists, and for a failure it is the state from before + // the deploy - see newFailedOperation for when that reaches the service. state json.RawMessage // updateFields is the update mask to send if this operation updates one the service @@ -62,9 +62,10 @@ type recordedOperation struct { // looks: everything an update is allowed to change. var describesResource = []string{"state", "error_message", "resource_id", "status"} -// failedKeepingState is the update mask for a failure. A failure says why the -// resource stopped rather than how it looks, and the state it reports is the pre-deploy -// one - older than whatever a write already recorded - so it leaves state alone. +// failedKeepingState is the update mask for a failure updating an operation this version +// already recorded: mark it failed and leave state alone. That is right either way - state +// means the resource is as it was written, and no state means a delete went through and +// nothing replaced it, so the resource really is gone and the deployment should say so. var failedKeepingState = []string{"error_message", "status"} // isFailure reports whether the operation records a resource that did not apply. @@ -72,14 +73,6 @@ func (op recordedOperation) isFailure() bool { return op.status == bundledeployments.OperationStatusOperationStatusFailed } -// checkStateSize rejects a state the service will not accept. -func checkStateSize(state json.RawMessage) error { - if len(state) > maxOperationStateSize { - return fmt.Errorf("serialized state is %d bytes, which exceeds the %d byte limit for recording deployment history", len(state), maxOperationStateSize) - } - return nil -} - // newStateOperation describes a state write for upload. state is the serialized // RecordedState envelope the state DB just persisted, and nil for a delete, where // the resource is gone. It errors when the state exceeds maxOperationStateSize. @@ -89,8 +82,8 @@ func newStateOperation(info dstate.OperationInfo, resourceID string, state json. return recordedOperation{}, err } - if err := checkStateSize(state); err != nil { - return recordedOperation{}, err + if len(state) > maxOperationStateSize { + return recordedOperation{}, fmt.Errorf("serialized state is %d bytes, which exceeds the %d byte limit for recording deployment history", len(state), maxOperationStateSize) } status := bundledeployments.OperationStatusOperationStatusSucceeded @@ -110,12 +103,13 @@ func newStateOperation(info dstate.OperationInfo, resourceID string, state json. // newFailedOperation records an operation that did not apply, so the deployment // history says why a resource failed rather than just omitting it. // -// priorState is the resource's state from before the deploy, carried through -// unchanged: an action other than a create acts on a resource that still exists, and -// the service rejects such an operation without state because dropping it would -// leave DMS unable to describe a resource it still owns. It is nil for a create, -// where there is no prior state and no resource to describe - which is also why the -// resourceID may be empty for CREATE and RECREATE. +// priorState is the resource's state from before the deploy, and only reaches the service +// when this failure is the first thing recorded for the resource in this version: nothing +// touched the resource, so it is still there and the deployment has to keep describing it +// or the next plan creates a second one. Once this version has recorded an operation, that +// operation says where the resource stands and the failure leaves its state alone - see +// failedKeepingState. It is nil for a create, which has no prior state and no resource to +// describe, which is also why the resourceID may be empty for CREATE and RECREATE. func newFailedOperation(action deployplan.ActionType, resourceID string, priorState json.RawMessage, cause error) (recordedOperation, error) { actionType, err := deployActionToSDK(action) if err != nil { @@ -124,8 +118,8 @@ func newFailedOperation(action deployplan.ActionType, resourceID string, priorSt // A guard: this state came back from the state DB, so it was within the limit when // it was written. - if err := checkStateSize(priorState); err != nil { - return recordedOperation{}, err + if len(priorState) > maxOperationStateSize { + return recordedOperation{}, fmt.Errorf("serialized state is %d bytes, which exceeds the %d byte limit for recording deployment history", len(priorState), maxOperationStateSize) } message := cause.Error() @@ -145,8 +139,7 @@ func newFailedOperation(action deployplan.ActionType, resourceID string, priorSt // priorRecord returns the resource's id and state from before this deploy, in the // same envelope form the success path uploads, or empty values when the resource has -// no prior record (a create). A failed operation reports these unchanged: the resource -// is whatever it was before the attempt. +// no prior record (a create). // // Both come from the same pre-deploy entry because the service requires an id // alongside state: state describes a resource that exists, so it needs the id to say @@ -178,14 +171,15 @@ type operationRecorder struct { // "deployments/{deployment_id}/versions/{version_id}". parent string - // mu guards recorded. + // mu guards sequenceIDs. mu sync.Mutex - // recorded holds what the service has per resource key, which is how a resource - // already recorded in this version is recognised. The service names operations + // sequenceIDs holds the sequence id the service returned per resource key, which is + // both how a resource already recorded in this version is recognised and the + // concurrency precondition for updating it. The service names operations // "operations/{resource_key}", so it keeps one per resource per version: the second // write for a resource has to update that operation. - recorded map[string]recordedState + sequenceIDs map[string]string } // NewOperationRecorder returns an operationUploader backed by the DMS operations @@ -199,22 +193,12 @@ func NewOperationRecorder(apiClient *client.DatabricksClient, deploymentID strin // operationClient. func newOperationRecorder(ops operationClient, deploymentID string, version int64) operationUploader { return &operationRecorder{ - ops: ops, - parent: fmt.Sprintf("deployments/%s/versions/%d", deploymentID, version), - recorded: make(map[string]recordedState), + ops: ops, + parent: fmt.Sprintf("deployments/%s/versions/%d", deploymentID, version), + sequenceIDs: make(map[string]string), } } -// recordedState is what the service holds for a resource in this version. -type recordedState struct { - // sequenceID is echoed as the concurrency precondition on the next update. - sequenceID string - - // hasState says whether the recorded operation describes the resource. A failure - // must not disturb that description, and must supply one when it is missing. - hasState bool -} - func (r *operationRecorder) upload(ctx context.Context, resourceKey string, op recordedOperation) error { // The read path re-adds the prefix; see dstate.ResourceKeyPrefix. dmsKey := strings.TrimPrefix(resourceKey, dstate.ResourceKeyPrefix) @@ -240,29 +224,27 @@ func (r *operationRecorder) upload(ctx context.Context, resourceKey string, op r } r.mu.Lock() - has, recorded := r.recorded[dmsKey] + sequenceID, recorded := r.sequenceIDs[dmsKey] r.mu.Unlock() - mask := op.updateFields - if !has.hasState { - // Nothing recorded describes the resource yet, so this operation has to, even a - // failure that would rather leave state alone: a resource recorded without state - // is dropped from the deployment, and the next plan creates it again. - mask = describesResource - } - var result operationResponse var err error if recorded { - // Only the masked fields and sequence_id are read on an update; action_type - // stays as the operation was created, so sending it would just be misleading. - result, err = r.ops.UpdateOperation(ctx, r.parent, dmsKey, mask, updateOperationRequest{ - State: operation.State, + update := updateOperationRequest{ ErrorMessage: operation.ErrorMessage, - ResourceId: operation.ResourceId, Status: operation.Status, - SequenceId: has.sequenceID, - }) + SequenceId: sequenceID, + } + // Send only what the mask names. The service would ignore the rest, and state is + // the largest field by far, so a failure that keeps the recorded state sends none. + if slices.Contains(op.updateFields, "state") { + update.State = operation.State + update.ResourceId = operation.ResourceId + } + + // action_type stays as the operation was created, so sending it would just be + // misleading. + result, err = r.ops.UpdateOperation(ctx, r.parent, dmsKey, op.updateFields, update) } else { result, err = r.ops.CreateOperation(ctx, r.parent, dmsKey, operation) } @@ -270,16 +252,9 @@ func (r *operationRecorder) upload(ctx context.Context, resourceKey string, op r return err } - // Remember what the service now holds, so the next write for this resource updates - // rather than re-creates, and a failure can tell whether the resource is already - // described. A mask that left state out leaves whatever was there. - hasState := has.hasState - if slices.Contains(mask, "state") { - hasState = op.state != nil - } - + // The next write for this resource updates this operation rather than re-creating it. r.mu.Lock() - r.recorded[dmsKey] = recordedState{sequenceID: result.SequenceId, hasState: hasState} + r.sequenceIDs[dmsKey] = result.SequenceId r.mu.Unlock() return nil diff --git a/bundle/direct/oprecorder_test.go b/bundle/direct/oprecorder_test.go index 9fa74c7b2bd..fb6eb7b898e 100644 --- a/bundle/direct/oprecorder_test.go +++ b/bundle/direct/oprecorder_test.go @@ -120,35 +120,35 @@ func TestOperationRecorderFailureAfterAStateWriteKeepsTheState(t *testing.T) { assert.Empty(t, update.update.ResourceId) } -func TestOperationRecorderFailureCarryingStateSendsIt(t *testing.T) { - // A recreate records its in-progress delete, which has no state, and then the create - // that replaces the resource. If the create's write is still waiting when the create - // fails, the failure carries that write's state (see coalesce) - and the update has - // to name state in the mask, or the service keeps the nothing the delete recorded and - // drops the resource from the deployment. +func TestOperationRecorderFailedRecreateKeepsTheResourceGone(t *testing.T) { + // A recreate records its in-progress delete, which has no state, and then fails before + // the create writes any. The failure must not fill that gap with the pre-deploy state: + // the delete went through and the create did not, so the resource really is gone, and + // an operation with no state is how the deployment says so. The next plan creates it, + // which is what needs to happen. f := &fakeOpClient{sequence: "2"} r := newOperationRecorder(f, "dep-1", 2) uploadOne(t, r, "resources.jobs.foo", deployplan.Recreate, "old-id", nil) - failed, err := newFailedOperation(deployplan.Recreate, "new-id", envelope(t, "the replacement"), errors.New("boom")) + failed, err := newFailedOperation(deployplan.Recreate, "old-id", envelope(t, "before the deploy"), errors.New("boom")) require.NoError(t, err) require.NoError(t, r.upload(t.Context(), "resources.jobs.foo", failed)) require.Len(t, f.calls, 2) update := f.calls[1] assert.Equal(t, "update", update.method) - assert.Equal(t, []string{"state", "error_message", "resource_id", "status"}, update.fields) - require.NotNil(t, update.update.State) - assert.Equal(t, "new-id", update.update.ResourceId) + assert.Equal(t, []string{"error_message", "status"}, update.fields) + assert.Nil(t, update.update.State) + assert.Equal(t, bundledeployments.OperationStatusOperationStatusFailed, update.update.Status) + assert.Equal(t, "boom", update.update.ErrorMessage) } func TestOperationRecorderFailureCarryingALaterWriteSendsIt(t *testing.T) { // Two writes for one resource: the first is uploaded, the second is still waiting - // when the resource fails, so the failure carries it (see coalesce). That state is - // newer than what the service holds, so the update has to name state - leaving it out - // would keep the first write's state and the next plan would read back something this - // deploy has already moved past. + // when the resource fails, so the failure takes it over (see coalesce). The local + // state holds that second write, so the update has to name state - leaving it out + // would keep the first write's state, and the next plan reads DMS. f := &fakeOpClient{sequence: "4"} r := newOperationRecorder(f, "dep-1", 2) @@ -169,9 +169,10 @@ func TestOperationRecorderFailureCarryingALaterWriteSendsIt(t *testing.T) { } func TestOperationRecorderFailureBeforeAnyWriteCarriesPriorState(t *testing.T) { - // Nothing has been recorded for the resource, so the failure creates the operation - // and has to carry the prior state itself - the resource still exists, and the - // service rejects an operation that records state without an id. + // Nothing was recorded for the resource in this version, so the failure creates the + // operation and carries the prior state: nothing touched the resource, so it is still + // there, and an operation without state would drop it from the deployment and have the + // next plan create a second one. f := &fakeOpClient{sequence: "1"} r := newOperationRecorder(f, "dep-1", 2) @@ -230,6 +231,8 @@ func TestNewFailedOperationRecordsError(t *testing.T) { assert.Equal(t, "cluster spec is invalid", op.errorMessage) // The resource was never written, so there is no state to serve back for it. assert.Nil(t, op.state) + // An update only marks the operation failed; see failedKeepingState. + assert.Equal(t, failedKeepingState, op.updateFields) } func TestNewFailedOperationRecordsPriorStateWithID(t *testing.T) { diff --git a/bundle/direct/opsink.go b/bundle/direct/opsink.go index 265d926bff8..caa838cf70d 100644 --- a/bundle/direct/opsink.go +++ b/bundle/direct/opsink.go @@ -134,23 +134,15 @@ func (s *operationSink) record(resourceKey string, op recordedOperation) { // coalesce replaces a waiting operation with the one that supersedes it. The newer one // says how the resource looks now, so its fields win. // -// A failure is different, because it says why the resource stopped rather than how it -// looks, and the state it carries is from before the deploy. Which state should reach -// the service depends on whether the write it replaces got uploaded: -// -// - Uploaded: the service already has the right state. The failure sends none, and -// updateMask leaves it out so it stays. -// - Not uploaded: nobody has told the service anything yet, so this failure is the -// only chance to. It takes the state of the write it replaces, and updateMask -// includes it. -// -// Either way the pre-deploy state is not what goes: it is older than the write. +// A failure is the exception: it reports the state from before the deploy, while the write +// it supersedes says what this deploy actually did to the resource. So the failure takes +// that write's state over, along with the mask that writes it - present or absent, because +// absent is what a recreate's delete step leaves and the resource really is gone. Waiting +// or already uploaded, the write's state is what gets recorded either way. func coalesce(older, newer recordedOperation) recordedOperation { - if newer.isFailure() && older.state != nil { + if newer.isFailure() { newer.state = older.state newer.resourceID = older.resourceID - // The mask comes with the state: the failure is now reporting how the resource - // looks, so it has to say so rather than leave state alone. newer.updateFields = older.updateFields } return newer diff --git a/bundle/direct/opsink_test.go b/bundle/direct/opsink_test.go index 960213c8e7a..43fb5fd47a9 100644 --- a/bundle/direct/opsink_test.go +++ b/bundle/direct/opsink_test.go @@ -183,6 +183,32 @@ func TestOperationSinkCoalescedFailureKeepsTheStateItReplaces(t *testing.T) { assert.Equal(t, "run did not succeed: FAILED", f.errorMessageFor("resources.job_runs.my_run")) } +func TestOperationSinkCoalescedFailureAfterADeleteKeepsTheResourceGone(t *testing.T) { + // A recreate's delete step writes no state, and the create that follows fails. Whether + // that delete was uploaded or is still waiting must not change what DMS ends up with: + // the resource was deleted, so the failure carries the delete's absent state rather + // than the pre-deploy state, and the resource stays gone. + f := &fakeUploader{block: make(chan struct{}), started: make(chan string, 2)} + s := newOperationSink(t.Context(), f) + + recordState(t, s, "resources.jobs.busy", "v1") + assert.Equal(t, "resources.jobs.busy", <-f.started) + + s.RecordOperation(t.Context(), "resources.schemas.foo", dstate.OperationInfo{Action: deployplan.Recreate, InProgress: true}, "old-id", nil) + s.recordFailure(t.Context(), "resources.schemas.foo", deployplan.Recreate, "old-id", envelope(t, "before the deploy"), errors.New("Catalog 'other' does not exist")) + + close(f.block) + require.NoError(t, s.close()) + + assert.Equal(t, []string{ + `resources.jobs.busy={"state":{"name":"v1"}}`, + `resources.schemas.foo=`, + }, f.recorded()) + assert.Equal(t, + bundledeployments.OperationStatusOperationStatusFailed, + f.statusFor("resources.schemas.foo")) +} + func TestOperationSinkCoalescedFailureDoesNotRevertToPriorState(t *testing.T) { // An update that succeeded and then failed waiting carries the pre-deploy state, // which the write it supersedes has already moved past. Sending that would record From caee7e5a6f6c418ed94a89f8b55e0be4b5260d4b Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 17 Aug 2026 13:28:01 +0000 Subject: [PATCH 094/125] acceptance: cover a failed update carrying the pre-deploy state An update whose API call fails writes no state, so the failure is the first operation recorded for the resource in that version and carries the state from before the deploy. The update did not go through, so that state is still the best rendition of the resource we have, and recording nothing would drop it from the deployment and have the next plan create a second one. The failed recreate covered alongside this is the opposite case: there the delete did go through, so the blank record is the accurate one. Co-authored-by: Isaac --- .../bundle/dms/failed-update/databricks.yml | 12 +++ .../bundle/dms/failed-update/out.test.toml | 3 + .../bundle/dms/failed-update/output.txt | 88 +++++++++++++++++++ acceptance/bundle/dms/failed-update/script | 25 ++++++ 4 files changed, 128 insertions(+) create mode 100644 acceptance/bundle/dms/failed-update/databricks.yml create mode 100644 acceptance/bundle/dms/failed-update/out.test.toml create mode 100644 acceptance/bundle/dms/failed-update/output.txt create mode 100644 acceptance/bundle/dms/failed-update/script diff --git a/acceptance/bundle/dms/failed-update/databricks.yml b/acceptance/bundle/dms/failed-update/databricks.yml new file mode 100644 index 00000000000..a11761098cf --- /dev/null +++ b/acceptance/bundle/dms/failed-update/databricks.yml @@ -0,0 +1,12 @@ +bundle: + name: dms-failed-update + +experimental: + record_deployment_history: true + +resources: + schemas: + foo: + name: dms_failed_update_schema + catalog_name: main + comment: v1 diff --git a/acceptance/bundle/dms/failed-update/out.test.toml b/acceptance/bundle/dms/failed-update/out.test.toml new file mode 100644 index 00000000000..7daaf6fd56a --- /dev/null +++ b/acceptance/bundle/dms/failed-update/out.test.toml @@ -0,0 +1,3 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/dms/failed-update/output.txt b/acceptance/bundle/dms/failed-update/output.txt new file mode 100644 index 00000000000..82641fb0a32 --- /dev/null +++ b/acceptance/bundle/dms/failed-update/output.txt @@ -0,0 +1,88 @@ + +=== Deploy the schema, so there is an existing resource to update +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-failed-update/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +=== An update that fails before writing any state records the state from before the deploy. Nothing touched the schema, so it still exists, and an operation without state would drop it from the deployment +>>> update_file.py databricks.yml comment: v1 comment: v2 + +>>> fault.py PATCH /api/2.1/unity-catalog/schemas/* 400 0 1 INVALID_PARAMETER_VALUE + +>>> musterr [CLI] bundle deploy --auto-approve +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-failed-update/default/files... +Deploying resources... +Error: cannot update resources.schemas.foo: updating id=main.dms_failed_update_schema: Fault injected by test. (400 INVALID_PARAMETER_VALUE) + +Endpoint: PATCH [DATABRICKS_URL]/api/2.1/unity-catalog/schemas/main.dms_failed_update_schema +HTTP Status: 400 Bad Request +API error_code: INVALID_PARAMETER_VALUE +API message: Fault injected by test. + +Updating deployment state... + +>>> print_requests.py //api/2.0/bundle --sort +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions", + "q": { + "version_id": "2" + }, + "body": { + "cli_version": "[CLI_VERSION]", + "version_type": "VERSION_TYPE_DEPLOY", + "target_name": "default", + "display_name": "dms-failed-update", + "previous_version_id": "1", + "workspace_info": { + "file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-failed-update/default/files", + "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-failed-update/default" + } + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/complete", + "body": { + "completion_reason": "VERSION_COMPLETE_FAILURE" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations", + "q": { + "resource_key": "schemas.foo" + }, + "body": { + "action_type": "OPERATION_ACTION_TYPE_UPDATE", + "error_message": "updating id=main.dms_failed_update_schema: Fault injected by test.", + "resource_id": "main.dms_failed_update_schema", + "resource_key": "schemas.foo", + "state": "{\"state\":{\"catalog_name\":\"main\",\"comment\":\"v1\",\"name\":\"dms_failed_update_schema\"}}", + "status": "OPERATION_STATUS_FAILED" + } +} + +=== The schema is still listed, described as it was before the failed deploy +>>> MSYS_NO_PATHCONV=1 [CLI] api get /api/2.0/bundle/deployments/[NUMID]/resources +{ + "resources": [ + { + "last_action_type": "OPERATION_ACTION_TYPE_UPDATE", + "last_version_id": "2", + "name": "deployments/[NUMID]/resources/schemas.foo", + "resource_id": "main.dms_failed_update_schema", + "resource_key": "schemas.foo", + "resource_type": "", + "state": "{\"state\":{\"catalog_name\":\"main\",\"comment\":\"v1\",\"name\":\"dms_failed_update_schema\"}}" + } + ] +} + +=== Planning from DMS state alone updates the schema rather than creating a second one, which is what recording the prior state buys +>>> [CLI] bundle plan +update schemas.foo + +Plan: 0 to add, 1 to change, 0 to delete, 0 unchanged diff --git a/acceptance/bundle/dms/failed-update/script b/acceptance/bundle/dms/failed-update/script new file mode 100644 index 00000000000..f7f104fb5e7 --- /dev/null +++ b/acceptance/bundle/dms/failed-update/script @@ -0,0 +1,25 @@ +title "Deploy the schema, so there is an existing resource to update" +trace $CLI bundle deploy +rm -f out.requests.txt + +title "An update that fails before writing any state records the state from before the deploy. Nothing touched the schema, so it still exists, and an operation without state would drop it from the deployment" +trace update_file.py databricks.yml "comment: v1" "comment: v2" +# Fail the update call itself, so the deploy never writes state for the schema. +trace fault.py "PATCH /api/2.1/unity-catalog/schemas/*" 400 0 1 INVALID_PARAMETER_VALUE +trace musterr $CLI bundle deploy --auto-approve +trace print_requests.py //api/2.0/bundle --sort + +title "The schema is still listed, described as it was before the failed deploy" +# The deployment ID is the workspace node's ID; read it back the way the CLI does. +# Extracted with python, not jq: the ID exceeds 2^53 and jq 1.6 rounds it. +deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-failed-update/default/state/resources.deployment.json" -o json | python3 -c 'import sys,json; print(json.load(sys.stdin)["object_id"])') +# MSYS_NO_PATHCONV as above: Git Bash on Windows would rewrite the leading-'/' API +# path into C:/Program Files/Git/api/2.0/... before the CLI saw it. +trace MSYS_NO_PATHCONV=1 $CLI api get "/api/2.0/bundle/deployments/${deployment_id}/resources" + +title "Planning from DMS state alone updates the schema rather than creating a second one, which is what recording the prior state buys" +rm -rf .databricks +trace $CLI bundle plan +# plan probes the state files concurrently, so the recorded order varies. Only the +# operations above are asserted; drop the rest rather than diff a racy order. +rm -f out.requests.txt From 138ea82076a2c07d620ad81aa63a376b09b53817 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 17 Aug 2026 13:58:06 +0000 Subject: [PATCH 095/125] bundle: refresh a golden and drop a stale doc comment Adding test.toml to corrupted-wal-entry to opt it out of the recording variant also added a file to the bundle root, so the deploy syncs one more file. The comment for createDeploymentVersion outlived the function: the flow was split into PrepareDeployment and CreateVersion so a declined deploy never claims a version, and the old comment was left stacked above the new one. Co-authored-by: Isaac --- acceptance/bundle/deploy/wal/corrupted-wal-entry/output.txt | 2 +- libs/dms/recorder.go | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/acceptance/bundle/deploy/wal/corrupted-wal-entry/output.txt b/acceptance/bundle/deploy/wal/corrupted-wal-entry/output.txt index e9762bf4cb6..f24d2764418 100644 --- a/acceptance/bundle/deploy/wal/corrupted-wal-entry/output.txt +++ b/acceptance/bundle/deploy/wal/corrupted-wal-entry/output.txt @@ -11,7 +11,7 @@ Warn: Saved 1 corrupted WAL entries to [TEST_TMP_DIR]/.databricks/bundle/default Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/wal-corrupted-test/default/files... Created jobs.another_valid Created jobs.valid_job -Files: 6 uploaded, 0 deleted +Files: 7 uploaded, 0 deleted Resources: 2 created, 0 changed, 0 deleted, 0 unchanged >>> [CLI] bundle summary diff --git a/libs/dms/recorder.go b/libs/dms/recorder.go index a6a0442156c..2386549d3e2 100644 --- a/libs/dms/recorder.go +++ b/libs/dms/recorder.go @@ -255,9 +255,6 @@ func (r *Recorder) completeVersion(ctx context.Context, reason bundledeployments return nil } -// createDeploymentVersion ensures the deployment record exists, then creates a new -// version under it: with no ID it creates the deployment, otherwise it reads the -// existing one for the next version number. // PrepareDeployment makes sure the deployment exists and works out the version number // this deploy will create, without creating it. Both are needed before the plan, which // stamps them onto the resources it is computed from; the version itself is not created From a2b80149168eb4596191d00edcce1c0d28f671c2 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 17 Aug 2026 23:28:03 +0000 Subject: [PATCH 096/125] bundle: trim the comments in opsink.go Each one is now at most three lines and says the thing plainly. Nothing dropped that the code cannot say for itself: the record-outside-the-state-DB-lock requirement, why a failed upload fails the deploy, and why coalescing a failure takes the superseded write's state even when that state is absent. Co-authored-by: Isaac --- bundle/direct/opsink.go | 95 ++++++++++++++--------------------------- 1 file changed, 33 insertions(+), 62 deletions(-) diff --git a/bundle/direct/opsink.go b/bundle/direct/opsink.go index caa838cf70d..8cffe337f08 100644 --- a/bundle/direct/opsink.go +++ b/bundle/direct/opsink.go @@ -10,54 +10,39 @@ import ( "github.com/databricks/cli/bundle/direct/dstate" ) -// operationSinkQueueSize is how many resources may have an unrecorded state write -// before the next one waits. Recording is deliberately not free: a deploy that ran far -// ahead of the service would leave a long tail of resources applied but unrecorded, and -// DMS is the source of truth for the next plan. At one waiting operation per resource -// this bounds the lag to roughly the apply parallelism. +// operationSinkQueueSize is how many resources may be waiting to be recorded before the +// next write has to wait. Without a cap a deploy could finish far ahead of the service, +// and DMS is what the next plan reads. const operationSinkQueueSize = 10 -// operationSink uploads recorded operations from one background goroutine, so a deploy -// does not wait on every CreateOperation round trip and the service sees one request at -// a time. -// -// The queue carries resource keys and pending holds the operation for each, which is -// what makes coalescing fall out: a second write for a resource replaces the first in -// the map, and the key already in the queue picks up whichever operation is there when -// the uploader reaches it. -// -// close reports the first upload failure, which fails the deploy: DMS becomes the -// source of truth (see dstate.readDMSState), so a missing record would make the next -// deploy create a resource that already exists. +// operationSink uploads operations one at a time on a background goroutine, so a deploy +// never waits on a round trip. queue holds resource keys and pending holds the newest +// operation per key, so a second write for a resource simply replaces the first. type operationSink struct { uploader operationUploader - // queue carries the resource keys that have an operation waiting. A key is sent - // only when nothing was waiting for it, so a resource never occupies more than one - // slot; once the queue is full, recording waits for the uploader, which is what - // holds the deploy back. Callers must therefore record outside the state DB lock - - // see dstate.SaveState. + // queue holds the keys that have something waiting. One slot per resource, so a full + // queue means the deploy is that many resources ahead and the next write waits. Record + // outside the state DB lock, or that wait blocks every other resource too. queue chan string // done is closed once the uploader has drained the queue and returned. done chan struct{} - // stopQueue closes the queue. Wrapped so close can be deferred and still checked - // at a specific point. + // stopQueue closes the queue, wrapped so close can safely run twice. stopQueue func() // mu guards the fields below. mu sync.Mutex - // pending holds the operation waiting per resource key. A key is absent once the - // uploader has taken its operation. + // pending holds the newest operation per resource key, absent once the uploader takes it. pending map[string]recordedOperation err error } -// newOperationSink starts the uploader, returning nil when uploader is nil (recording -// off; every method is a no-op on a nil sink). ctx must outlive close. +// newOperationSink starts the uploader. It returns nil when recording is off, and every +// method is a no-op on a nil sink. ctx must outlive close. func newOperationSink(ctx context.Context, uploader operationUploader) *operationSink { if uploader == nil { return nil @@ -75,12 +60,9 @@ func newOperationSink(ctx context.Context, uploader operationUploader) *operatio return s } -// RecordOperation implements dstate.OperationSink: every state write becomes an -// operation, so DMS mirrors the WAL. state is already the serialized envelope, and -// nil for a delete. -// -// An earlier upload failure does not stop this: every write is still recorded, best -// effort, so DMS ends up as close to reality as it can get. +// RecordOperation implements dstate.OperationSink, turning every state write into an +// operation so DMS mirrors the local state. state is the serialized envelope, and nil for +// a delete. An earlier failure does not stop it: keep recording, best effort. func (s *operationSink) RecordOperation(ctx context.Context, resourceKey string, info dstate.OperationInfo, resourceID string, state json.RawMessage) { if s == nil { return @@ -95,8 +77,8 @@ func (s *operationSink) RecordOperation(ctx context.Context, resourceKey string, s.record(resourceKey, op) } -// recordFailure records that applying a resource failed, so the deployment history -// explains the failure instead of omitting the resource. +// recordFailure records that a resource did not apply, so the history says why rather +// than leaving the resource out. func (s *operationSink) recordFailure(ctx context.Context, resourceKey string, action deployplan.ActionType, resourceID string, priorState json.RawMessage, cause error) { if s == nil { return @@ -111,9 +93,8 @@ func (s *operationSink) recordFailure(ctx context.Context, resourceKey string, a s.record(resourceKey, op) } -// record makes op the operation waiting for resourceKey, waiting for the uploader when -// the queue is full. Recording on a closed sink panics; every caller must have returned -// before close. +// record makes op the one waiting for resourceKey, waiting itself while the queue is +// full. Recording after close panics, so every caller must return before close. func (s *operationSink) record(resourceKey string, op recordedOperation) { s.mu.Lock() waiting, queued := s.pending[resourceKey] @@ -123,22 +104,16 @@ func (s *operationSink) record(resourceKey string, op recordedOperation) { s.pending[resourceKey] = op s.mu.Unlock() - // Already queued: the uploader reads the map when it reaches the key, so it picks - // up what was just stored without a second token - and without waiting, since a - // resource that is already represented in the queue is not running ahead. + // Already queued: the uploader reads the map when it gets to the key, so it picks up + // what was just stored. No second slot, and no waiting. if !queued { s.queue <- resourceKey } } -// coalesce replaces a waiting operation with the one that supersedes it. The newer one -// says how the resource looks now, so its fields win. -// -// A failure is the exception: it reports the state from before the deploy, while the write -// it supersedes says what this deploy actually did to the resource. So the failure takes -// that write's state over, along with the mask that writes it - present or absent, because -// absent is what a recreate's delete step leaves and the resource really is gone. Waiting -// or already uploaded, the write's state is what gets recorded either way. +// coalesce merges a write that superseded another still waiting. The newer one wins, +// except a failure, which only carries pre-deploy state: it takes the superseded write's +// state and mask instead - absent state included, since a recreate's delete did remove it. func coalesce(older, newer recordedOperation) recordedOperation { if newer.isFailure() { newer.state = older.state @@ -164,23 +139,21 @@ func (s *operationSink) run(ctx context.Context) { for resourceKey := range s.queue { op, ok := s.take(resourceKey) if !ok { - // A key is queued only when nothing is waiting for it, so this cannot - // happen; the check is here so a stray token could never upload a - // zero-valued operation. + // Unreachable: a key is queued only when nothing was waiting for it. Guard so + // a stray key could never upload a zero-valued operation. continue } - // Keep going after a failure, so one bad upload does not drop the records for - // every resource behind it. + // Keep going after a failure, so one bad upload does not drop everything behind it. if err := s.uploader.upload(ctx, resourceKey, op); err != nil { s.setErr(fmt.Errorf("recording operation for %s with the deployment metadata service: %w", resourceKey, err)) } } } -// close drains the recorded operations and returns the first upload error. Every -// record caller must have returned first; calling close twice is safe, so it can be -// deferred and still checked at a specific point. +// close drains what is waiting and returns the first upload error, which fails the deploy: +// DMS is the source of truth, so a missing record would have the next deploy create a +// resource that already exists. Safe to call twice. func (s *operationSink) close() error { if s == nil { return nil @@ -191,8 +164,7 @@ func (s *operationSink) close() error { return s.firstErr() } -// setErr keeps the first recording error; later ones are dropped because one failure -// is enough to fail the deploy. +// setErr keeps the first error; one failure is enough to fail the deploy. func (s *operationSink) setErr(err error) { s.mu.Lock() defer s.mu.Unlock() @@ -202,8 +174,7 @@ func (s *operationSink) setErr(err error) { } } -// firstErr returns the first recording error, or nil if everything so far was -// recorded. A nil sink (recording disabled) never errors. +// firstErr returns the first recording error, or nil. A nil sink never errors. func (s *operationSink) firstErr() error { if s == nil { return nil From 9eb910638503469cccf3e08f955430f2413a6973 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 17 Aug 2026 23:43:11 +0000 Subject: [PATCH 097/125] bundle: keep the write's operation when a failure coalesces onto it coalesce copied state, resource_id and the update mask off the superseded write onto the failure, so three correlated fields had to be kept consistent by hand and the mask was only right because the superseded operation was always a state write. A retried resource breaks that: the newer operation would be the write, and the failure's stale id and state would have overwritten it. Inverted instead. A later write wins outright, and a failure keeps the write's operation and stamps on only what it owns - status and error message. The mask is never assigned, so it travels with whichever operation survives, and an absent state is kept for free, which is what a recreate's delete has to leave behind. Behaviour is unchanged; two tests now pin both directions. Co-authored-by: Isaac --- bundle/direct/opsink.go | 22 +++++++++++++-------- bundle/direct/opsink_test.go | 38 ++++++++++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+), 8 deletions(-) diff --git a/bundle/direct/opsink.go b/bundle/direct/opsink.go index 8cffe337f08..c748aca75ec 100644 --- a/bundle/direct/opsink.go +++ b/bundle/direct/opsink.go @@ -111,16 +111,22 @@ func (s *operationSink) record(resourceKey string, op recordedOperation) { } } -// coalesce merges a write that superseded another still waiting. The newer one wins, -// except a failure, which only carries pre-deploy state: it takes the superseded write's -// state and mask instead - absent state included, since a recreate's delete did remove it. +// coalesce merges an operation with the one that superseded it while still waiting. A later +// write says everything about the resource, so it wins outright. A failure knows only why the +// resource stopped, so the write is kept and the failure stamps its outcome onto it. func coalesce(older, newer recordedOperation) recordedOperation { - if newer.isFailure() { - newer.state = older.state - newer.resourceID = older.resourceID - newer.updateFields = older.updateFields + if !newer.isFailure() { + return newer } - return newer + + // Keeping the write keeps its state and mask - an absent state included, which is right: + // a recreate's delete really did remove the resource. + older.status = newer.status + older.errorMessage = newer.errorMessage + // An update that empties a resource records its write as a delete (see DeploymentUnit + // .Update), so the two can disagree. Report what the plan set out to do. + older.action = newer.action + return older } // take claims the operation waiting for resourceKey. diff --git a/bundle/direct/opsink_test.go b/bundle/direct/opsink_test.go index 43fb5fd47a9..c6dfbd5c36a 100644 --- a/bundle/direct/opsink_test.go +++ b/bundle/direct/opsink_test.go @@ -261,6 +261,44 @@ func TestOperationSinkCoalescedDeleteStillClearsState(t *testing.T) { f.actionFor("resources.jobs.foo")) } +func TestCoalesceLetsAWriteSupersedeAFailure(t *testing.T) { + // A failure is not the last word. If a resource were retried and then wrote state, that + // write describes the resource and has to win whole - its state, its id, and its mask, + // which names error_message so the recorded failure is cleared. The service rejects a + // succeeded operation that still carries an error. + failed, err := newFailedOperation(deployplan.Update, "id-old", envelope(t, "before"), errors.New("boom")) + require.NoError(t, err) + retried, err := newStateOperation(dstate.OperationInfo{Action: deployplan.Update}, "id-new", envelope(t, "after the retry")) + require.NoError(t, err) + + got := coalesce(failed, retried) + + assert.Equal(t, bundledeployments.OperationStatusOperationStatusSucceeded, got.status) + assert.Empty(t, got.errorMessage) + assert.Equal(t, "id-new", got.resourceID) + assert.JSONEq(t, string(envelope(t, "after the retry")), string(got.state)) + assert.Equal(t, describesResource, got.updateFields) +} + +func TestCoalesceKeepsTheWritesStateAndMask(t *testing.T) { + // The other direction: a failure contributes only its outcome, so the write's state, id + // and mask survive and the failure's own pre-deploy state is dropped as the older of the + // two. Action comes from the failure, which reports what the plan set out to do. + write, err := newStateOperation(dstate.OperationInfo{Action: deployplan.Delete}, "id-new", nil) + require.NoError(t, err) + failed, err := newFailedOperation(deployplan.Update, "id-old", envelope(t, "before"), errors.New("boom")) + require.NoError(t, err) + + got := coalesce(write, failed) + + assert.Equal(t, bundledeployments.OperationStatusOperationStatusFailed, got.status) + assert.Equal(t, "boom", got.errorMessage) + assert.Equal(t, "id-new", got.resourceID) + assert.Nil(t, got.state) + assert.Equal(t, describesResource, got.updateFields) + assert.Equal(t, bundledeployments.OperationActionTypeOperationActionTypeUpdate, got.action) +} + func TestOperationSinkRecordDuringUploadIsStillUploaded(t *testing.T) { f := &fakeUploader{block: make(chan struct{}), started: make(chan string, 2)} s := newOperationSink(t.Context(), f) From c7e749bc626304a30497575da8ecca14b41d03f4 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 17 Aug 2026 23:49:23 +0000 Subject: [PATCH 098/125] bundle: merge the update mask field by field when coalescing coalesce decided what a failure does and does not report, which is the service's contract rather than the CLI's business. It now merges mechanically: each field comes from whichever operation named it in its mask, the newer one wins when both did, and the merged mask is the union so neither operation's fields are dropped. Which fields an operation claims stays where it is built, next to the reason - describesResource and failedKeepingState. coalesce no longer mentions failures at all, so isFailure had no callers left and is gone. The union is ordered by describesResource, so the mask on the wire is unchanged. Co-authored-by: Isaac --- bundle/direct/oprecorder.go | 20 +++++++------ bundle/direct/opsink.go | 56 +++++++++++++++++++++++++++---------- 2 files changed, 53 insertions(+), 23 deletions(-) diff --git a/bundle/direct/oprecorder.go b/bundle/direct/oprecorder.go index ce4c894ce19..fe410a3db65 100644 --- a/bundle/direct/oprecorder.go +++ b/bundle/direct/oprecorder.go @@ -58,20 +58,24 @@ type recordedOperation struct { updateFields []string } +// The fields an UpdateOperation may change. Any other path is rejected with +// INVALID_PARAMETER_VALUE, so this is the full universe a mask can name. +const ( + fieldState = "state" + fieldErrorMessage = "error_message" + fieldResourceID = "resource_id" + fieldStatus = "status" +) + // describesResource is the update mask for an operation that says how the resource -// looks: everything an update is allowed to change. -var describesResource = []string{"state", "error_message", "resource_id", "status"} +// looks: everything an update is allowed to change. Its order is the canonical one. +var describesResource = []string{fieldState, fieldErrorMessage, fieldResourceID, fieldStatus} // failedKeepingState is the update mask for a failure updating an operation this version // already recorded: mark it failed and leave state alone. That is right either way - state // means the resource is as it was written, and no state means a delete went through and // nothing replaced it, so the resource really is gone and the deployment should say so. -var failedKeepingState = []string{"error_message", "status"} - -// isFailure reports whether the operation records a resource that did not apply. -func (op recordedOperation) isFailure() bool { - return op.status == bundledeployments.OperationStatusOperationStatusFailed -} +var failedKeepingState = []string{fieldErrorMessage, fieldStatus} // newStateOperation describes a state write for upload. state is the serialized // RecordedState envelope the state DB just persisted, and nil for a delete, where diff --git a/bundle/direct/opsink.go b/bundle/direct/opsink.go index c748aca75ec..b3e72be2e0b 100644 --- a/bundle/direct/opsink.go +++ b/bundle/direct/opsink.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "fmt" + "slices" "sync" "github.com/databricks/cli/bundle/deployplan" @@ -111,22 +112,47 @@ func (s *operationSink) record(resourceKey string, op recordedOperation) { } } -// coalesce merges an operation with the one that superseded it while still waiting. A later -// write says everything about the resource, so it wins outright. A failure knows only why the -// resource stopped, so the write is kept and the failure stamps its outcome onto it. +// coalesce merges an operation with the one that superseded it while still waiting. Each +// field comes from whichever operation claimed it in its mask, and the newer one wins when +// both did; the merged mask is the union, so neither operation's fields get dropped. +// +// What a given operation claims is decided where it is built, not here - see +// describesResource and failedKeepingState. func coalesce(older, newer recordedOperation) recordedOperation { - if !newer.isFailure() { - return newer - } - - // Keeping the write keeps its state and mask - an absent state included, which is right: - // a recreate's delete really did remove the resource. - older.status = newer.status - older.errorMessage = newer.errorMessage - // An update that empties a resource records its write as a delete (see DeploymentUnit - // .Update), so the two can disagree. Report what the plan set out to do. - older.action = newer.action - return older + merged := older + + // action_type is fixed when the operation is created, so no mask names it. An update + // that empties a resource records its write as a delete (see DeploymentUnit.Update), so + // the two can disagree; report what the plan set out to do. + merged.action = newer.action + merged.updateFields = unionFields(older.updateFields, newer.updateFields) + + if slices.Contains(newer.updateFields, fieldState) { + merged.state = newer.state + } + if slices.Contains(newer.updateFields, fieldResourceID) { + merged.resourceID = newer.resourceID + } + if slices.Contains(newer.updateFields, fieldErrorMessage) { + merged.errorMessage = newer.errorMessage + } + if slices.Contains(newer.updateFields, fieldStatus) { + merged.status = newer.status + } + + return merged +} + +// unionFields returns every field either mask names, in describesResource's order so the +// merged mask is deterministic on the wire. +func unionFields(older, newer []string) []string { + merged := make([]string, 0, len(describesResource)) + for _, field := range describesResource { + if slices.Contains(older, field) || slices.Contains(newer, field) { + merged = append(merged, field) + } + } + return merged } // take claims the operation waiting for resourceKey. From 7d8ed7e13aa82917794cdd400967eed270afcfaf Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 18 Aug 2026 00:34:43 +0000 Subject: [PATCH 099/125] bundle: hold every comment in this PR to three lines A sweep over the 82 comment blocks this PR added that ran longer. Each is now at most three lines and says the thing plainly, with the reasons kept: why recording happens outside the state DB lock, why a version is claimed before it is created, why a test's expectation is what it is, and the API quirks behind the hand-written request types. Also corrects a claim in DeploymentUnit.Update: the service drops a resource from a deployment when its recorded state is empty, not because the action is a delete. Co-authored-by: Isaac --- acceptance/bundle/deploy/readplan/test.toml | 7 +- acceptance/bundle/dms/emptied-resource/script | 6 +- acceptance/bundle/dms/no-drift/script | 6 +- .../dms/operation-upload-fails/test.toml | 6 +- acceptance/bundle/dms/partial-update/script | 7 +- acceptance/bundle/dms/record/script | 6 +- acceptance/bundle/dms/test.toml | 6 +- .../bundle/invariant/continue_293/test.toml | 6 +- .../invariant/delete_idempotent/test.toml | 6 +- .../invariant/destroy_idempotent/test.toml | 6 +- acceptance/bundle/migrate/test.toml | 12 +-- .../bundle/resources/jobs/big_id/test.toml | 9 +- .../resources/jobs/delete_task/test.toml | 6 +- .../jobs/remote_delete/deploy/test.toml | 6 +- .../bundle/resources/jobs/update/test.toml | 6 +- acceptance/bundle/templates/test.toml | 8 +- acceptance/bundle/test.toml | 7 +- bundle/config/experimental.go | 9 +- .../mutator/initialize_deployment_history.go | 9 +- .../validate_record_deployment_history.go | 13 +-- .../metadata/annotate_deployment_version.go | 14 +-- bundle/direct/apply.go | 20 +--- bundle/direct/bundle_apply.go | 17 ++-- bundle/direct/dstate/dms.go | 42 ++++---- bundle/direct/dstate/state.go | 26 ++--- bundle/direct/dstate/state_test.go | 7 +- bundle/direct/opclient.go | 21 ++-- bundle/direct/oprecorder.go | 99 ++++++------------- bundle/direct/oprecorder_test.go | 25 ++--- bundle/direct/opsink.go | 26 +++-- bundle/direct/opsink_test.go | 43 ++++---- bundle/direct/pkg.go | 8 +- bundle/env/dms.go | 17 +--- .../force_allow_record_deployment_history.go | 7 +- bundle/phases/deploy.go | 31 ++---- bundle/phases/destroy.go | 11 +-- bundle/phases/dms.go | 24 +---- cmd/bundle/utils/process.go | 11 +-- libs/dms/recorder.go | 23 ++--- libs/dms/resolve.go | 9 +- libs/testserver/bundle.go | 91 ++++++----------- libs/testserver/fake_workspace.go | 13 +-- libs/workspaceurls/urls.go | 10 +- 43 files changed, 243 insertions(+), 499 deletions(-) diff --git a/acceptance/bundle/deploy/readplan/test.toml b/acceptance/bundle/deploy/readplan/test.toml index 24b27839975..0d4c0bd5b6e 100644 --- a/acceptance/bundle/deploy/readplan/test.toml +++ b/acceptance/bundle/deploy/readplan/test.toml @@ -1,6 +1,3 @@ -# These tests apply a saved plan, which does not carry the deployment stamp: on a first -# deploy there is no deployment to resolve when `bundle plan` runs, so the plan it writes -# leaves the field unset and applying it plans an update the next time. Same reason as -# EnvMatrixExclude.dms_no_readplan in acceptance/bundle/test.toml, which only covers the -# tests that take the saved-plan path through the READPLAN matrix variable. +# Saved plans don't carry the deployment stamp. Applying one plans an update on the next run. +# See dms_no_readplan in acceptance/bundle/test.toml. EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/dms/emptied-resource/script b/acceptance/bundle/dms/emptied-resource/script index 929cf6d8cc1..954cfc4a496 100644 --- a/acceptance/bundle/dms/emptied-resource/script +++ b/acceptance/bundle/dms/emptied-resource/script @@ -6,10 +6,8 @@ title "Revoke the grant, so the grants node empties out" trace update_file.py databricks.yml 'grants: [{principal: someone@example.com, privileges: [USE_SCHEMA]}]' 'grants: []' trace $CLI bundle deploy -# The emptied node is recorded as a delete, not as the update that emptied it: the service -# drops a resource from the deployment only on a delete. Recorded as an update it stayed -# listed with an id but no state, and reading it back failed the next plan with -# "unexpected end of JSON input". +# The emptied node records as a delete, not an update. The service only drops +# a resource on delete; otherwise it stays listed with no state. trace print_requests.py //api/2.0/bundle --sort title "Plan again: reading state back from the service works and reports no work" diff --git a/acceptance/bundle/dms/no-drift/script b/acceptance/bundle/dms/no-drift/script index c67c2c28a74..f947997ab92 100644 --- a/acceptance/bundle/dms/no-drift/script +++ b/acceptance/bundle/dms/no-drift/script @@ -1,10 +1,8 @@ title "Deploy, then plan without touching anything: the deployment stamp must not show as drift" trace $CLI bundle deploy -# Only the deploy phase stamps deployment.deployment_id (it is not known until the -# version is claimed), so plan sees it in the state and in the workspace but not in the -# local config. Without an ignore_local_changes rule that absence plans an update on a -# job and a pipeline nobody edited. +# Only deploy stamps deployment.deployment_id. Plan sees it in state but not in config, +# so it must ignore it or report spurious changes. trace $CLI bundle plan | contains.py "Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged" title "A second deploy is a no-op too: no update request for either resource" diff --git a/acceptance/bundle/dms/operation-upload-fails/test.toml b/acceptance/bundle/dms/operation-upload-fails/test.toml index ce222ac9e87..cf53442405d 100644 --- a/acceptance/bundle/dms/operation-upload-fails/test.toml +++ b/acceptance/bundle/dms/operation-upload-fails/test.toml @@ -2,10 +2,8 @@ # failed, so recording them would make the output nondeterministic. RecordRequests = false -# The service rejects every recorded operation. Deploy must stop rather than -# create every remaining resource: a completed version makes DMS the source of -# truth for resource state, so resources it has no record of would be created a -# second time by the next deploy. +# Completed versions make DMS the source of truth; unrecorded resources get recreated. +# Deploy must stop rather than continue creating. [[Server]] Pattern = "POST /api/2.0/bundle/deployments/{deployment_id}/versions/{version_id}/operations" Response.StatusCode = 500 diff --git a/acceptance/bundle/dms/partial-update/script b/acceptance/bundle/dms/partial-update/script index 52340c21037..a9e69d2369a 100644 --- a/acceptance/bundle/dms/partial-update/script +++ b/acceptance/bundle/dms/partial-update/script @@ -3,11 +3,8 @@ trace $CLI bundle deploy trace print_requests.py //api/2.0/bundle title "Recreate writes state twice - the entry is dropped, then the new resource is saved" -# The service keeps one operation per resource per version, so both writes land on the -# same one: the drop opens it as IN_PROGRESS carrying the deleted id but no state, and -# the save that follows patches it to SUCCEEDED with the new id. A deploy that dies in -# between therefore leaves the resource described as mid-recreate rather than as the -# resource it already deleted. +# One operation per resource per version; both writes land on the same one. +# The drop opens it IN_PROGRESS, the save patches it to SUCCEEDED. trace update_file.py databricks.yml "catalog_name: main" "catalog_name: other" trace $CLI bundle deploy --auto-approve trace print_requests.py //api/2.0/bundle diff --git a/acceptance/bundle/dms/record/script b/acceptance/bundle/dms/record/script index 895a9033a39..0829655016a 100644 --- a/acceptance/bundle/dms/record/script +++ b/acceptance/bundle/dms/record/script @@ -3,10 +3,8 @@ trace $CLI bundle deploy trace print_requests.py //api/2.0/bundle title "The deployment ID is the ID of the workspace node the service registered under initial_parent_path; the CLI stores nothing locally" -# MSYS_NO_PATHCONV stops Git Bash on Windows from rewriting the leading-'/' path -# into C:/Program Files/Git/Workspace/... before the CLI sees it. Set per command -# rather than in test.toml: trace exports it in a subshell, so it cannot reach the -# python helpers, whose PATH does need converting. +# MSYS_NO_PATHCONV prevents Git Bash from rewriting the leading-/ path on Windows. +# Set per command rather than test.toml so trace can export it to its subshell. trace MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-record/default/state/resources.deployment.json" | jq '{object_type,path}' trace jq 'has("deployment_id")' .databricks/bundle/default/resources.json diff --git a/acceptance/bundle/dms/test.toml b/acceptance/bundle/dms/test.toml index 041c3da7fc1..5c384661f03 100644 --- a/acceptance/bundle/dms/test.toml +++ b/acceptance/bundle/dms/test.toml @@ -15,8 +15,6 @@ Ignore = [ '.databricks', ] -# experimental.record_deployment_history is rejected outright (see -# validate.ValidateRecordDeploymentHistory). These tests exercise the feature itself, -# so they force allow it the same way DMS development does. bundle/dms/not-supported -# covers the rejection. +# experimental.record_deployment_history is normally rejected; these tests force-allow it +# to exercise the feature. bundle/dms/not-supported covers the rejection path. Env.DATABRICKS_BUNDLE_FORCE_ALLOW_RECORD_DEPLOYMENT_HISTORY = "1" diff --git a/acceptance/bundle/invariant/continue_293/test.toml b/acceptance/bundle/invariant/continue_293/test.toml index 775d604935e..5357c87e57b 100644 --- a/acceptance/bundle/invariant/continue_293/test.toml +++ b/acceptance/bundle/invariant/continue_293/test.toml @@ -1,7 +1,5 @@ -# The seed deploy runs an old CLI that knows nothing about the deployment metadata -# service, so the resources it creates are recorded nowhere. Reading state from the -# service then finds none and plans a create on top of them. Adopting resources a -# pre-DMS CLI deployed is a migration story of its own, not something this test covers. +# The seed deploy runs an old CLI unaware of DMS, so resources aren't recorded. +# Reading state from the service finds nothing; adopting pre-DMS resources is a separate story. EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] # $resources references to permissions and grants are not supported on v0.293.0 diff --git a/acceptance/bundle/invariant/delete_idempotent/test.toml b/acceptance/bundle/invariant/delete_idempotent/test.toml index df880553ee2..0371d73cc1e 100644 --- a/acceptance/bundle/invariant/delete_idempotent/test.toml +++ b/acceptance/bundle/invariant/delete_idempotent/test.toml @@ -1,7 +1,5 @@ -# Recording needs a bundle it has seen from the start. This test rewinds state and -# wipes the remote path the deployment record lives under, so recording refuses it. -# TODO(DMS): drop this once existing state can be handed over to the service (see -# the TODO in dstate.Open). +# Recording needs a bundle from the start. This test rewinds state and wipes the +# deployment record path, so recording refuses it. EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/invariant/destroy_idempotent/test.toml b/acceptance/bundle/invariant/destroy_idempotent/test.toml index 50bd84790d5..96237960ec4 100644 --- a/acceptance/bundle/invariant/destroy_idempotent/test.toml +++ b/acceptance/bundle/invariant/destroy_idempotent/test.toml @@ -1,7 +1,5 @@ -# Recording needs a bundle it has seen from the start. This test rewinds state and -# wipes the remote path the deployment record lives under, so recording refuses it. -# TODO(DMS): drop this once existing state can be handed over to the service (see -# the TODO in dstate.Open). +# Recording needs a bundle from the start. This test rewinds state and wipes the +# deployment record path, so recording refuses it. EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/migrate/test.toml b/acceptance/bundle/migrate/test.toml index 375f5445d1a..00027f9abdf 100644 --- a/acceptance/bundle/migrate/test.toml +++ b/acceptance/bundle/migrate/test.toml @@ -3,13 +3,11 @@ RecordRequests = true IncludeRequestHeaders = ["User-Agent"] Ignore = [".databricks"] -# Each test's script sets DATABRICKS_BUNDLE_ENGINE inline; pin the outer -# matrix to ["direct"] so CI's engine filter includes these tests without -# also running the same script twice per engine. +# Tests set DATABRICKS_BUNDLE_ENGINE inline. Pin to ["direct"] so CI's engine +# filter includes them without running the script twice. EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -# These tests deploy on terraform first and then migrate to direct. Recording is a -# direct-engine feature, so the resources the terraform half creates are recorded nowhere -# and the migration reads state the service does not have. Migrating a deployment onto the -# service is a story of its own; these tests are not it. +# These tests deploy on terraform and then migrate to direct. Recording is direct-only, so +# the terraform half is recorded nowhere and the migration reads state the service lacks. +# Migrating an existing deployment onto the service is its own story. EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/resources/jobs/big_id/test.toml b/acceptance/bundle/resources/jobs/big_id/test.toml index bc436b2d544..ec7d104af8d 100644 --- a/acceptance/bundle/resources/jobs/big_id/test.toml +++ b/acceptance/bundle/resources/jobs/big_id/test.toml @@ -1,12 +1,9 @@ -# terraform fails with: -# panic: Error reading level state: strconv.ParseInt: parsing "[NUMID]": value out of range +# terraform fails: strconv.ParseInt on "[NUMID]" value out of range EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ['direct'] EnvMatrix.READPLAN = ["", "1"] -# `bundle deploy --plan` applies the resource state the plan was saved with, and the plan -# is written before the deployment version exists, so the deployment stamp never reaches -# the applied resource and the next plan reports it as a change. Recording is skipped here -# until the stamp is written into the saved plan too. +# `bundle deploy --plan` applies pre-saved state; the stamp isn't written into the plan, +# so the next plan reports it as a change. Recording skipped until stamp reaches saved plans. EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] [[Repls]] diff --git a/acceptance/bundle/resources/jobs/delete_task/test.toml b/acceptance/bundle/resources/jobs/delete_task/test.toml index 83bd492c4cf..68bc7febe74 100644 --- a/acceptance/bundle/resources/jobs/delete_task/test.toml +++ b/acceptance/bundle/resources/jobs/delete_task/test.toml @@ -1,7 +1,5 @@ RecordRequests = false EnvMatrix.READPLAN = ["", "1"] -# `bundle deploy --plan` applies the resource state the plan was saved with, and the plan -# is written before the deployment version exists, so the deployment stamp never reaches -# the applied resource and the next plan reports it as a change. Recording is skipped here -# until the stamp is written into the saved plan too. +# `bundle deploy --plan` applies pre-saved state; the stamp isn't written into the plan, +# so the next plan reports it as a change. Recording skipped until stamp reaches saved plans. EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/resources/jobs/remote_delete/deploy/test.toml b/acceptance/bundle/resources/jobs/remote_delete/deploy/test.toml index 83bd492c4cf..68bc7febe74 100644 --- a/acceptance/bundle/resources/jobs/remote_delete/deploy/test.toml +++ b/acceptance/bundle/resources/jobs/remote_delete/deploy/test.toml @@ -1,7 +1,5 @@ RecordRequests = false EnvMatrix.READPLAN = ["", "1"] -# `bundle deploy --plan` applies the resource state the plan was saved with, and the plan -# is written before the deployment version exists, so the deployment stamp never reaches -# the applied resource and the next plan reports it as a change. Recording is skipped here -# until the stamp is written into the saved plan too. +# `bundle deploy --plan` applies pre-saved state; the stamp isn't written into the plan, +# so the next plan reports it as a change. Recording skipped until stamp reaches saved plans. EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/resources/jobs/update/test.toml b/acceptance/bundle/resources/jobs/update/test.toml index d8ef3c22ad0..0020ce7fedf 100644 --- a/acceptance/bundle/resources/jobs/update/test.toml +++ b/acceptance/bundle/resources/jobs/update/test.toml @@ -1,6 +1,4 @@ EnvMatrix.READPLAN = ["", "1"] -# `bundle deploy --plan` applies the resource state the plan was saved with, and the plan -# is written before the deployment version exists, so the deployment stamp never reaches -# the applied resource and the next plan reports it as a change. Recording is skipped here -# until the stamp is written into the saved plan too. +# `bundle deploy --plan` applies pre-saved state; the stamp isn't written into the plan, +# so the next plan reports it as a change. Recording skipped until stamp reaches saved plans. EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/templates/test.toml b/acceptance/bundle/templates/test.toml index 344203ecd58..bb62167d1b6 100644 --- a/acceptance/bundle/templates/test.toml +++ b/acceptance/bundle/templates/test.toml @@ -1,9 +1,7 @@ -# Local-only: At the moment, there are many differences across different envs w.r.t to catalog use, node type and so on. +# Local-only: many env differences (catalog use, node type, etc). -# A template test materializes a whole project and deploys it, taking tens of seconds each, -# and some diff against a sibling test's output directory. Running all of that a second time -# for deployment history recording costs minutes and adds no coverage the rest of the suite -# does not already give, so these opt out. +# Template tests materialize and deploy full projects, taking tens of seconds each. +# Recording adds minutes without new coverage the rest of the suite provides. EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] [[Server]] diff --git a/acceptance/bundle/test.toml b/acceptance/bundle/test.toml index 3ef7efd376c..90a7cece928 100644 --- a/acceptance/bundle/test.toml +++ b/acceptance/bundle/test.toml @@ -11,11 +11,8 @@ EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] EnvMatrixExclude.dms_needs_direct = ["DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY=true", "DATABRICKS_BUNDLE_ENGINE=terraform"] EnvMatrixExclude.dms_local_only = ["DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY=true", "CONFIG_Cloud=true"] -# A saved plan does not carry the deployment stamp. On a first deploy there is no -# deployment to resolve when `bundle plan` runs, so the plan it writes leaves the field -# unset; `deploy --plan` then creates the resources without it and the next plan reports -# drift. Stamping at plan time would mean `bundle plan` creating the deployment record, -# which is a design decision, so the saved-plan path is left out of the DMS run for now. +# Saved plans don't carry the deployment stamp. A first plan writes the deployment record, +# so `deploy --plan` creates resources without it and reports drift on the next plan. EnvMatrixExclude.dms_no_readplan = ["DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY=true", "READPLAN=1"] # Recording is gated off for users (see validate.ValidateRecordDeploymentHistory), so diff --git a/bundle/config/experimental.go b/bundle/config/experimental.go index c3f1465d880..dc82eff75f5 100644 --- a/bundle/config/experimental.go +++ b/bundle/config/experimental.go @@ -50,13 +50,8 @@ type Experimental struct { // at which point we can deprecate or remove this field all together. SkipNamePrefixForSchema bool `json:"skip_name_prefix_for_schema,omitempty"` - // RecordDeploymentHistory opts the bundle into the deployment metadata - // service (DMS), which records deployment history and tracks what changed - // across deployments. - // - // Only supported for a bundle with no deployed resources yet: DMS becomes the - // source of truth for resource state, and resources tracked in an existing - // state file cannot be handed over to it yet. See dstate.DeploymentState.Open. + // RecordDeploymentHistory opts into the deployment metadata service (DMS). + // Only for bundles with no deployed resources yet; DMS becomes the source of truth for state. RecordDeploymentHistory bool `json:"record_deployment_history,omitempty"` } diff --git a/bundle/config/mutator/initialize_deployment_history.go b/bundle/config/mutator/initialize_deployment_history.go index b90800e4e70..07d576c01f2 100644 --- a/bundle/config/mutator/initialize_deployment_history.go +++ b/bundle/config/mutator/initialize_deployment_history.go @@ -13,13 +13,8 @@ import ( type initializeDeploymentHistory struct{} -// InitializeDeploymentHistory populates bundle.deployment.history with the -// deployment recorded by the deployment metadata service, for the output of the -// 'bundle summary' command. -// -// NOTE: this makes extra API calls, so like InitializeURLs it should only be used -// when the fields are needed. It is a no-op unless the bundle records deployment -// history. +// InitializeDeploymentHistory populates bundle.deployment.history from DMS for 'bundle summary'. +// Makes extra API calls; only use when needed. No-op unless recording is on. func InitializeDeploymentHistory() bundle.Mutator { return &initializeDeploymentHistory{} } diff --git a/bundle/config/validate/validate_record_deployment_history.go b/bundle/config/validate/validate_record_deployment_history.go index 4f0137fae0f..a9ee61af1c5 100644 --- a/bundle/config/validate/validate_record_deployment_history.go +++ b/bundle/config/validate/validate_record_deployment_history.go @@ -21,16 +21,9 @@ func (v *validateRecordDeploymentHistory) Name() string { return "validate:validate_record_deployment_history" } -// Apply rejects experimental.record_deployment_history. -// -// Recording deployment history is implemented end to end, but the service side is not -// ready for users: the deployment metadata service is only deployed to dev and staging, -// and reading state back needs the workspace APIs to expose the deployment's tree node, -// which is still behind a flag. Enabling this today also makes DMS the source of truth -// for resource state, so a bundle that turns it on cannot be turned back. -// -// DATABRICKS_BUNDLE_FORCE_ALLOW_RECORD_DEPLOYMENT_HISTORY force allows it for the CLI's -// own tests and for DMS development. +// Apply rejects experimental.record_deployment_history. The feature is complete +// but not yet exposed: DMS is dev/staging only, and turning it on makes DMS the +// source of truth for state (irreversible). Force-allow via DATABRICKS_BUNDLE_FORCE_ALLOW_RECORD_DEPLOYMENT_HISTORY for CLI tests. func (v *validateRecordDeploymentHistory) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { if b.Config.Experimental == nil || !b.Config.Experimental.RecordDeploymentHistory { return nil diff --git a/bundle/deploy/metadata/annotate_deployment_version.go b/bundle/deploy/metadata/annotate_deployment_version.go index 46e0ab7ed41..c08f7b52345 100644 --- a/bundle/deploy/metadata/annotate_deployment_version.go +++ b/bundle/deploy/metadata/annotate_deployment_version.go @@ -12,12 +12,9 @@ type annotateDeployment struct { deploymentID string } -// AnnotateDeployment stamps the DMS deployment onto every job and pipeline, so a -// resource in the workspace points back at the deployment that produced it (which is -// how lineage resolves a job to its bundle). -// -// It runs before the plan is computed, since a resource whose deployment is unset -// locally but set in the workspace would otherwise show as drift. +// AnnotateDeployment stamps the DMS deployment onto every job and pipeline, so a workspace +// resource points back at the deployment that produced it - how lineage resolves a job to its +// bundle. It runs before the plan, or the stamp would show as drift against local config. func AnnotateDeployment(deploymentID string) bundle.Mutator { return &annotateDeployment{deploymentID: deploymentID} } @@ -43,9 +40,8 @@ type annotateDeploymentVersion struct { version int64 } -// AnnotateDeploymentVersion stamps the DMS version onto every job and pipeline. It -// is separate from AnnotateDeployment because the version only exists once -// CreateVersion has claimed one, which happens during deploy. +// AnnotateDeploymentVersion stamps the DMS version onto every job and pipeline. +// Separate from AnnotateDeployment because version only exists after CreateVersion runs. func AnnotateDeploymentVersion(version int64) bundle.Mutator { return &annotateDeploymentVersion{version: version} } diff --git a/bundle/direct/apply.go b/bundle/direct/apply.go index b320f8c5850..37ba1b8191a 100644 --- a/bundle/direct/apply.go +++ b/bundle/direct/apply.go @@ -116,14 +116,8 @@ func (d *DeploymentUnit) Recreate(ctx context.Context, db *dstate.DeploymentStat log.Warnf(ctx, "Treating %s id=%s as already deleted despite delete error: %s", d.ResourceKey, oldID, err) } - // Drop the state entry so a subsequent failure of Create or WaitAfterDelete - // leaves no malformed (empty-ID) entry behind. The next plan will see "no - // state" and retry as Create. - // - // Recorded as a recreate, not a delete: if the create below fails, this is the - // operation DMS is left with, and it says the resource is mid-recreate rather - // than deliberately removed. In-progress for the same reason - the create that - // follows updates the same operation to succeeded. + // Drop state so failure doesn't leave a malformed empty-ID entry. Recorded as + // recreate not delete: if create below fails, DMS sees mid-recreate not removed. err = db.DeleteState(ctx, d.ResourceKey, dstate.OperationInfo{Action: deployplan.Recreate, InProgress: true}) if err != nil { return fmt.Errorf("deleting state: %w", err) @@ -163,13 +157,9 @@ func (d *DeploymentUnit) Update(ctx context.Context, db *dstate.DeploymentState, } if empty { - // The update emptied the resource out (e.g. all grants revoked). Keeping an entry - // would report the node as tracked-and-unchanged forever, while a fresh deploy of - // the same config plans no node at all; drop it so the two agree. - // - // Recorded as a delete, not the update that caused it: the resource is no longer - // tracked, and DMS drops it from the deployment only for a delete. Recording an - // update would leave it listed with no state, which the next plan cannot read. + // The update emptied the resource (e.g. all grants revoked), so drop the entry: a + // fresh deploy of the same config would plan no node at all. Recorded as a delete, + // which is what it did to the state, rather than the update that caused it. err = db.DeleteState(ctx, d.ResourceKey, dstate.OperationInfo{Action: deployplan.Delete}) if err != nil { return fmt.Errorf("deleting state id=%s: %w", id, err) diff --git a/bundle/direct/bundle_apply.go b/bundle/direct/bundle_apply.go index c16f728cdd2..c30f2fb48e3 100644 --- a/bundle/direct/bundle_apply.go +++ b/bundle/direct/bundle_apply.go @@ -35,16 +35,13 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa return } - // Operations are recorded with DMS from one background goroutine so a resource's - // deploy is not held up by the CreateOperation round trip. It is drained below, - // once every apply worker has finished recording. - // - // The state DB records through it, so every state write becomes an operation and - // DMS mirrors the WAL. + // The state DB records every write through this sink, so DMS mirrors the WAL. Uploads run + // on one background goroutine, off the apply path, and are drained below once every + // worker has finished recording. opSink := newOperationSink(ctx, b.OpRec) if opSink != nil { - // Assigned only when non-nil: a nil *operationSink in an interface is not a - // nil interface, so the state DB's nil check would not see it. + // Only when non-nil: a nil *operationSink in an interface is not a nil interface, so + // the state DB's nil check would not see it. b.StateDB.SetOperationSink(opSink) } @@ -137,8 +134,8 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa // TODO: redo calcDiff to downgrade planned action if possible (?) // - // Success is recorded by the state writes inside Deploy, so a resource that - // writes state more than once (a recreate) reports each step. + // Success is recorded by the state writes inside Deploy, so a recreate reports + // each of its steps. err = d.Deploy(ctx, &b.StateDB, sv.Value, action, entry) if err != nil { // Both are empty for a create that never got an ID, which is what the diff --git a/bundle/direct/dstate/dms.go b/bundle/direct/dstate/dms.go index 9b26757f06f..04674cb9b8b 100644 --- a/bundle/direct/dstate/dms.go +++ b/bundle/direct/dstate/dms.go @@ -9,49 +9,41 @@ import ( "github.com/databricks/databricks-sdk-go/service/bundledeployments" ) -// ResourceKeyPrefix is what a state key carries and a DMS resource key does not: -// state calls a job "resources.jobs.foo", DMS calls it "jobs.foo". Stripped on the -// way out and re-added on the way back, so both sides must use this one constant or -// operations silently land under keys nothing reads. +// ResourceKeyPrefix is what a state key carries and a DMS resource key does not: state calls +// a job "resources.jobs.foo", DMS calls it "jobs.foo". Both sides must use this one constant +// or operations land under keys nothing reads. const ResourceKeyPrefix = "resources." -// RecordedState is what the CLI serializes into the DMS Operation.State field. It -// wraps the config rather than being it, so depends_on survives the round trip: DMS -// has no field for dependency edges, and they cannot be recomputed once references -// are resolved to literals. Nesting them in the config would collide with resource -// fields of the same name (e.g. jobs.Task.depends_on). +// RecordedState is what the CLI serializes into the DMS Operation.State field. It wraps the +// config so depends_on survives the round trip: DMS has no field for dependency edges, and +// nesting them in the config would collide with resource fields of the same name. type RecordedState struct { State json.RawMessage `json:"state"` DependsOn []deployplan.DependsOnEntry `json:"depends_on,omitempty"` } -// OperationInfo is what a state write reports to the deployment metadata service. -// The caller describes the write; the sink does not infer it from the state being nil. +// OperationInfo is what a state write reports to DMS. The caller describes the write; the +// sink does not infer it from the state being nil. type OperationInfo struct { // Action is the operation DMS records for this write. Action deployplan.ActionType - // InProgress marks a write that is half of a larger change, so an interrupted - // deploy does not leave the resource described as finished. The service keeps one - // operation per resource per version, so the second write updates this same - // operation to succeeded. Only a recreate's delete sets it. + // InProgress marks a write that is half of a larger change, so an interrupted deploy does + // not leave the resource described as finished. Only a recreate's delete sets it; the + // create that follows updates the same operation to succeeded. InProgress bool } -// OperationSink records one resource operation with the deployment metadata service. -// SaveState and DeleteState call it for every state write, so what DMS holds mirrors -// the WAL - including the intermediate writes of a recreate. -// -// It does not return an error: the upload happens on a background worker, and the -// deploy learns about a failure when the queue is drained. +// OperationSink records one resource operation with DMS. Every state write calls it, so what +// DMS holds mirrors the WAL. It returns no error: the upload runs in the background, and the +// deploy learns of a failure when the queue is drained. type OperationSink interface { RecordOperation(ctx context.Context, resourceKey string, info OperationInfo, resourceID string, state json.RawMessage) } -// readDMSState replaces the file-derived resource state with the state recorded in -// DMS. Recording is only enabled for net-new deployments, so once a deployment -// exists DMS owns its resource set outright - an empty set means a successful deploy -// of nothing, not missing data. The caller holds db.mu. +// readDMSState replaces the file-derived resource state with what DMS recorded. Recording is +// only enabled for net-new deployments, so DMS owns the resource set outright: an empty set +// means a successful deploy of nothing, not missing data. The caller holds db.mu. func (db *DeploymentState) readDMSState(ctx context.Context, src *DMSSource) error { resources, err := fetchDeploymentResources(ctx, src.Client, src.DeploymentID) if err != nil { diff --git a/bundle/direct/dstate/state.go b/bundle/direct/dstate/state.go index 05cdee67271..7d11cc1a6a0 100644 --- a/bundle/direct/dstate/state.go +++ b/bundle/direct/dstate/state.go @@ -147,10 +147,9 @@ func (db *DeploymentState) SaveState(ctx context.Context, key, newID string, sta return err } - // Recorded after the WAL write, so DMS never reports a state the deploy failed to - // persist locally, and outside the lock because recording applies backpressure: - // it waits when the service is behind, and waiting under db.mu would hold up every - // other resource's write rather than just this one. + // Recorded after the WAL write, so DMS never reports state the deploy failed to persist, + // and outside the lock because recording waits when the service is behind - waiting under + // db.mu would hold up every other resource's write. if sink != nil { sink.RecordOperation(ctx, key, info, newID, recorded) } @@ -302,10 +301,9 @@ type DMSSource struct { DeploymentID string } -// Open reads the deployment state from disk (and recovers the WAL when -// withRecovery is set). With a non-nil dmsSource, resources come from DMS rather -// than the file. Lineage and serial always come from the file, since that is -// what the write path increments. +// Open reads the deployment state from disk, recovering the WAL when withRecovery is set. +// With a non-nil dmsSource the resources come from DMS instead; lineage and serial still +// come from the file, since that is what the write path increments. func (db *DeploymentState) Open(ctx context.Context, path string, withRecovery WithRecovery, withWrite WithWrite, dmsSource *DMSSource) error { db.mu.Lock() defer db.mu.Unlock() @@ -355,15 +353,9 @@ func (db *DeploymentState) Open(ctx context.Context, path string, withRecovery W } if dmsSource != nil { - // Only bundles that start out empty can be recorded. Once DMS owns a - // deployment it is authoritative for the whole resource set (see - // readDMSState), so pre-existing resources it never saw would look absent and - // get created a second time. - // - // TODO(DMS): allow this by upgrading the state in place, writing it at - // featureStateVersion with a feature flag plus a tombstone per resource so an - // older CLI refuses the state instead of deploying against resources it - // cannot see. + // Only empty bundles can be recorded. Once DMS owns the deployment, pre-existing + // resources it never saw would be created again. TODO: support migration via state + // upgrade with feature flag and per-resource tombstones. if dmsSource.DeploymentID == "" && len(db.Data.State) > 0 { // The remedy is ordered deliberately: this error also blocks destroy, so the // setting has to come out first or there is no way to tear the bundle down. diff --git a/bundle/direct/dstate/state_test.go b/bundle/direct/dstate/state_test.go index ef51b1ae312..5ebced6829e 100644 --- a/bundle/direct/dstate/state_test.go +++ b/bundle/direct/dstate/state_test.go @@ -40,10 +40,9 @@ func TestStateWritesRecordOperations(t *testing.T) { want []string }{ { - // The service keeps one operation per resource per version, so the drop - // opens it (no state: the old resource is gone and the new one does not - // exist yet) and the save completes it. A deploy that stops in between - // leaves the resource described as mid-recreate. + // The service keeps one operation per resource per version, so the drop opens it + // with no state and the save completes it. A deploy that stops in between leaves + // the resource recorded as mid-recreate. name: "recreate reports both of its writes", write: func(t *testing.T, db *DeploymentState) { require.NoError(t, db.SaveState(t.Context(), "jobs.my_job", "123", map[string]string{"key": "old"}, nil, OperationInfo{Action: deployplan.Create})) diff --git a/bundle/direct/opclient.go b/bundle/direct/opclient.go index 17bfaee3867..d9d1e53df4c 100644 --- a/bundle/direct/opclient.go +++ b/bundle/direct/opclient.go @@ -11,27 +11,18 @@ import ( "github.com/databricks/databricks-sdk-go/service/bundledeployments" ) -// The CLI calls the operations API directly rather than through the generated -// client because the SDK cannot read the response: it types sequence_id as an -// int64, while the service sends it as a JSON string (proto3 encodes 64-bit ints -// that way), so unmarshalling a CreateOperation response fails with -// "invalid character '1' after top-level value". The write itself succeeds - the -// status is 200 - so only the response parse is affected. -// -// TODO(DMS): this whole file goes away once the SDK types sequence_id as a string. -// The fix belongs in the OpenAPI spec the SDK is generated from, not here; until then -// every other DMS call still goes through the SDK, so keep the bypass to operations. +// These calls bypass the SDK because it cannot read the response: it types sequence_id as +// an int64 while the service sends a JSON string, so a CreateOperation response fails to +// unmarshal. TODO(DMS): drop this file once the OpenAPI spec types the field as a string. // operationResponse is the part of an operation response the CLI reads back. type operationResponse struct { - // SequenceId is the concurrency token for the next update of this operation. - // Typed as a string because that is what the service sends; see above. + // SequenceId is the concurrency token for the next update, typed as the service sends it. SequenceId string `json:"sequence_id,omitempty"` } -// updateOperationRequest carries the fields a later write for the same resource -// changes. action_type and resource_key are omitted: the service fixes them when -// the operation is created and ignores them here. +// updateOperationRequest carries the fields a later write for the same resource changes. +// action_type and resource_key are left out: the service fixes them at creation. type updateOperationRequest struct { State string `json:"state,omitempty"` ErrorMessage string `json:"error_message,omitempty"` diff --git a/bundle/direct/oprecorder.go b/bundle/direct/oprecorder.go index fe410a3db65..16c77acccf7 100644 --- a/bundle/direct/oprecorder.go +++ b/bundle/direct/oprecorder.go @@ -14,68 +14,48 @@ import ( "github.com/databricks/databricks-sdk-go/service/bundledeployments" ) -// maxOperationStateSize is the largest serialized state DMS accepts per -// operation. Uploading more is rejected server-side, so fail early with a -// message that names the resource. +// maxOperationStateSize is the largest serialized state DMS accepts per operation. More is +// rejected server-side, so fail early with a message naming the resource. const maxOperationStateSize = 64 * 1024 -// maxOperationErrorMessageSize is the largest error message DMS accepts per -// operation. A longer message is truncated rather than rejected, so a failing -// resource is still recorded with its error instead of the recording itself -// failing and masking the error we are trying to report. +// maxOperationErrorMessageSize is the largest error message DMS accepts. A longer one is +// truncated rather than rejected, so recording cannot fail and hide the error it reports. const maxOperationErrorMessageSize = 16 * 1024 -// operationStatusInProgress marks an operation whose writes are not finished; see -// dstate.OperationInfo.InProgress for when a write asks for it. -// -// Declared here rather than used from the SDK: the enum value is generated from the -// OpenAPI spec, which trails the service proto (databricks-eng/universe#2394529). +// operationStatusInProgress marks an operation whose writes are not finished. Not taken from +// the SDK: the enum is generated from the OpenAPI spec, which trails the service proto +// (databricks-eng/universe#2394529). const operationStatusInProgress bundledeployments.OperationStatus = "OPERATION_STATUS_IN_PROGRESS" -// recordedOperation is an applied resource operation, serialized and waiting to be -// uploaded to the deployment metadata service (DMS). -// -// The payload is built on the apply worker rather than in the uploader so the sink -// does not hold on to the live resource struct, and so a malformed state fails the -// resource that produced it instead of the drain at the end of apply. +// recordedOperation is an applied resource operation waiting to be uploaded. It is built on +// the apply worker, not in the uploader, so a malformed state fails the resource that +// produced it rather than the drain at the end of apply. type recordedOperation struct { action bundledeployments.OperationActionType resourceID string status bundledeployments.OperationStatus - // errorMessage is why the operation failed. It is set only when status is - // failed, which the service enforces. + // errorMessage is set only when status is failed, which the service enforces. errorMessage string - // state is the serialized local config after the operation. It is nil for a delete, - // where the resource no longer exists, and for a failure it is the state from before - // the deploy - see newFailedOperation for when that reaches the service. + // state is the serialized config after the operation: nil for a delete, and the + // pre-deploy state for a failure (see newFailedOperation). state json.RawMessage - // updateFields is the update mask to send if this operation updates one the service - // already has. The service takes it literally: a field named here is written, a field - // left out keeps the value it had. + // updateFields is the mask to send when updating an operation the service already has. + // It is taken literally: a field named here is written, one left out keeps its value. updateFields []string } -// The fields an UpdateOperation may change. Any other path is rejected with -// INVALID_PARAMETER_VALUE, so this is the full universe a mask can name. -const ( - fieldState = "state" - fieldErrorMessage = "error_message" - fieldResourceID = "resource_id" - fieldStatus = "status" -) - -// describesResource is the update mask for an operation that says how the resource -// looks: everything an update is allowed to change. Its order is the canonical one. -var describesResource = []string{fieldState, fieldErrorMessage, fieldResourceID, fieldStatus} +// describesResource is the update mask for an operation that says how the resource looks: +// every field an update may change. Any other path is rejected with INVALID_PARAMETER_VALUE, +// so this doubles as the canonical field list and order. +var describesResource = []string{"state", "error_message", "resource_id", "status"} -// failedKeepingState is the update mask for a failure updating an operation this version -// already recorded: mark it failed and leave state alone. That is right either way - state -// means the resource is as it was written, and no state means a delete went through and +// failedKeepingState is the update mask for a failure: mark it failed and leave state alone. +// State means the resource is as it was written; no state means a delete went through and // nothing replaced it, so the resource really is gone and the deployment should say so. -var failedKeepingState = []string{fieldErrorMessage, fieldStatus} +var failedKeepingState = []string{"error_message", "status"} // newStateOperation describes a state write for upload. state is the serialized // RecordedState envelope the state DB just persisted, and nil for a delete, where @@ -104,24 +84,16 @@ func newStateOperation(info dstate.OperationInfo, resourceID string, state json. }, nil } -// newFailedOperation records an operation that did not apply, so the deployment -// history says why a resource failed rather than just omitting it. -// -// priorState is the resource's state from before the deploy, and only reaches the service -// when this failure is the first thing recorded for the resource in this version: nothing -// touched the resource, so it is still there and the deployment has to keep describing it -// or the next plan creates a second one. Once this version has recorded an operation, that -// operation says where the resource stands and the failure leaves its state alone - see -// failedKeepingState. It is nil for a create, which has no prior state and no resource to -// describe, which is also why the resourceID may be empty for CREATE and RECREATE. +// newFailedOperation records an operation that did not apply, so the history says why a +// resource failed rather than omitting it. priorState (nil for a create, as resourceID may +// also be) reaches the service only when nothing else was recorded for the resource yet. func newFailedOperation(action deployplan.ActionType, resourceID string, priorState json.RawMessage, cause error) (recordedOperation, error) { actionType, err := deployActionToSDK(action) if err != nil { return recordedOperation{}, err } - // A guard: this state came back from the state DB, so it was within the limit when - // it was written. + // A guard: the state DB accepted this state, so it was within the limit when written. if len(priorState) > maxOperationStateSize { return recordedOperation{}, fmt.Errorf("serialized state is %d bytes, which exceeds the %d byte limit for recording deployment history", len(priorState), maxOperationStateSize) } @@ -141,14 +113,9 @@ func newFailedOperation(action deployplan.ActionType, resourceID string, priorSt }, nil } -// priorRecord returns the resource's id and state from before this deploy, in the -// same envelope form the success path uploads, or empty values when the resource has -// no prior record (a create). -// -// Both come from the same pre-deploy entry because the service requires an id -// alongside state: state describes a resource that exists, so it needs the id to say -// which one. Reading the id from live state instead would return "" for a failed -// recreate, whose delete step already dropped it, and the mismatch is rejected. +// priorRecord returns the resource's id and state from before this deploy, in the envelope +// form the success path uploads, or empty values when there is no prior record. Both come +// from one entry: the service rejects state without an id. func priorRecord(db *dstate.DeploymentState, resourceKey string) (string, json.RawMessage) { entry, ok := db.GetResourceEntry(resourceKey) if !ok || len(entry.State) == 0 { @@ -178,11 +145,9 @@ type operationRecorder struct { // mu guards sequenceIDs. mu sync.Mutex - // sequenceIDs holds the sequence id the service returned per resource key, which is - // both how a resource already recorded in this version is recognised and the - // concurrency precondition for updating it. The service names operations - // "operations/{resource_key}", so it keeps one per resource per version: the second - // write for a resource has to update that operation. + // sequenceIDs holds the sequence id the service returned per resource key: both how an + // already-recorded resource is recognised and the precondition for updating it. The + // service keeps one operation per resource per version, so a second write must update it. sequenceIDs map[string]string } diff --git a/bundle/direct/oprecorder_test.go b/bundle/direct/oprecorder_test.go index 023a08145a0..e50ac5ad75d 100644 --- a/bundle/direct/oprecorder_test.go +++ b/bundle/direct/oprecorder_test.go @@ -94,10 +94,8 @@ func TestOperationRecorderUpdatesSecondWriteForSameResource(t *testing.T) { } func TestOperationRecorderFailureAfterAStateWriteKeepsTheState(t *testing.T) { - // A create whose WaitAfterCreate fails has already written state for a resource - // that exists remotely. Updating the operation must only mark it failed: sending - // the failure's own empty state and id would clear both, and a resource with no - // state is dropped from the deployment, so the next plan would re-create it. + // The create wrote state for a resource that exists, then the wait failed. The update + // only marks it failed: sending empty state would drop the resource from the deployment. f := &fakeOpClient{sequence: "3"} r := newOperationRecorder(f, "dep-1", 2) @@ -121,11 +119,8 @@ func TestOperationRecorderFailureAfterAStateWriteKeepsTheState(t *testing.T) { } func TestOperationRecorderFailedRecreateKeepsTheResourceGone(t *testing.T) { - // A recreate records its in-progress delete, which has no state, and then fails before - // the create writes any. The failure must not fill that gap with the pre-deploy state: - // the delete went through and the create did not, so the resource really is gone, and - // an operation with no state is how the deployment says so. The next plan creates it, - // which is what needs to happen. + // The recreate's delete is recorded with no state, and then the create fails. The failure + // must not fill that gap with the pre-deploy state: the resource really is gone. f := &fakeOpClient{sequence: "2"} r := newOperationRecorder(f, "dep-1", 2) @@ -145,10 +140,8 @@ func TestOperationRecorderFailedRecreateKeepsTheResourceGone(t *testing.T) { } func TestOperationRecorderFailureCarryingALaterWriteSendsIt(t *testing.T) { - // Two writes for one resource: the first is uploaded, the second is still waiting - // when the resource fails, so the failure takes it over (see coalesce). The local - // state holds that second write, so the update has to name state - leaving it out - // would keep the first write's state, and the next plan reads DMS. + // Two writes, the first uploaded and the second still waiting when the resource failed, + // so the failure took it over. The update must name state or the first write's stands. f := &fakeOpClient{sequence: "4"} r := newOperationRecorder(f, "dep-1", 2) @@ -169,10 +162,8 @@ func TestOperationRecorderFailureCarryingALaterWriteSendsIt(t *testing.T) { } func TestOperationRecorderFailureBeforeAnyWriteCarriesPriorState(t *testing.T) { - // Nothing was recorded for the resource in this version, so the failure creates the - // operation and carries the prior state: nothing touched the resource, so it is still - // there, and an operation without state would drop it from the deployment and have the - // next plan create a second one. + // Nothing was recorded yet, so the failure creates the operation and carries the prior + // state. Without it the resource is dropped and the next plan creates a second one. f := &fakeOpClient{sequence: "1"} r := newOperationRecorder(f, "dep-1", 2) diff --git a/bundle/direct/opsink.go b/bundle/direct/opsink.go index b3e72be2e0b..540f276f1c6 100644 --- a/bundle/direct/opsink.go +++ b/bundle/direct/opsink.go @@ -112,31 +112,27 @@ func (s *operationSink) record(resourceKey string, op recordedOperation) { } } -// coalesce merges an operation with the one that superseded it while still waiting. Each -// field comes from whichever operation claimed it in its mask, and the newer one wins when -// both did; the merged mask is the union, so neither operation's fields get dropped. -// -// What a given operation claims is decided where it is built, not here - see -// describesResource and failedKeepingState. +// coalesce merges an operation with the one that superseded it while still waiting. Each field +// comes from whichever operation claimed it in its mask, newer winning when both did, and the +// mask is the union. What an operation claims is decided where it is built, not here. func coalesce(older, newer recordedOperation) recordedOperation { merged := older - - // action_type is fixed when the operation is created, so no mask names it. An update - // that empties a resource records its write as a delete (see DeploymentUnit.Update), so - // the two can disagree; report what the plan set out to do. - merged.action = newer.action merged.updateFields = unionFields(older.updateFields, newer.updateFields) - if slices.Contains(newer.updateFields, fieldState) { + if slices.Contains(newer.updateFields, "state") { + // Claiming state means this operation last acted on the resource, so its action_type is + // the one to record. One that claims none only reports an outcome, and the service + // would keep the earlier action anyway - action_type is fixed once the operation exists. merged.state = newer.state + merged.action = newer.action } - if slices.Contains(newer.updateFields, fieldResourceID) { + if slices.Contains(newer.updateFields, "resource_id") { merged.resourceID = newer.resourceID } - if slices.Contains(newer.updateFields, fieldErrorMessage) { + if slices.Contains(newer.updateFields, "error_message") { merged.errorMessage = newer.errorMessage } - if slices.Contains(newer.updateFields, fieldStatus) { + if slices.Contains(newer.updateFields, "status") { merged.status = newer.status } diff --git a/bundle/direct/opsink_test.go b/bundle/direct/opsink_test.go index c6dfbd5c36a..d7a6a7c79f3 100644 --- a/bundle/direct/opsink_test.go +++ b/bundle/direct/opsink_test.go @@ -152,10 +152,8 @@ func TestOperationSinkCoalescesWritesBehindAnUpload(t *testing.T) { } func TestOperationSinkCoalescedFailureKeepsTheStateItReplaces(t *testing.T) { - // A create writes state and then fails waiting for the resource to come up. If - // the failure catches the write before it is uploaded, it must not replace that - // state with its own emptiness: a resource recorded without state is dropped from - // the deployment, so the next plan would create it a second time. + // The create writes state and then fails before the upload. The failure must not replace + // that state with its own emptiness, which would drop the resource from the deployment. f := &fakeUploader{block: make(chan struct{}), started: make(chan string, 2)} s := newOperationSink(t.Context(), f) @@ -184,10 +182,8 @@ func TestOperationSinkCoalescedFailureKeepsTheStateItReplaces(t *testing.T) { } func TestOperationSinkCoalescedFailureAfterADeleteKeepsTheResourceGone(t *testing.T) { - // A recreate's delete step writes no state, and the create that follows fails. Whether - // that delete was uploaded or is still waiting must not change what DMS ends up with: - // the resource was deleted, so the failure carries the delete's absent state rather - // than the pre-deploy state, and the resource stays gone. + // The recreate's delete writes no state. When the create then fails, the failure takes + // that absent state rather than the pre-deploy one, so the resource stays gone. f := &fakeUploader{block: make(chan struct{}), started: make(chan string, 2)} s := newOperationSink(t.Context(), f) @@ -210,11 +206,9 @@ func TestOperationSinkCoalescedFailureAfterADeleteKeepsTheResourceGone(t *testin } func TestOperationSinkCoalescedFailureDoesNotRevertToPriorState(t *testing.T) { - // An update that succeeded and then failed waiting carries the pre-deploy state, - // which the write it supersedes has already moved past. Sending that would record - // the resource as it was before the deploy, and the next plan would read it back as - // current. Once the write is uploaded the wire mask keeps it out; before that, this - // does. + // An update that succeeded and then failed waiting carries the pre-deploy state, which the + // write it supersedes has moved past. Sending it would record the resource as it was before + // the deploy, and the next plan would read that back as current. f := &fakeUploader{block: make(chan struct{}), started: make(chan string, 2)} s := newOperationSink(t.Context(), f) @@ -262,10 +256,9 @@ func TestOperationSinkCoalescedDeleteStillClearsState(t *testing.T) { } func TestCoalesceLetsAWriteSupersedeAFailure(t *testing.T) { - // A failure is not the last word. If a resource were retried and then wrote state, that - // write describes the resource and has to win whole - its state, its id, and its mask, - // which names error_message so the recorded failure is cleared. The service rejects a - // succeeded operation that still carries an error. + // A failure is not the last word. A retry that writes state wins whole - state, id and + // mask - and the mask names error_message so the recorded failure is cleared. The service + // rejects a succeeded operation that still carries an error. failed, err := newFailedOperation(deployplan.Update, "id-old", envelope(t, "before"), errors.New("boom")) require.NoError(t, err) retried, err := newStateOperation(dstate.OperationInfo{Action: deployplan.Update}, "id-new", envelope(t, "after the retry")) @@ -281,9 +274,9 @@ func TestCoalesceLetsAWriteSupersedeAFailure(t *testing.T) { } func TestCoalesceKeepsTheWritesStateAndMask(t *testing.T) { - // The other direction: a failure contributes only its outcome, so the write's state, id - // and mask survive and the failure's own pre-deploy state is dropped as the older of the - // two. Action comes from the failure, which reports what the plan set out to do. + // A failure claims only status and error_message, so the write's state, id and mask + // survive. The action is the write's too: an update that empties a resource records its + // write as a delete, and the service keeps whichever action created the operation. write, err := newStateOperation(dstate.OperationInfo{Action: deployplan.Delete}, "id-new", nil) require.NoError(t, err) failed, err := newFailedOperation(deployplan.Update, "id-old", envelope(t, "before"), errors.New("boom")) @@ -296,7 +289,7 @@ func TestCoalesceKeepsTheWritesStateAndMask(t *testing.T) { assert.Equal(t, "id-new", got.resourceID) assert.Nil(t, got.state) assert.Equal(t, describesResource, got.updateFields) - assert.Equal(t, bundledeployments.OperationActionTypeOperationActionTypeUpdate, got.action) + assert.Equal(t, bundledeployments.OperationActionTypeOperationActionTypeDelete, got.action) } func TestOperationSinkRecordDuringUploadIsStillUploaded(t *testing.T) { @@ -322,11 +315,9 @@ func TestOperationSinkRecordDuringUploadIsStillUploaded(t *testing.T) { } func TestOperationSinkRecordWaitsWhenTheQueueIsFull(t *testing.T) { - // Recording holds the deploy back rather than letting it run arbitrarily far ahead - // of what the service has been told: once every slot holds a resource, the next - // write waits for the uploader. - // started is buffered for every upload: nothing reads it after the first, and an - // uploader blocked sending to it would never drain the queue. + // Recording holds the deploy back once every slot is taken. started is buffered for every + // upload: nothing reads it after the first, and an uploader blocked sending to it would + // never drain the queue. f := &fakeUploader{block: make(chan struct{}), started: make(chan string, operationSinkQueueSize+4)} s := newOperationSink(t.Context(), f) diff --git a/bundle/direct/pkg.go b/bundle/direct/pkg.go index 03864af5da2..b7a7a1a4693 100644 --- a/bundle/direct/pkg.go +++ b/bundle/direct/pkg.go @@ -45,11 +45,9 @@ type DeploymentBundle struct { RemoteStateCache sync.Map StateCache structvar.Cache - // OpRec uploads each applied resource operation to the deployment metadata - // service (DMS). It is nil unless the bundle opts into recording deployment - // history, in which case the phases package sets it after CreateVersion. - // Apply queues the operations and drains them before returning, so the - // uploads do not block the resources being deployed. + // OpRec uploads applied operations to DMS. Nil unless the bundle records deployment + // history, in which case the deploy phase sets it once CreateVersion has claimed a + // version. Apply drains it before returning. OpRec operationUploader } diff --git a/bundle/env/dms.go b/bundle/env/dms.go index 435a9ac3bcc..783dd9a7851 100644 --- a/bundle/env/dms.go +++ b/bundle/env/dms.go @@ -2,14 +2,9 @@ package env import "context" -// RecordDeploymentHistoryVariable names the environment variable that turns on -// deployment history recording without setting experimental.record_deployment_history -// in the bundle. It exists for the CLI's own acceptance tests, which run the whole -// bundle suite with recording enabled: setting it here beats adding the field to every -// databricks.yml. -// -// Like ForceAllowRecordDeploymentHistoryVariable it is deliberately undocumented; see -// validate.ValidateRecordDeploymentHistory for why the feature is still gated off. +// RecordDeploymentHistoryVariable enables recording without setting the config field. +// Exists for CLI acceptance tests so the whole bundle suite runs with recording on. +// Deliberately undocumented; see validate.ValidateRecordDeploymentHistory. const RecordDeploymentHistoryVariable = "DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY" // recordDeploymentHistoryEnv reports whether the environment turns on deployment @@ -20,10 +15,8 @@ func recordDeploymentHistoryEnv(ctx context.Context) bool { return value == "true" } -// RecordsDeploymentHistory reports whether this deploy records deployment history, -// from either the bundle setting or RecordDeploymentHistoryVariable. It is the single -// predicate the recording code paths branch on, so the env var and the config field -// cannot drift. +// RecordsDeploymentHistory reports whether recording is on, from config or env var. +// Single predicate for all recording code paths; keeps them in sync. func RecordsDeploymentHistory(ctx context.Context, configured bool) bool { return configured || recordDeploymentHistoryEnv(ctx) } diff --git a/bundle/env/force_allow_record_deployment_history.go b/bundle/env/force_allow_record_deployment_history.go index 297ccb6f6e3..1b675a23e58 100644 --- a/bundle/env/force_allow_record_deployment_history.go +++ b/bundle/env/force_allow_record_deployment_history.go @@ -2,11 +2,8 @@ package env import "context" -// ForceAllowRecordDeploymentHistoryVariable names the environment variable that force -// allows experimental.record_deployment_history. It is deliberately undocumented: the -// feature is complete but cannot be exposed to users yet (see -// validate.ValidateRecordDeploymentHistory for why), and this variable exists so the -// CLI's own tests and the developers working on DMS can exercise the code path meanwhile. +// ForceAllowRecordDeploymentHistoryVariable force-allows the recording feature for CLI tests and DMS development. +// Deliberately undocumented; see validate.ValidateRecordDeploymentHistory. const ForceAllowRecordDeploymentHistoryVariable = "DATABRICKS_BUNDLE_FORCE_ALLOW_RECORD_DEPLOYMENT_HISTORY" // ForceAllowRecordDeploymentHistory reports whether the environment force allows diff --git a/bundle/phases/deploy.go b/bundle/phases/deploy.go index 625a16fc8c3..4638f0ff81d 100644 --- a/bundle/phases/deploy.go +++ b/bundle/phases/deploy.go @@ -128,11 +128,8 @@ func deployCore(ctx context.Context, b *bundle.Bundle, plan *deployplan.Plan, st } } -// logFileSummary reports what the file sync did. Separate from the resource summary -// because a deploy that only changes business logic (a .py or .sql file) leaves every -// resource unchanged, so without this line its summary is all zeros and looks like a -// no-op. Called on the failure paths too: the files were uploaded before whatever -// failed afterwards, so the count is accurate even then. +// logFileSummary reports what the file sync did. Separate because a pure-code deploy +// leaves all resources unchanged, so would appear as a no-op without this line. func logFileSummary(ctx context.Context, b *bundle.Bundle) { if b.Quiet >= bundle.QuietAll { return @@ -206,10 +203,8 @@ func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHand // lock is acquired here // - // Set up DMS recording of this deployment as a version. The version itself is - // created once the deploy is approved. CompleteVersion is deferred before - // lock.Release so it runs while the lock is still held (defers run - // last-in-first-out), and is a no-op until CreateVersion has run. + // The version is created only after approval; CompleteVersion is deferred before + // lock.Release and no-ops until then. recorder, err := newDeploymentRecorder(ctx, b, stateEngine, dms.VersionTypeDeploy) if err != nil { logdiag.LogError(ctx, err) @@ -281,13 +276,8 @@ func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHand return } - // Settle the deployment and the version number it will use before planning: the - // plan snapshots the resource config, so both have to be stamped on before it is - // computed or the applied resources would not carry them. The version itself is - // created after approval. - // - // This cannot move earlier: on a first deploy the deployment is registered under - // the state directory, which the upload above is what creates. + // Settle deployment and version before planning. Plan snapshots the config, so + // both must be stamped before it is computed. Version itself is created after approval. if err := recorder.PrepareDeployment(ctx); err != nil { logdiag.LogError(ctx, err) return @@ -343,12 +333,9 @@ func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHand haveApproval, approvalErr := approvalForDeploy(ctx, b, plan) if !haveApproval { - // No version was created, so there is nothing to complete: the deferred - // CompleteVersion is a no-op until CreateVersion has run. The version number - // this deploy would have used is simply left for the next one to take. - // - // Both outcomes land here - the user declining, and a console that cannot - // prompt at all, which returns an error instead. + // No version was created, so the deferred CompleteVersion is a no-op and the version + // number is left for the next deploy. Both the user declining and a console that + // cannot prompt land here. if approvalErr != nil { logdiag.LogError(ctx, approvalErr) return diff --git a/bundle/phases/destroy.go b/bundle/phases/destroy.go index 1ce02ed1c81..1d28df250ca 100644 --- a/bundle/phases/destroy.go +++ b/bundle/phases/destroy.go @@ -159,10 +159,7 @@ func destroyCore(ctx context.Context, b *bundle.Bundle, plan *deployplan.Plan, e return } - // Complete the version before deleting the remote files. The deployment is a - // node under the state directory, so files.Delete removes it and any later call - // fails with 404. CompleteVersion is idempotent, so the deferred call in Destroy - // is a no-op after this. + // Complete version before deleting remote files; the deployment node is under statePath. if err := recorder.CompleteVersion(ctx, true); err != nil { logdiag.LogError(ctx, err) return @@ -206,10 +203,8 @@ func Destroy(ctx context.Context, b *bundle.Bundle, engine engine.EngineType) { return } - // Set up DMS recording of this destroy as a version. The version is not - // created until the destroy is approved (below), so a cancelled destroy - // records nothing; the deferred CompleteVersion is a no-op until then. It is - // deferred before lock.Release so it runs while the lock is still held. + // Set up DMS recording of this destroy. Version is created after approval; cancelled + // destroy records nothing. Deferred before lock.Release to hold the lock. recorder, err := newDeploymentRecorder(ctx, b, engine, dms.VersionTypeDestroy) if err != nil { logdiag.LogError(ctx, err) diff --git a/bundle/phases/dms.go b/bundle/phases/dms.go index 9803ebef9f6..c6e9d5ff259 100644 --- a/bundle/phases/dms.go +++ b/bundle/phases/dms.go @@ -20,18 +20,9 @@ import ( "github.com/databricks/databricks-sdk-go/service/bundledeployments" ) -// newDeploymentRecorder returns a dms.Recorder for the current deployment, or -// nil when DMS recording does not apply. A nil recorder is a no-op, so callers -// do not need to branch on it. -// -// Recording is enabled only when the bundle asks for it (see -// recordsDeploymentHistory) AND the engine is direct: DMS resource state is tracked -// per direct-engine deployment. Returning nil for terraform leaves those untouched. -// -// The deployment ID is resolved from the workspace, not local state (see -// dms.ResolveDeploymentID). The lookup happens here, after the deployment lock is -// held, so it sees any deployment a concurrent deploy created. It is empty on the -// first recorded deploy, where the recorder creates the deployment instead. +// newDeploymentRecorder returns a recorder for the deployment, or nil if recording +// does not apply. Enabled only for direct engine and when the bundle opts in. +// The deployment ID is resolved from the workspace node, empty on first deploy. func newDeploymentRecorder(ctx context.Context, b *bundle.Bundle, eng engine.EngineType, versionType dms.VersionType) (*dms.Recorder, error) { if !recordsDeploymentHistory(ctx, b) { return nil, nil @@ -84,13 +75,8 @@ func setOperationRecorder(ctx context.Context, b *bundle.Bundle, recorder *dms.R b.DeploymentBundle.OpRec = direct.NewOperationRecorder(apiClient, recorder.DeploymentID(), recorder.Version()) } -// logDeploymentVersion links to the version this deploy was recorded under, so the -// user can follow it while the deploy runs rather than hunting for the ID afterwards. -// A nil recorder means recording is off, and a zero version means the version was -// never created. -// -// The workspace ID is left out of the URL: the page redirects correctly without it, -// and omitting it keeps the line short enough to stay clickable in a terminal. +// logDeploymentVersion logs the deployment version URL. Workspace ID is omitted +// so the page stays clickable in a terminal and redirects correctly without it. func logDeploymentVersion(ctx context.Context, b *bundle.Bundle, recorder *dms.Recorder) { if recorder == nil || recorder.Version() == 0 { return diff --git a/cmd/bundle/utils/process.go b/cmd/bundle/utils/process.go index 0ee606f797a..c602ea1ec6f 100644 --- a/cmd/bundle/utils/process.go +++ b/cmd/bundle/utils/process.go @@ -231,9 +231,7 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle if needDirectState { _, localPath := b.StateFilenameDirect(ctx) - // Recording makes the service the source of truth for resource state, so a - // deploy has to plan against what it holds rather than a local file that - // another machine's deploy may have left behind. + // Recording makes the service the source of truth for state. var dmsSource *dstate.DMSSource if env.RecordsDeploymentHistory(ctx, b.Config.Experimental != nil && b.Config.Experimental.RecordDeploymentHistory) { w := b.WorkspaceClient(ctx) @@ -247,10 +245,9 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle DeploymentID: deploymentID, } - // Stamp the deployment onto the resources before anything diffs them. - // The workspace has it, so a plan that left it unset would report drift - // on a resource nobody touched. The version is stamped by the deploy - // phase instead, once it claims one. + // Stamp the deployment before anything diffs the resources: the workspace + // has it, so leaving it unset would report drift on an untouched resource. + // The deploy phase stamps the version, once it claims one. bundle.ApplyContext(ctx, b, metadata.AnnotateDeployment(deploymentID)) if logdiag.HasError(ctx) { return b, stateDesc, root.ErrAlreadyPrinted diff --git a/libs/dms/recorder.go b/libs/dms/recorder.go index 2386549d3e2..ad973189663 100644 --- a/libs/dms/recorder.go +++ b/libs/dms/recorder.go @@ -79,10 +79,7 @@ func (a *apiVersionCreator) CreateVersion(ctx context.Context, deploymentID, ver return &version, nil } -// Recorder records a single deploy/destroy as a version with DMS. The server -// assigns the deployment ID on the first deploy and later deploys reuse it; a -// destroy deletes the record, so the next deploy starts over (see -// ResolveDeploymentID). +// Recorder records a deploy/destroy as a version with DMS. Server assigns ID on first deploy. type Recorder struct { svc bundledeployments.BundleDeploymentsInterface versions versionCreator @@ -169,9 +166,8 @@ func (r *Recorder) Version() int64 { return r.versionNum } -// CreateVersion registers a new version with DMS, claiming it for the duration -// of the deployment. A nil Recorder is a no-op, so callers can leave it nil -// when recording is disabled. +// CreateVersion registers a new version with DMS, claiming it for the deployment. +// Nil Recorder is a no-op. func (r *Recorder) CreateVersion(ctx context.Context) error { if r == nil { return nil @@ -186,10 +182,9 @@ func (r *Recorder) CreateVersion(ctx context.Context) error { versionID := strconv.FormatInt(r.versionNum, 10) - // The server rejects the call unless versionID is numerically greater than - // last_version_id and previous_version_id matches it. That is what makes claiming - // the number up front safe: another deploy that took it between PrepareDeployment - // and here is reported rather than overwritten. + // The server rejects this unless versionID exceeds last_version_id and + // previous_version_id matches it, which is what makes claiming the number up front + // safe: a deploy that took it in the meantime is reported, not overwritten. version, err := r.versions.CreateVersion(ctx, r.deploymentID, versionID, createVersionRequest{ CliVersion: build.GetInfo().Version, VersionType: r.versionType, @@ -255,10 +250,8 @@ func (r *Recorder) completeVersion(ctx context.Context, reason bundledeployments return nil } -// PrepareDeployment makes sure the deployment exists and works out the version number -// this deploy will create, without creating it. Both are needed before the plan, which -// stamps them onto the resources it is computed from; the version itself is not created -// until CreateVersion, so a deploy the user declines never claims one. +// PrepareDeployment ensures the deployment exists and determines the version number to create, +// without creating it. Both are needed before planning; version itself is created by CreateVersion. func (r *Recorder) PrepareDeployment(ctx context.Context) error { if r == nil { return nil diff --git a/libs/dms/resolve.go b/libs/dms/resolve.go index 9bbf5a84b56..42269c9db83 100644 --- a/libs/dms/resolve.go +++ b/libs/dms/resolve.go @@ -15,13 +15,8 @@ import ( // match DeploymentWhsClient.DEPLOYMENT_NODE_NAME on the service side. const DeploymentNodeName = "resources.deployment.json" -// ResolveDeploymentID returns the DMS deployment ID for the bundle whose state -// lives under statePath, or empty if it has never recorded a deployment. -// -// The CLI stores the ID nowhere: DMS registers the deployment as a workspace -// node and that node's ID *is* the deployment ID, so a get-status is the lookup. -// This keeps the workspace the single source of truth — a destroyed deployment -// reports absent instead of leaving a dangling ID in the local state file. +// ResolveDeploymentID returns the DMS deployment ID from the workspace node registered +// under statePath, or empty if never recorded. The workspace node ID *is* the deployment ID. func ResolveDeploymentID(ctx context.Context, w *databricks.WorkspaceClient, statePath string) (string, error) { nodePath := path.Join(statePath, DeploymentNodeName) diff --git a/libs/testserver/bundle.go b/libs/testserver/bundle.go index 47d2ff1160a..31fe2ee4fc6 100644 --- a/libs/testserver/bundle.go +++ b/libs/testserver/bundle.go @@ -15,10 +15,8 @@ import ( // Handlers for the Deployment Metadata Service (DMS) API under /api/2.0/bundle. // State is kept in FakeWorkspace.dmsDeployments, keyed by deployment ID. -// dmsDeploymentNodeName is the name of the workspace node the service creates -// for every deployment. It must match DEPLOYMENT_NODE_NAME on the service side -// (DeploymentWhsClient); the literal is repeated here rather than shared with -// the CLI so a test would catch the CLI drifting from the service. +// dmsDeploymentNodeName is the workspace node name the service uses for deployments. +// It must match DEPLOYMENT_NODE_NAME on the service side (DeploymentWhsClient). const dmsDeploymentNodeName = "resources.deployment.json" // dmsUpdatableOperationFields are the update_mask paths UpdateOperation accepts. Any @@ -39,11 +37,9 @@ type dmsDeployment struct { // one per resource per version, so a resource written twice in a version updates // its operation rather than adding another. operations map[string]*bundledeployments.Operation - // lastSuccessfulVersionID is the highest version that completed - // successfully. The server advances last_successful_version_id only on - // success (unlike last_version_id), and the read path treats a non-empty - // value as "DMS owns the state". Tracked separately because the SDK - // Deployment struct does not yet carry the field (still stage:DEVELOPMENT). + // lastSuccessfulVersionID is the highest version completed successfully. + // The read path treats a non-empty value as "DMS owns the state"; + // the SDK Deployment struct does not yet carry this field. lastSuccessfulVersionID string } @@ -77,10 +73,8 @@ func (s *FakeWorkspace) CreateDeployment(req Request) Response { }, } - // The record is created together with the node, so a get on it always resolves - // for a node that exists. It carries no version yet: last_version_id stays empty - // until the first CreateVersion, which is how a client that registers a - // deployment and then fails leaves a record with no versions. + // The record carries no version yet; last_version_id stays empty until + // the first CreateVersion. A failed registration leaves a record with no versions. deploymentID := strconv.FormatInt(objectID, 10) s.dmsDeploymentNodes[deploymentID] = nodePath @@ -110,14 +104,9 @@ func (s *FakeWorkspace) GetDeployment(deploymentID string) Response { return Response{Body: body} } -// deploymentBody renders a deployment the way the real server does: the typed -// fields plus last_successful_version_id, which the generated SDK struct does -// not carry yet (still stage:DEVELOPMENT) but the read path reads off the raw -// JSON. -// -// The extra field cannot be added by embedding Deployment in a wrapper struct: -// Deployment has its own MarshalJSON, which is promoted to the wrapper and -// silently drops any sibling field. +// deploymentBody renders a deployment with last_successful_version_id, which +// the SDK struct doesn't yet carry. Embedding in a wrapper won't work because +// Deployment.MarshalJSON silently drops sibling fields. func deploymentBody(d *dmsDeployment) (map[string]any, error) { raw, err := json.Marshal(d.deployment) if err != nil { @@ -173,9 +162,8 @@ func (s *FakeWorkspace) CreateVersion(req Request, deploymentID string) Response return dmsNotFound("deployment " + deploymentID) } - // Mirror the server-side checks: version_id must be numerically greater than - // the most recent version (not exactly one more), and previous_version_id must - // name that version, which is what detects a concurrent deploy. + // version_id must be numerically greater than the most recent version, + // and previous_version_id must name that version to detect concurrent deploys. next, err := strconv.ParseInt(versionID, 10, 64) if err != nil || next < 1 { return dmsInvalidArgument("version_id must be a positive integer, got " + versionID) @@ -191,15 +179,10 @@ func (s *FakeWorkspace) CreateVersion(req Request, deploymentID string) Response return dmsAborted("previous_version_id is outdated; the deployment's most recent version is " + d.deployment.LastVersionId) } - // Not modelled: the deployment lock. Creating a version takes one on the real - // service (VersionStorage.LOCK_DURATION_MS), which refuses a second deploy with - // "deployment N is locked by version M (lock expires at ...)" until a two-minute - // lease elapses without a heartbeat. Refusing purely on an in-flight version is - // wrong here: several tests kill the CLI mid-apply, which leaves the version - // in-progress forever, where the real service would let the lease expire. Modelling - // it needs the lease clock, and Version carries no heartbeat field to hang it on. + // Note: deployment lock not modelled. Tests kill the CLI mid-apply, leaving + // the version in-progress forever, whereas the real service lets the lease expire. - // bundle_root_path is relative to git_folder_path, so the service rejects a + // bundle_root_path is relative to git_folder_path, so the service rejects // workspace_info that carries one without the other. if ws := version.WorkspaceInfo; ws != nil && (ws.GitFolderPath == "") != (ws.BundleRootPath == "") { return dmsInvalidArgument("workspace_info.git_folder_path and workspace_info.bundle_root_path must be set together") @@ -266,9 +249,7 @@ func (s *FakeWorkspace) CreateOperation(req Request, deploymentID, versionID str return dmsNotFound("deployment " + deploymentID) } - // A delete carries no state, so resource_id is the only thing identifying which - // resource it refers to; the service rejects a delete without one. A failed - // delete is exempt only for create-flavored actions, which may not have an ID. + // delete requires resource_id. Create-flavored actions may lack an ID. if op.ActionType == bundledeployments.OperationActionTypeOperationActionTypeDelete && op.ResourceId == "" { return dmsInvalidArgument("resource_id is required for OPERATION_ACTION_TYPE_DELETE operations") } @@ -278,17 +259,13 @@ func (s *FakeWorkspace) CreateOperation(req Request, deploymentID, versionID str return dmsInvalidArgument("error_message is only allowed when status is OPERATION_STATUS_FAILED") } - // State describes a resource that exists, so an operation with state must identify - // which resource via resource_id. This applies to both succeeded and failed - // operations: a failed operation reports prior state to document what existed - // before the attempt failed. + // An operation with state must identify its resource via resource_id, + // even for failed operations reporting prior state. if op.State != "" && op.ResourceId == "" { return dmsInvalidArgument("resource_id is required for an operation that records state") } - // The service names operations after the resource key, so it keeps one per - // resource per version: creating a second one for the same resource conflicts, - // and the caller has to use UpdateOperation instead. + // One operation per resource per version; duplicates conflict. opName := "deployments/" + deploymentID + "/versions/" + versionID + "/operations/" + resourceKey if _, exists := d.operations[opName]; exists { return Response{ @@ -302,19 +279,15 @@ func (s *FakeWorkspace) CreateOperation(req Request, deploymentID, versionID str op.SequenceId = 1 d.operations[opName] = &op - // The service sends sequence_id as a JSON string (proto3 encodes 64-bit ints - // that way) while the SDK struct types it as an int64, so the response is built - // by hand to match the wire format the CLI actually parses. + // sequence_id is a JSON string on the wire but int64 in the SDK struct; + // build the response by hand to match what the CLI parses. body, err := operationBody(&op) if err != nil { return Response{StatusCode: 500, Body: map[string]string{"message": err.Error()}} } - // Reflect the operation onto the deployment-level resource set the way the backend - // does: state is what projects a resource, so an operation without it removes the - // resource instead of listing one (OperationStorage.createOperation buffers a delete - // when the entity has no state). Together with the invariant that state requires a - // resource_id, that means a listed resource always has an id. + // State projects a resource; no state deletes it. Together with the invariant + // that state requires resource_id, a listed resource always has an id. if op.State == "" { delete(d.resources, resourceKey) } else { @@ -380,10 +353,8 @@ func (s *FakeWorkspace) UpdateOperation(req Request, deploymentID, versionID, re if updateMask == "" { return dmsInvalidArgument("update_mask is required") } - // Only the paths named in the mask are written; every other field of the stored - // operation is left as it is. A caller that omits state keeps the state already - // recorded, which is how a failure marks an operation failed without erasing the - // resource it had written. + // Only masked paths are written; other fields keep their values. + // Omitting state keeps the already-recorded state. update := map[string]bool{} for path := range strings.SplitSeq(updateMask, ",") { path = strings.TrimSpace(path) @@ -409,10 +380,8 @@ func (s *FakeWorkspace) UpdateOperation(req Request, deploymentID, versionID, re return dmsAborted("sequence_id is outdated; the operation is at " + strconv.FormatInt(existing.SequenceId, 10)) } - // The invariants hold over the operation the update leaves behind, not the request: - // a field the mask leaves out keeps the value it already had, so a failure that - // updates only status and error_message is checked against the state and id the - // earlier write recorded. + // Invariants check the operation after the update, not the request. + // The mask leaves unspecified fields unchanged. after := *existing if update["state"] { after.State = op.State @@ -432,10 +401,8 @@ func (s *FakeWorkspace) UpdateOperation(req Request, deploymentID, versionID, re return dmsInvalidArgument("error_message is only allowed when status is OPERATION_STATUS_FAILED") } - // State describes a resource that exists, so an operation with state must identify - // which resource via resource_id. This applies to both succeeded and failed - // operations: a failed operation reports prior state to document what existed - // before the attempt failed. + // An operation with state must identify its resource via resource_id, + // even for failed operations reporting prior state. if after.State != "" && after.ResourceId == "" { return dmsInvalidArgument("resource_id is required for an operation that records state") } diff --git a/libs/testserver/fake_workspace.go b/libs/testserver/fake_workspace.go index bb24cbf5451..1b2559d3ba2 100644 --- a/libs/testserver/fake_workspace.go +++ b/libs/testserver/fake_workspace.go @@ -233,10 +233,8 @@ type FakeWorkspace struct { PostgresSyncedTables map[string]postgres.SyncedTable PostgresOperations map[string]postgres.Operation - // Branches and endpoints that the server provisioned implicitly together - // with their parent (e.g. the production branch on a new project, or the - // primary endpoint on a new branch). The real backend rejects independent - // deletion of these — they go away only when the parent is deleted. + // Branches and endpoints provisioned implicitly with their parent. + // Independent deletion is rejected; they delete only with the parent. postgresImplicitBranches map[string]bool postgresImplicitEndpoints map[string]bool @@ -248,10 +246,9 @@ type FakeWorkspace struct { // deployment ID. Each record carries its versions and latest resource state. dmsDeployments map[string]*dmsDeployment - // dmsDeploymentNodes maps deployment ID to the workspace node CreateDeployment - // registered for it. A deployment appears here before it has a record in - // dmsDeployments: the record is created by its first version, so the node is - // what makes an ID valid in between. + // dmsDeploymentNodes maps deployment ID to the workspace node CreateDeployment made for + // it. An ID appears here before dmsDeployments has a record, which its first version + // creates, so the node is what makes the ID valid in between. dmsDeploymentNodes map[string]string } diff --git a/libs/workspaceurls/urls.go b/libs/workspaceurls/urls.go index cd4ce5026a7..bd0744e6fef 100644 --- a/libs/workspaceurls/urls.go +++ b/libs/workspaceurls/urls.go @@ -71,14 +71,8 @@ func ResourceTypes() []string { return names } -// DeploymentURL returns the workspace URL for a bundle deployment recorded with -// the deployment metadata service, of the form -// -// /deployments/?version= -// -// The version pins the page to the deploy that produced it. It is separate from -// ResourceURL because a deployment is not a bundle resource type: it has no entry -// in resourceURLPatterns and takes a query parameter none of those do. +// DeploymentURL returns the workspace URL for a bundle deployment: +// /deployments/?version=. Version pins the page to the deploy that produced it. func DeploymentURL(baseURL url.URL, deploymentID string, version int64) string { if deploymentID == "" { return "" From fece1fdab752f8ec3049089a6bfbfbf5ec2bca3f Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 18 Aug 2026 00:52:35 +0000 Subject: [PATCH 100/125] bundle: fixes from a per-area review of this PR Eight reviews, one per logical part of the PR, each asked to fix what it found. Four real problems came out: Truncating an over-long error message cut it at a byte boundary, so a cut landing inside a rune produced invalid UTF-8 for a field the service stores as a string. It now drops the partial rune, bounded so a message that was already invalid loses those bytes rather than being stripped away. CreateVersion did not recognise the 409 ABORTED it gets when another deploy claimed the same version number between PrepareDeployment and the call. The conflict now names the version and says to try again, and still wraps the cause. setOperationRecorder logged its error and the deploy carried on without a recorder. Both callers now stop. The deferred CompleteVersion still runs, so the version is completed as failed and the lock is released. The fake service required resource_id on a DELETE at creation but not on an update, so a CLI that cleared it would have passed the suite. Also covers readDMSState replacing local state and accepting an empty resource list, the escape-hatch env var, and that a failed upload leaves the recorded sequence id alone so the next write still updates rather than re-creating. Co-authored-by: Isaac --- .../bundle/dms/not-supported/output.txt | 2 +- .../validate_record_deployment_history.go | 2 +- ...validate_record_deployment_history_test.go | 2 +- bundle/direct/dstate/dms_test.go | 49 +++++++++++++ bundle/direct/oprecorder.go | 10 +++ bundle/direct/oprecorder_test.go | 69 +++++++++++++++++++ bundle/env/dms_test.go | 20 ++++++ bundle/phases/deploy.go | 3 + bundle/phases/destroy.go | 3 + libs/dms/recorder.go | 5 ++ libs/dms/recorder_test.go | 34 +++++++++ libs/testserver/bundle.go | 5 ++ 12 files changed, 201 insertions(+), 3 deletions(-) diff --git a/acceptance/bundle/dms/not-supported/output.txt b/acceptance/bundle/dms/not-supported/output.txt index b665d545d12..e788699dba0 100644 --- a/acceptance/bundle/dms/not-supported/output.txt +++ b/acceptance/bundle/dms/not-supported/output.txt @@ -1,7 +1,7 @@ === record_deployment_history is rejected: the service side is not ready for users yet >>> musterr [CLI] bundle validate -Error: experimental.record_deployment_history is not supported yet +Error: experimental.record_deployment_history is not supported yet; remove this setting from your bundle configuration at experimental.record_deployment_history in databricks.yml:5:30 diff --git a/bundle/config/validate/validate_record_deployment_history.go b/bundle/config/validate/validate_record_deployment_history.go index a9ee61af1c5..b7121f8ac1e 100644 --- a/bundle/config/validate/validate_record_deployment_history.go +++ b/bundle/config/validate/validate_record_deployment_history.go @@ -33,7 +33,7 @@ func (v *validateRecordDeploymentHistory) Apply(ctx context.Context, b *bundle.B } return diag.Diagnostics{{ Severity: diag.Error, - Summary: recordDeploymentHistoryPath + " is not supported yet", + Summary: recordDeploymentHistoryPath + " is not supported yet; remove this setting from your bundle configuration", Paths: []dyn.Path{dyn.MustPathFromString(recordDeploymentHistoryPath)}, Locations: b.Config.GetLocations(recordDeploymentHistoryPath), }} diff --git a/bundle/config/validate/validate_record_deployment_history_test.go b/bundle/config/validate/validate_record_deployment_history_test.go index 1bb172766f2..51d41827aeb 100644 --- a/bundle/config/validate/validate_record_deployment_history_test.go +++ b/bundle/config/validate/validate_record_deployment_history_test.go @@ -43,7 +43,7 @@ func TestValidateRecordDeploymentHistory(t *testing.T) { } require.Len(t, diags, 1) assert.Equal(t, diag.Error, diags[0].Severity) - assert.Equal(t, "experimental.record_deployment_history is not supported yet", diags[0].Summary) + assert.Equal(t, "experimental.record_deployment_history is not supported yet; remove this setting from your bundle configuration", diags[0].Summary) assert.Equal(t, recordDeploymentHistoryPath, diags[0].Paths[0].String()) }) } diff --git a/bundle/direct/dstate/dms_test.go b/bundle/direct/dstate/dms_test.go index 23a9e7f3a97..1120d09bb5d 100644 --- a/bundle/direct/dstate/dms_test.go +++ b/bundle/direct/dstate/dms_test.go @@ -66,3 +66,52 @@ func TestFetchDeploymentResourcesRejectsMalformedState(t *testing.T) { _, err := fetchDeploymentResources(t.Context(), f, "dep-1") assert.ErrorContains(t, err, "interpreting state recorded for resources.jobs.foo") } + +func TestReadDMSStateReplacesLocalState(t *testing.T) { + // readDMSState should replace the file-derived state with what DMS has, + // even if the file has different resources. + src := &DMSSource{ + Client: &fakeResourceLister{resources: []bundledeployments.Resource{ + {ResourceKey: "jobs.foo", ResourceId: "dms-id", State: `{"state":{"name":"from-dms"}}`}, + }}, + DeploymentID: "dep-1", + } + + var db DeploymentState + db.Data.State = map[string]ResourceEntry{ + "resources.jobs.bar": {ID: "file-id", State: json.RawMessage(`{"name":"from-file"}`)}, + } + db.stateIDs = map[string]string{"resources.jobs.bar": "file-id"} + db.Path = "test-path" + + err := db.readDMSState(t.Context(), src) + require.NoError(t, err) + + // State now reflects DMS, not the file. + assert.Equal(t, map[string]ResourceEntry{ + "resources.jobs.foo": {ID: "dms-id", State: json.RawMessage(`{"name":"from-dms"}`)}, + }, db.Data.State) + assert.Equal(t, map[string]string{"resources.jobs.foo": "dms-id"}, db.stateIDs) +} + +func TestReadDMSStateAcceptsEmptyResourceList(t *testing.T) { + // An empty DMS response is valid: it means a successful deploy of nothing. + src := &DMSSource{ + Client: &fakeResourceLister{resources: []bundledeployments.Resource{}}, + DeploymentID: "dep-1", + } + + var db DeploymentState + db.Data.State = map[string]ResourceEntry{ + "resources.jobs.bar": {ID: "file-id", State: json.RawMessage(`{"name":"from-file"}`)}, + } + db.stateIDs = map[string]string{"resources.jobs.bar": "file-id"} + db.Path = "test-path" + + err := db.readDMSState(t.Context(), src) + require.NoError(t, err) + + // State is now empty, reflecting the empty DMS response. + assert.Empty(t, db.Data.State) + assert.Empty(t, db.stateIDs) +} diff --git a/bundle/direct/oprecorder.go b/bundle/direct/oprecorder.go index 16c77acccf7..0436a5e72f5 100644 --- a/bundle/direct/oprecorder.go +++ b/bundle/direct/oprecorder.go @@ -7,6 +7,7 @@ import ( "slices" "strings" "sync" + "unicode/utf8" "github.com/databricks/cli/bundle/deployplan" "github.com/databricks/cli/bundle/direct/dstate" @@ -101,6 +102,15 @@ func newFailedOperation(action deployplan.ActionType, resourceID string, priorSt message := cause.Error() if len(message) > maxOperationErrorMessageSize { message = message[:maxOperationErrorMessageSize] + // The cut can land inside a rune, and the service stores a string. Drop the partial + // one: at most UTFMax-1 bytes of it can be left, so a message that was already + // invalid loses those bytes rather than being stripped away entirely. + for range utf8.UTFMax - 1 { + if utf8.ValidString(message) { + break + } + message = message[:len(message)-1] + } } return recordedOperation{ diff --git a/bundle/direct/oprecorder_test.go b/bundle/direct/oprecorder_test.go index e50ac5ad75d..033df1215b6 100644 --- a/bundle/direct/oprecorder_test.go +++ b/bundle/direct/oprecorder_test.go @@ -7,6 +7,7 @@ import ( "strings" "sync" "testing" + "unicode/utf8" "github.com/databricks/cli/bundle/deployplan" "github.com/databricks/cli/bundle/direct/dstate" @@ -254,6 +255,74 @@ func TestNewFailedOperationTruncatesLongError(t *testing.T) { assert.Len(t, op.errorMessage, maxOperationErrorMessageSize) } +func TestNewFailedOperationPreservesUTF8OnTruncation(t *testing.T) { + // The cut lands one byte into the emoji, so a byte-wise truncation would leave a partial + // rune behind and the service stores state and messages as strings. + msg := strings.Repeat("a", maxOperationErrorMessageSize-1) + "❌" + "x" + + op, err := newFailedOperation(deployplan.Update, "job-123", nil, errors.New(msg)) + require.NoError(t, err) + + assert.True(t, utf8.ValidString(op.errorMessage)) + // The whole emoji went, so the message is shorter than the limit rather than exactly it. + assert.Equal(t, strings.Repeat("a", maxOperationErrorMessageSize-1), op.errorMessage) +} + +func TestOperationRecorderReturnsAPIErrors(t *testing.T) { + // A failed upload returns its error and leaves the recorded sequence id alone, so a later + // write for the same resource still updates the operation with the precondition the + // service last gave us rather than trying to create a second one. + failingClient := &failingOpClient{sequence: "9", failOn: 1} + r := newOperationRecorder(failingClient, "dep-1", 2) + + uploadOne(t, r, "resources.jobs.foo", deployplan.Create, "job-1", envelope(t, "first")) + + second, err := newStateOperation(dstate.OperationInfo{Action: deployplan.Update}, "job-2", envelope(t, "second")) + require.NoError(t, err) + err = r.upload(t.Context(), "resources.jobs.foo", second) + require.Error(t, err) + assert.Equal(t, "injected error", err.Error()) + + // The third write is what proves the sequence id survived the failure. + uploadOne(t, r, "resources.jobs.foo", deployplan.Update, "job-3", envelope(t, "third")) + + require.Len(t, failingClient.calls, 3) + assert.Equal(t, "create", failingClient.calls[0].method) + assert.Equal(t, "update", failingClient.calls[1].method) + assert.Equal(t, "update", failingClient.calls[2].method) + assert.Equal(t, "9", failingClient.calls[2].update.SequenceId) +} + +// failingOpClient fails the call at index failOn and reports sequence on the rest. +type failingOpClient struct { + mu sync.Mutex + calls []fakeOpCall + sequence string + failOn int +} + +func (f *failingOpClient) CreateOperation(ctx context.Context, parent, resourceKey string, op bundledeployments.Operation) (operationResponse, error) { + f.mu.Lock() + defer f.mu.Unlock() + callNum := len(f.calls) + f.calls = append(f.calls, fakeOpCall{method: "create", parent: parent, resourceKey: resourceKey, op: op}) + if callNum == f.failOn { + return operationResponse{}, errors.New("injected error") + } + return operationResponse{SequenceId: f.sequence}, nil +} + +func (f *failingOpClient) UpdateOperation(ctx context.Context, parent, resourceKey string, fields []string, body updateOperationRequest) (operationResponse, error) { + f.mu.Lock() + defer f.mu.Unlock() + callNum := len(f.calls) + f.calls = append(f.calls, fakeOpCall{method: "update", parent: parent, resourceKey: resourceKey, update: body, fields: fields}) + if callNum == f.failOn { + return operationResponse{}, errors.New("injected error") + } + return operationResponse{SequenceId: "2"}, nil +} + func TestDeployActionToSDK(t *testing.T) { cases := []struct { action deployplan.ActionType diff --git a/bundle/env/dms_test.go b/bundle/env/dms_test.go index daa45dc00a2..6a9e8d05b51 100644 --- a/bundle/env/dms_test.go +++ b/bundle/env/dms_test.go @@ -39,3 +39,23 @@ func TestRecordsDeploymentHistory(t *testing.T) { ctx := env.Set(t.Context(), RecordDeploymentHistoryVariable, "true") assert.True(t, RecordsDeploymentHistory(ctx, false)) } + +func TestForceAllowRecordDeploymentHistory(t *testing.T) { + for _, tc := range []struct { + value string + want bool + }{ + {"", false}, + {"1", true}, + {"yes", true}, + {"true", true}, + {"false", true}, // any non-empty value enables the escape hatch + } { + ctx := env.Set(t.Context(), ForceAllowRecordDeploymentHistoryVariable, tc.value) + assert.Equal(t, tc.want, ForceAllowRecordDeploymentHistory(ctx), "value %q", tc.value) + } +} + +func TestForceAllowRecordDeploymentHistoryUnset(t *testing.T) { + assert.False(t, ForceAllowRecordDeploymentHistory(t.Context())) +} diff --git a/bundle/phases/deploy.go b/bundle/phases/deploy.go index 4638f0ff81d..7e12f2f73ab 100644 --- a/bundle/phases/deploy.go +++ b/bundle/phases/deploy.go @@ -354,6 +354,9 @@ func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHand // Record operations under that version, so DMS holds the deployed resource state. setOperationRecorder(ctx, b, recorder) + if logdiag.HasError(ctx) { + return + } deployCore(ctx, b, plan, stateEngine, requestedEngine) if logdiag.HasError(ctx) { diff --git a/bundle/phases/destroy.go b/bundle/phases/destroy.go index 1d28df250ca..3405daf7bc0 100644 --- a/bundle/phases/destroy.go +++ b/bundle/phases/destroy.go @@ -277,6 +277,9 @@ func Destroy(ctx context.Context, b *bundle.Bundle, engine engine.EngineType) { return } setOperationRecorder(ctx, b, recorder) + if logdiag.HasError(ctx) { + return + } destroyCore(ctx, b, plan, engine, recorder) } else { cmdio.LogString(ctx, "Destroy cancelled!") diff --git a/libs/dms/recorder.go b/libs/dms/recorder.go index ad973189663..d3c878479cb 100644 --- a/libs/dms/recorder.go +++ b/libs/dms/recorder.go @@ -196,6 +196,11 @@ func (r *Recorder) CreateVersion(ctx context.Context) error { WorkspaceInfo: r.metadata.Workspace, }) if err != nil { + // A 409 ABORTED means another deploy claimed this version number in between + // PrepareDeployment and here. + if isAbortedErr(err) { + return fmt.Errorf("another deploy already claimed version %s of this deployment, try again: %w", versionID, err) + } return fmt.Errorf("failed to create deployment version: %w", err) } diff --git a/libs/dms/recorder_test.go b/libs/dms/recorder_test.go index b247cf2edd8..3e11ae20f69 100644 --- a/libs/dms/recorder_test.go +++ b/libs/dms/recorder_test.go @@ -51,10 +51,14 @@ type fakeVersionRequest struct { // createVersionRequest), so the two use different signatures. type fakeVersions struct { requests *[]fakeVersionRequest + err error } func (f fakeVersions) CreateVersion(ctx context.Context, deploymentID, versionID string, body createVersionRequest) (*bundledeployments.Version, error) { *f.requests = append(*f.requests, fakeVersionRequest{deploymentID: deploymentID, versionID: versionID, body: body}) + if f.err != nil { + return nil, f.err + } return &bundledeployments.Version{VersionId: versionID}, nil } @@ -276,6 +280,36 @@ func TestRecorderCreateVersionUsesThePreparedNumber(t *testing.T) { assert.Equal(t, int64(5), r.Version()) } +func TestRecorderCreateVersionDetectsAbortedConflict(t *testing.T) { + f := &fakeDMS{ + getDeployment: func(id string) (*bundledeployments.Deployment, error) { + return &bundledeployments.Deployment{Name: "deployments/" + id, LastVersionId: "4"}, nil + }, + } + // Simulate concurrency conflict: another deploy claimed the version number. + conflictErr := &apierr.APIError{ + StatusCode: 409, + ErrorCode: "ABORTED", + } + r := NewRecorder(RecorderOptions{ + Service: f, + Versions: &fakeVersions{requests: &f.versions, err: conflictErr}, + DeploymentID: "stored-id", + StatePath: testStatePath, + Metadata: Metadata{TargetName: "dev", DisplayName: testDisplayName}, + VersionType: VersionTypeDeploy, + }) + + require.NoError(t, r.PrepareDeployment(t.Context())) + err := r.CreateVersion(t.Context()) + + // Names the version that was taken and tells the user to retry, and keeps the + // underlying ABORTED so callers can still match on it. + assert.ErrorContains(t, err, "another deploy already claimed version 5") + assert.ErrorContains(t, err, "try again") + assert.ErrorIs(t, err, conflictErr) +} + func TestDeploymentIDFromName(t *testing.T) { id, err := deploymentIDFromName("deployments/abc-123") require.NoError(t, err) diff --git a/libs/testserver/bundle.go b/libs/testserver/bundle.go index 31fe2ee4fc6..aadfa6f2fa4 100644 --- a/libs/testserver/bundle.go +++ b/libs/testserver/bundle.go @@ -401,6 +401,11 @@ func (s *FakeWorkspace) UpdateOperation(req Request, deploymentID, versionID, re return dmsInvalidArgument("error_message is only allowed when status is OPERATION_STATUS_FAILED") } + // Delete operations require resource_id. Create-flavored actions may lack an ID initially. + if after.ActionType == bundledeployments.OperationActionTypeOperationActionTypeDelete && after.ResourceId == "" { + return dmsInvalidArgument("resource_id is required for OPERATION_ACTION_TYPE_DELETE operations") + } + // An operation with state must identify its resource via resource_id, // even for failed operations reporting prior state. if after.State != "" && after.ResourceId == "" { From 77c9d2cde236146d1d742e6fc5016d5d87aa84b8 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 18 Aug 2026 09:11:29 +0000 Subject: [PATCH 101/125] bundle: restore comments this PR had no business changing The comment sweep rewrote comments that predate this PR. Restored verbatim from main, with the PR's own additions kept as separate sentences where it genuinely adds a fact: bundle/direct/apply.go why the state entry is dropped before the wait, and why an emptied resource drops it bundle/config/experimental.go RecordDeploymentHistory's doc comment libs/testserver/fake_workspace.go implicitly-provisioned branches/endpoints acceptance/bundle/migrate why the engine matrix is pinned acceptance/.../jobs/big_id the terraform panic text, which the trim lost acceptance/bundle/templates why the test is local-only bundle/phases/deploy.go logFileSummary, including that it is called on the failure paths and still accurate Two comments still differ from main on purpose: both track code this PR changed (print_requests.py gained --nostamp, and the excluded-traffic list gained the DMS calls). Co-authored-by: Isaac --- acceptance/bundle/migrate/test.toml | 5 +++-- .../bundle/resources/jobs/big_id/test.toml | 3 ++- acceptance/bundle/templates/test.toml | 2 +- bundle/config/experimental.go | 8 ++++++-- bundle/direct/apply.go | 17 ++++++++++++----- bundle/phases/deploy.go | 7 +++++-- libs/testserver/fake_workspace.go | 6 ++++-- 7 files changed, 33 insertions(+), 15 deletions(-) diff --git a/acceptance/bundle/migrate/test.toml b/acceptance/bundle/migrate/test.toml index 00027f9abdf..22dc1b062b8 100644 --- a/acceptance/bundle/migrate/test.toml +++ b/acceptance/bundle/migrate/test.toml @@ -3,8 +3,9 @@ RecordRequests = true IncludeRequestHeaders = ["User-Agent"] Ignore = [".databricks"] -# Tests set DATABRICKS_BUNDLE_ENGINE inline. Pin to ["direct"] so CI's engine -# filter includes them without running the script twice. +# Each test's script sets DATABRICKS_BUNDLE_ENGINE inline; pin the outer +# matrix to ["direct"] so CI's engine filter includes these tests without +# also running the same script twice per engine. EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] # These tests deploy on terraform and then migrate to direct. Recording is direct-only, so diff --git a/acceptance/bundle/resources/jobs/big_id/test.toml b/acceptance/bundle/resources/jobs/big_id/test.toml index ec7d104af8d..471575685c2 100644 --- a/acceptance/bundle/resources/jobs/big_id/test.toml +++ b/acceptance/bundle/resources/jobs/big_id/test.toml @@ -1,4 +1,5 @@ -# terraform fails: strconv.ParseInt on "[NUMID]" value out of range +# terraform fails with: +# panic: Error reading level state: strconv.ParseInt: parsing "[NUMID]": value out of range EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ['direct'] EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/templates/test.toml b/acceptance/bundle/templates/test.toml index bb62167d1b6..6c4d9689ac8 100644 --- a/acceptance/bundle/templates/test.toml +++ b/acceptance/bundle/templates/test.toml @@ -1,4 +1,4 @@ -# Local-only: many env differences (catalog use, node type, etc). +# Local-only: At the moment, there are many differences across different envs w.r.t to catalog use, node type and so on. # Template tests materialize and deploy full projects, taking tens of seconds each. # Recording adds minutes without new coverage the rest of the suite provides. diff --git a/bundle/config/experimental.go b/bundle/config/experimental.go index dc82eff75f5..005c4dfcade 100644 --- a/bundle/config/experimental.go +++ b/bundle/config/experimental.go @@ -50,8 +50,12 @@ type Experimental struct { // at which point we can deprecate or remove this field all together. SkipNamePrefixForSchema bool `json:"skip_name_prefix_for_schema,omitempty"` - // RecordDeploymentHistory opts into the deployment metadata service (DMS). - // Only for bundles with no deployed resources yet; DMS becomes the source of truth for state. + // RecordDeploymentHistory opts the bundle into the deployment metadata + // service (DMS), which records deployment history and tracks what changed + // across deployments. + // + // Only for bundles with no deployed resources yet: DMS becomes the source of + // truth for their state. RecordDeploymentHistory bool `json:"record_deployment_history,omitempty"` } diff --git a/bundle/direct/apply.go b/bundle/direct/apply.go index 37ba1b8191a..9c4c0292b35 100644 --- a/bundle/direct/apply.go +++ b/bundle/direct/apply.go @@ -116,8 +116,12 @@ func (d *DeploymentUnit) Recreate(ctx context.Context, db *dstate.DeploymentStat log.Warnf(ctx, "Treating %s id=%s as already deleted despite delete error: %s", d.ResourceKey, oldID, err) } - // Drop state so failure doesn't leave a malformed empty-ID entry. Recorded as - // recreate not delete: if create below fails, DMS sees mid-recreate not removed. + // Drop the state entry so a subsequent failure of Create or WaitAfterDelete + // leaves no malformed (empty-ID) entry behind. The next plan will see "no + // state" and retry as Create. + // + // Recorded as a recreate rather than a delete: if the create below fails, this is the + // operation DMS is left with, and it says the resource is mid-recreate. err = db.DeleteState(ctx, d.ResourceKey, dstate.OperationInfo{Action: deployplan.Recreate, InProgress: true}) if err != nil { return fmt.Errorf("deleting state: %w", err) @@ -157,9 +161,12 @@ func (d *DeploymentUnit) Update(ctx context.Context, db *dstate.DeploymentState, } if empty { - // The update emptied the resource (e.g. all grants revoked), so drop the entry: a - // fresh deploy of the same config would plan no node at all. Recorded as a delete, - // which is what it did to the state, rather than the update that caused it. + // The update emptied the resource out (e.g. all grants revoked). Keeping an entry + // would report the node as tracked-and-unchanged forever, while a fresh deploy of + // the same config plans no node at all; drop it so the two agree. + // + // Recorded as a delete, which is what it did to the state, not the update that + // caused it. err = db.DeleteState(ctx, d.ResourceKey, dstate.OperationInfo{Action: deployplan.Delete}) if err != nil { return fmt.Errorf("deleting state id=%s: %w", id, err) diff --git a/bundle/phases/deploy.go b/bundle/phases/deploy.go index 7e12f2f73ab..257703517d1 100644 --- a/bundle/phases/deploy.go +++ b/bundle/phases/deploy.go @@ -128,8 +128,11 @@ func deployCore(ctx context.Context, b *bundle.Bundle, plan *deployplan.Plan, st } } -// logFileSummary reports what the file sync did. Separate because a pure-code deploy -// leaves all resources unchanged, so would appear as a no-op without this line. +// logFileSummary reports what the file sync did. Separate from the resource summary +// because a deploy that only changes business logic (a .py or .sql file) leaves every +// resource unchanged, so without this line its summary is all zeros and looks like a +// no-op. Called on the failure paths too: the files were uploaded before whatever +// failed afterwards, so the count is accurate even then. func logFileSummary(ctx context.Context, b *bundle.Bundle) { if b.Quiet >= bundle.QuietAll { return diff --git a/libs/testserver/fake_workspace.go b/libs/testserver/fake_workspace.go index 1b2559d3ba2..f04d80a2e79 100644 --- a/libs/testserver/fake_workspace.go +++ b/libs/testserver/fake_workspace.go @@ -233,8 +233,10 @@ type FakeWorkspace struct { PostgresSyncedTables map[string]postgres.SyncedTable PostgresOperations map[string]postgres.Operation - // Branches and endpoints provisioned implicitly with their parent. - // Independent deletion is rejected; they delete only with the parent. + // Branches and endpoints that the server provisioned implicitly together + // with their parent (e.g. the production branch on a new project, or the + // primary endpoint on a new branch). The real backend rejects independent + // deletion of these — they go away only when the parent is deleted. postgresImplicitBranches map[string]bool postgresImplicitEndpoints map[string]bool From 7ea813b9de2215a68cc2efb8562d33daa018b602 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 18 Aug 2026 09:50:33 +0000 Subject: [PATCH 102/125] bundle: record the API status and error code with a failure A failure recorded cause.Error(), which for an SDK error is the bare message: the status code and error code are appended at display time by diag.FormatAPIErrorSummary and never reached the service. The deployment history exists to explain why a resource failed, and the error code is usually the most actionable part of that, so record the summarized form instead. The resource prefix is still left off: the operation already carries resource_key and action_type, so "cannot recreate resources.schemas.foo" would only repeat them. Co-authored-by: Isaac --- acceptance/bundle/dms/failed-recreate/output.txt | 2 +- acceptance/bundle/dms/failed-update/output.txt | 2 +- acceptance/bundle/dms/record-failure/output.txt | 2 +- bundle/direct/oprecorder.go | 5 ++++- 4 files changed, 7 insertions(+), 4 deletions(-) diff --git a/acceptance/bundle/dms/failed-recreate/output.txt b/acceptance/bundle/dms/failed-recreate/output.txt index dfbd67f371b..4cfe2ca1c76 100644 --- a/acceptance/bundle/dms/failed-recreate/output.txt +++ b/acceptance/bundle/dms/failed-recreate/output.txt @@ -33,7 +33,7 @@ Files: 3 uploaded, 0 deleted "update_mask": "error_message,status" }, "body": { - "error_message": "Fault injected by test.", + "error_message": "Fault injected by test. (400 INVALID_PARAMETER_VALUE)", "status": "OPERATION_STATUS_FAILED", "sequence_id": "1" } diff --git a/acceptance/bundle/dms/failed-update/output.txt b/acceptance/bundle/dms/failed-update/output.txt index 95e34913170..14dfd9d1fcb 100644 --- a/acceptance/bundle/dms/failed-update/output.txt +++ b/acceptance/bundle/dms/failed-update/output.txt @@ -56,7 +56,7 @@ Files: 3 uploaded, 0 deleted }, "body": { "action_type": "OPERATION_ACTION_TYPE_UPDATE", - "error_message": "updating id=main.dms_failed_update_schema: Fault injected by test.", + "error_message": "updating id=main.dms_failed_update_schema: Fault injected by test. (400 INVALID_PARAMETER_VALUE)", "resource_id": "main.dms_failed_update_schema", "resource_key": "schemas.foo", "state": "{\"state\":{\"catalog_name\":\"main\",\"comment\":\"v1\",\"name\":\"dms_failed_update_schema\"}}", diff --git a/acceptance/bundle/dms/record-failure/output.txt b/acceptance/bundle/dms/record-failure/output.txt index e1a3eb32915..abbbb421818 100644 --- a/acceptance/bundle/dms/record-failure/output.txt +++ b/acceptance/bundle/dms/record-failure/output.txt @@ -52,7 +52,7 @@ Files: 5 uploaded, 0 deleted }, "body": { "action_type": "OPERATION_ACTION_TYPE_CREATE", - "error_message": "cluster spec is invalid", + "error_message": "cluster spec is invalid (400 INVALID_PARAMETER_VALUE)", "resource_key": "jobs.doomed", "status": "OPERATION_STATUS_FAILED" } diff --git a/bundle/direct/oprecorder.go b/bundle/direct/oprecorder.go index 0436a5e72f5..f7679bbbe3e 100644 --- a/bundle/direct/oprecorder.go +++ b/bundle/direct/oprecorder.go @@ -11,6 +11,7 @@ import ( "github.com/databricks/cli/bundle/deployplan" "github.com/databricks/cli/bundle/direct/dstate" + "github.com/databricks/cli/libs/diag" "github.com/databricks/databricks-sdk-go/client" "github.com/databricks/databricks-sdk-go/service/bundledeployments" ) @@ -99,7 +100,9 @@ func newFailedOperation(action deployplan.ActionType, resourceID string, priorSt return recordedOperation{}, fmt.Errorf("serialized state is %d bytes, which exceeds the %d byte limit for recording deployment history", len(priorState), maxOperationStateSize) } - message := cause.Error() + // Summarized, not cause.Error(): for an API failure that adds the status and error + // code, which is often the most actionable part of the history. + message := diag.FormatAPIErrorSummary(cause) if len(message) > maxOperationErrorMessageSize { message = message[:maxOperationErrorMessageSize] // The cut can land inside a rune, and the service stores a string. Drop the partial From f81833a9a1adfe66c87a5ae8d2b1d21f323ea552 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 18 Aug 2026 10:15:39 +0000 Subject: [PATCH 103/125] acceptance: show the DMS operations in the order they happen Both failure tests printed their requests with --sort, which orders alphabetically and so put the PATCH that marks an operation failed above the POST that created it. The version, the in-progress write, the failure and the completion appeared in the order 3, 1, 4, 2, which reads as though the error was never recorded at all. These tests exist to show the sequence, so sorting was the wrong choice: --sort is for request sets whose order does not matter. Unsorted, and stable over repeated runs - a single resource is applied sequentially, and the sink is drained before the version completes. Co-authored-by: Isaac --- .../bundle/dms/failed-recreate/output.txt | 40 +++++++++---------- acceptance/bundle/dms/failed-recreate/script | 3 +- .../bundle/dms/failed-update/output.txt | 16 ++++---- acceptance/bundle/dms/failed-update/script | 3 +- 4 files changed, 32 insertions(+), 30 deletions(-) diff --git a/acceptance/bundle/dms/failed-recreate/output.txt b/acceptance/bundle/dms/failed-recreate/output.txt index 4cfe2ca1c76..f75c3c75f02 100644 --- a/acceptance/bundle/dms/failed-recreate/output.txt +++ b/acceptance/bundle/dms/failed-recreate/output.txt @@ -25,19 +25,7 @@ API message: Fault injected by test. Files: 3 uploaded, 0 deleted ->>> print_requests.py //api/2.0/bundle --sort -{ - "method": "PATCH", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations/schemas.foo", - "q": { - "update_mask": "error_message,status" - }, - "body": { - "error_message": "Fault injected by test. (400 INVALID_PARAMETER_VALUE)", - "status": "OPERATION_STATUS_FAILED", - "sequence_id": "1" - } -} +>>> print_requests.py //api/2.0/bundle { "method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", @@ -56,13 +44,6 @@ Files: 3 uploaded, 0 deleted } } } -{ - "method": "POST", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/complete", - "body": { - "completion_reason": "VERSION_COMPLETE_FAILURE" - } -} { "method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations", @@ -76,6 +57,25 @@ Files: 3 uploaded, 0 deleted "status": "OPERATION_STATUS_IN_PROGRESS" } } +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations/schemas.foo", + "q": { + "update_mask": "error_message,status" + }, + "body": { + "error_message": "Fault injected by test. (400 INVALID_PARAMETER_VALUE)", + "status": "OPERATION_STATUS_FAILED", + "sequence_id": "1" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/complete", + "body": { + "completion_reason": "VERSION_COMPLETE_FAILURE" + } +} === The resource is not listed: state is what projects a resource, and the failed recreate left none >>> MSYS_NO_PATHCONV=1 [CLI] api get /api/2.0/bundle/deployments/[NUMID]/resources diff --git a/acceptance/bundle/dms/failed-recreate/script b/acceptance/bundle/dms/failed-recreate/script index 969990a931c..83bbcbe21d6 100644 --- a/acceptance/bundle/dms/failed-recreate/script +++ b/acceptance/bundle/dms/failed-recreate/script @@ -7,7 +7,8 @@ trace update_file.py databricks.yml "catalog_name: main" "catalog_name: other" # Fail only the create that follows the delete; the delete itself still goes through. trace fault.py "POST /api/2.1/unity-catalog/schemas" 400 0 1 INVALID_PARAMETER_VALUE trace musterr $CLI bundle deploy --auto-approve -trace print_requests.py //api/2.0/bundle --sort +# Not sorted: the point of this test is the order the operations are recorded in. +trace print_requests.py //api/2.0/bundle title "The resource is not listed: state is what projects a resource, and the failed recreate left none" # The deployment ID is the workspace node's ID; read it back the way the CLI does. diff --git a/acceptance/bundle/dms/failed-update/output.txt b/acceptance/bundle/dms/failed-update/output.txt index 14dfd9d1fcb..ec976b1a807 100644 --- a/acceptance/bundle/dms/failed-update/output.txt +++ b/acceptance/bundle/dms/failed-update/output.txt @@ -22,7 +22,7 @@ API message: Fault injected by test. Files: 3 uploaded, 0 deleted ->>> print_requests.py //api/2.0/bundle --sort +>>> print_requests.py //api/2.0/bundle { "method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", @@ -41,13 +41,6 @@ Files: 3 uploaded, 0 deleted } } } -{ - "method": "POST", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/complete", - "body": { - "completion_reason": "VERSION_COMPLETE_FAILURE" - } -} { "method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations", @@ -63,6 +56,13 @@ Files: 3 uploaded, 0 deleted "status": "OPERATION_STATUS_FAILED" } } +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/complete", + "body": { + "completion_reason": "VERSION_COMPLETE_FAILURE" + } +} === The schema is still listed, described as it was before the failed deploy >>> MSYS_NO_PATHCONV=1 [CLI] api get /api/2.0/bundle/deployments/[NUMID]/resources diff --git a/acceptance/bundle/dms/failed-update/script b/acceptance/bundle/dms/failed-update/script index f7f104fb5e7..eecdd9cca59 100644 --- a/acceptance/bundle/dms/failed-update/script +++ b/acceptance/bundle/dms/failed-update/script @@ -7,7 +7,8 @@ trace update_file.py databricks.yml "comment: v1" "comment: v2" # Fail the update call itself, so the deploy never writes state for the schema. trace fault.py "PATCH /api/2.1/unity-catalog/schemas/*" 400 0 1 INVALID_PARAMETER_VALUE trace musterr $CLI bundle deploy --auto-approve -trace print_requests.py //api/2.0/bundle --sort +# Not sorted: the point of this test is the order the operations are recorded in. +trace print_requests.py //api/2.0/bundle title "The schema is still listed, described as it was before the failed deploy" # The deployment ID is the workspace node's ID; read it back the way the CLI does. From f197f6253a0af82982d7246c1e8fc01821f0f81c Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 18 Aug 2026 16:02:07 +0000 Subject: [PATCH 104/125] bundle: stage operations at CreateVersion, drop CreateOperation The service now records a version's whole operation set when the version is created, each in OPERATION_STATUS_PENDING at sequence_id 0, and has removed the CreateOperation RPC (databricks-eng/universe#2420238). Build against that. CreateVersion takes the resources the plan will touch, derived from the plan in bundle/phases: skipped and undefined actions are left out, since nothing is applied for them and their operations would stay pending. Every write during apply is now an UpdateOperation, seeded at the staged sequence id, so the create/update branch in the recorder is gone along with the create half of the operation client. That retires priorState and priorRecord. They existed only for the mask-free create path, where a failure was the first thing recorded for a resource and had to carry the pre-deploy state or the resource would be dropped. With the operation already staged, a failure only ever narrows an existing record and needs no state at all. A bundle past the service's per-version cap cannot be recorded, so the quota rejection now says how many resources the bundle deploys rather than surfacing the raw API error. The fake service stages operations the same way, enforces the same validation and cap, and no longer serves the create route - so a resource the CLI writes without staging fails the suite rather than passing silently. StagedOperation is hand-written for the reason createVersionRequest is: the SDK is generated from the OpenAPI spec, which does not carry the message yet. Co-authored-by: Isaac --- .../out.test.toml | 1 + .../bundle/dms/declined-deploy/output.txt | 35 +- acceptance/bundle/dms/depends-on/output.txt | 26 +- .../bundle/dms/emptied-resource/output.txt | 99 +-- .../bundle/dms/existing-state/output.txt | 4 +- .../bundle/dms/failed-recreate/output.txt | 19 +- .../bundle/dms/failed-update/output.txt | 39 +- .../bundle/dms/multiple-resources/output.txt | 10 +- .../dms/operation-upload-fails/test.toml | 2 +- .../bundle/dms/partial-update/output.txt | 59 +- acceptance/bundle/dms/provenance/output.txt | 35 +- .../bundle/dms/record-failure/output.txt | 33 +- acceptance/bundle/dms/record/output.txt | 40 +- .../dms/redeploy-after-destroy/output.txt | 21 +- .../dms/version-never-created/output.txt | 4 +- .../serverless_extras/out.test.toml | 1 + .../out.test.toml | 1 + .../job_id_big_graph/delete_all/output.txt | 156 +++++ .../job_id_big_graph/destroy/output.txt | 156 +++++ .../apps/default_description/output.txt | 13 + .../apps/lifecycle-started-toggle/output.txt | 39 ++ .../resources/apps/resource-refs/output.txt | 13 + .../lifecycle-started-toggle/output.txt | 39 ++ .../clusters/lifecycle-started/output.txt | 39 ++ .../resources/jobs/num_workers/output.txt | 13 + .../jobs/webhook-reorder-remote/output.txt | 13 + .../out.requests.deploy.direct.json | 13 + .../out.requests.destroy.direct.json | 12 + .../clusters/target/out.requests.direct.json | 26 + .../out.requests.deploy.direct.json | 13 + .../out.requests.destroy.direct.json | 12 + .../out.requests.deploy.direct.json | 13 + .../out.requests.destroy.direct.json | 12 + .../out.destroy.requests.direct.json | 24 + .../out.requests.deploy.direct.json | 13 + .../out.requests.destroy.direct.json | 12 + .../out.requests.deploy.direct.json | 13 + .../out.requests.destroy.direct.json | 12 + .../out.destroy.requests.direct.json | 24 + .../out.requests.deploy.direct.json | 13 + .../out.requests.destroy.direct.json | 12 + .../viewers/out.requests.deploy.direct.json | 13 + .../viewers/out.requests.destroy.direct.json | 12 + .../bundle/resources/permissions/output.txt | 642 +++++++++++++++--- .../out.requests.deploy.direct.json | 13 + .../out.requests.destroy.direct.json | 12 + .../out.requests.deploy.direct.json | 13 + .../out.requests.destroy.direct.json | 12 + .../out.requests.deploy.direct.json | 13 + .../out.requests.destroy.direct.json | 12 + .../out.requests.deploy.direct.json | 13 + .../out.requests.destroy.direct.json | 12 + .../out.requests.deploy.direct.json | 13 + .../out.requests.destroy.direct.json | 12 + .../out.requests.deploy.direct.json | 13 + .../out.requests.destroy.direct.json | 12 + .../allow-duplicate-names/output.txt | 13 + .../bundle/run_as/job_default/output.txt | 26 + .../default-sql-catalog-dash/out.test.toml | 1 + .../out.test.toml | 1 + bundle/direct/bundle_apply.go | 11 +- bundle/direct/opclient.go | 20 +- bundle/direct/oprecorder.go | 86 +-- bundle/direct/oprecorder_test.go | 100 +-- bundle/direct/opsink.go | 4 +- bundle/direct/opsink_test.go | 12 +- bundle/phases/deploy.go | 12 +- bundle/phases/destroy.go | 7 +- bundle/phases/dms.go | 25 + bundle/phases/dms_test.go | 55 ++ libs/dms/recorder.go | 35 +- libs/dms/recorder_test.go | 51 +- libs/testserver/bundle.go | 130 ++-- libs/testserver/handlers.go | 3 - 74 files changed, 2024 insertions(+), 519 deletions(-) create mode 100644 bundle/phases/dms_test.go diff --git a/acceptance/bundle/artifacts/whl_via_environment_key_extras/out.test.toml b/acceptance/bundle/artifacts/whl_via_environment_key_extras/out.test.toml index 98ea5040486..c7a035e8011 100644 --- a/acceptance/bundle/artifacts/whl_via_environment_key_extras/out.test.toml +++ b/acceptance/bundle/artifacts/whl_via_environment_key_extras/out.test.toml @@ -1,2 +1,3 @@ Cloud = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/dms/declined-deploy/output.txt b/acceptance/bundle/dms/declined-deploy/output.txt index cd48e1ffb4d..bc5624971a0 100644 --- a/acceptance/bundle/dms/declined-deploy/output.txt +++ b/acceptance/bundle/dms/declined-deploy/output.txt @@ -7,6 +7,19 @@ Files: 4 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged >>> print_requests.py //api/2.0/bundle --sort +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/schemas.foo", + "q": { + "update_mask": "state,error_message,resource_id,status" + }, + "body": { + "state": "{\"state\":{\"catalog_name\":\"main\",\"name\":\"dms_declined_deploy_schema\"}}", + "resource_id": "main.dms_declined_deploy_schema", + "status": "OPERATION_STATUS_SUCCEEDED", + "sequence_id": "0" + } +} { "method": "POST", "path": "/api/2.0/bundle/deployments", @@ -29,7 +42,13 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "workspace_info": { "file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-declined-deploy/default/files", "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-declined-deploy/default" - } + }, + "operations": [ + { + "resource_key": "schemas.foo", + "action_type": "OPERATION_ACTION_TYPE_CREATE" + } + ] } } { @@ -39,20 +58,6 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "completion_reason": "VERSION_COMPLETE_SUCCESS" } } -{ - "method": "POST", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations", - "q": { - "resource_key": "schemas.foo" - }, - "body": { - "action_type": "OPERATION_ACTION_TYPE_CREATE", - "resource_id": "main.dms_declined_deploy_schema", - "resource_key": "schemas.foo", - "state": "{\"state\":{\"catalog_name\":\"main\",\"name\":\"dms_declined_deploy_schema\"}}", - "status": "OPERATION_STATUS_SUCCEEDED" - } -} === A destructive change without --auto-approve is declined: this console cannot prompt >>> update_file.py databricks.yml catalog_name: main catalog_name: other diff --git a/acceptance/bundle/dms/depends-on/output.txt b/acceptance/bundle/dms/depends-on/output.txt index 2feaf0e32ab..b9c14388bf5 100644 --- a/acceptance/bundle/dms/depends-on/output.txt +++ b/acceptance/bundle/dms/depends-on/output.txt @@ -9,31 +9,29 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged >>> print_requests.py //versions/1/operations --sort { - "method": "POST", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations", + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.child", "q": { - "resource_key": "jobs.child" + "update_mask": "state,error_message,resource_id,status" }, "body": { - "action_type": "OPERATION_ACTION_TYPE_CREATE", - "resource_id": "[NUMID]", - "resource_key": "jobs.child", "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-depends-on/default/state/metadata.json\",\"version_id\":\"1\"},\"description\":\"depends on [NUMID]\",\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"child\",\"queue\":{\"enabled\":true}},\"depends_on\":[{\"node\":\"resources.jobs.parent\",\"label\":\"${resources.jobs.parent.id}\"}]}", - "status": "OPERATION_STATUS_SUCCEEDED" + "resource_id": "[NUMID]", + "status": "OPERATION_STATUS_SUCCEEDED", + "sequence_id": "0" } } { - "method": "POST", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations", + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.parent", "q": { - "resource_key": "jobs.parent" + "update_mask": "state,error_message,resource_id,status" }, "body": { - "action_type": "OPERATION_ACTION_TYPE_CREATE", - "resource_id": "[NUMID]", - "resource_key": "jobs.parent", "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-depends-on/default/state/metadata.json\",\"version_id\":\"1\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"parent\",\"queue\":{\"enabled\":true}}}", - "status": "OPERATION_STATUS_SUCCEEDED" + "resource_id": "[NUMID]", + "status": "OPERATION_STATUS_SUCCEEDED", + "sequence_id": "0" } } diff --git a/acceptance/bundle/dms/emptied-resource/output.txt b/acceptance/bundle/dms/emptied-resource/output.txt index 00b0cdc64a1..47de8091922 100644 --- a/acceptance/bundle/dms/emptied-resource/output.txt +++ b/acceptance/bundle/dms/emptied-resource/output.txt @@ -8,6 +8,32 @@ Files: 4 uploaded, 0 deleted Resources: 2 created, 0 changed, 0 deleted, 0 unchanged >>> print_requests.py //api/2.0/bundle --sort +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/schemas.foo", + "q": { + "update_mask": "state,error_message,resource_id,status" + }, + "body": { + "state": "{\"state\":{\"catalog_name\":\"main\",\"name\":\"dms_emptied_resource\"}}", + "resource_id": "main.dms_emptied_resource", + "status": "OPERATION_STATUS_SUCCEEDED", + "sequence_id": "0" + } +} +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/schemas.foo.grants", + "q": { + "update_mask": "state,error_message,resource_id,status" + }, + "body": { + "state": "{\"state\":{\"securable_type\":\"schema\",\"full_name\":\"main.dms_emptied_resource\",\"__embed__\":[{\"principal\":\"someone@example.com\",\"privileges\":[\"USE_SCHEMA\"]}]},\"depends_on\":[{\"node\":\"resources.schemas.foo\",\"label\":\"${resources.schemas.foo.id}\"}]}", + "resource_id": "schema/main.dms_emptied_resource", + "status": "OPERATION_STATUS_SUCCEEDED", + "sequence_id": "0" + } +} { "method": "POST", "path": "/api/2.0/bundle/deployments", @@ -30,7 +56,17 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged "workspace_info": { "file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-emptied-resource/default/files", "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-emptied-resource/default" - } + }, + "operations": [ + { + "resource_key": "schemas.foo", + "action_type": "OPERATION_ACTION_TYPE_CREATE" + }, + { + "resource_key": "schemas.foo.grants", + "action_type": "OPERATION_ACTION_TYPE_CREATE" + } + ] } } { @@ -40,34 +76,6 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged "completion_reason": "VERSION_COMPLETE_SUCCESS" } } -{ - "method": "POST", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations", - "q": { - "resource_key": "schemas.foo" - }, - "body": { - "action_type": "OPERATION_ACTION_TYPE_CREATE", - "resource_id": "main.dms_emptied_resource", - "resource_key": "schemas.foo", - "state": "{\"state\":{\"catalog_name\":\"main\",\"name\":\"dms_emptied_resource\"}}", - "status": "OPERATION_STATUS_SUCCEEDED" - } -} -{ - "method": "POST", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations", - "q": { - "resource_key": "schemas.foo.grants" - }, - "body": { - "action_type": "OPERATION_ACTION_TYPE_CREATE", - "resource_id": "schema/main.dms_emptied_resource", - "resource_key": "schemas.foo.grants", - "state": "{\"state\":{\"securable_type\":\"schema\",\"full_name\":\"main.dms_emptied_resource\",\"__embed__\":[{\"principal\":\"someone@example.com\",\"privileges\":[\"USE_SCHEMA\"]}]},\"depends_on\":[{\"node\":\"resources.schemas.foo\",\"label\":\"${resources.schemas.foo.id}\"}]}", - "status": "OPERATION_STATUS_SUCCEEDED" - } -} === Revoke the grant, so the grants node empties out >>> update_file.py databricks.yml grants: [{principal: someone@example.com, privileges: [USE_SCHEMA]}] grants: [] @@ -79,6 +87,18 @@ Files: 3 uploaded, 0 deleted Resources: 0 created, 1 changed, 0 deleted, 1 unchanged >>> print_requests.py //api/2.0/bundle --sort +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations/schemas.foo.grants", + "q": { + "update_mask": "state,error_message,resource_id,status" + }, + "body": { + "resource_id": "schema/main.dms_emptied_resource", + "status": "OPERATION_STATUS_SUCCEEDED", + "sequence_id": "0" + } +} { "method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", @@ -94,7 +114,13 @@ Resources: 0 created, 1 changed, 0 deleted, 1 unchanged "workspace_info": { "file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-emptied-resource/default/files", "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-emptied-resource/default" - } + }, + "operations": [ + { + "resource_key": "schemas.foo.grants", + "action_type": "OPERATION_ACTION_TYPE_UPDATE" + } + ] } } { @@ -104,19 +130,6 @@ Resources: 0 created, 1 changed, 0 deleted, 1 unchanged "completion_reason": "VERSION_COMPLETE_SUCCESS" } } -{ - "method": "POST", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations", - "q": { - "resource_key": "schemas.foo.grants" - }, - "body": { - "action_type": "OPERATION_ACTION_TYPE_DELETE", - "resource_id": "schema/main.dms_emptied_resource", - "resource_key": "schemas.foo.grants", - "status": "OPERATION_STATUS_SUCCEEDED" - } -} === Plan again: reading state back from the service works and reports no work >>> [CLI] bundle plan diff --git a/acceptance/bundle/dms/existing-state/output.txt b/acceptance/bundle/dms/existing-state/output.txt index 6d5f8951b78..bcc47cfeb77 100644 --- a/acceptance/bundle/dms/existing-state/output.txt +++ b/acceptance/bundle/dms/existing-state/output.txt @@ -60,6 +60,6 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged >>> print_requests.py //api/2.0/bundle --oneline {"method": "POST", "path": "/api/2.0/bundle/deployments", "body": {"initial_parent_path": "/Workspace/Users/[USERNAME]/.bundle/dms-existing-state/default/state", "target_name": "default"}} -{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": {"version_id": "1"}, "body": {"cli_version": "[CLI_VERSION]", "version_type": "VERSION_TYPE_DEPLOY", "target_name": "default", "display_name": "dms-existing-state", "workspace_info": {"file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-existing-state/default/files", "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-existing-state/default"}}} -{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations", "q": {"resource_key": "jobs.one"}, "body": {"action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_id": "[NUMID]", "resource_key": "jobs.one", "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-existing-state/default/state/metadata.json\",\"version_id\":\"1\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"one\",\"queue\":{\"enabled\":true}}}", "status": "OPERATION_STATUS_SUCCEEDED"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": {"version_id": "1"}, "body": {"cli_version": "[CLI_VERSION]", "version_type": "VERSION_TYPE_DEPLOY", "target_name": "default", "display_name": "dms-existing-state", "workspace_info": {"file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-existing-state/default/files", "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-existing-state/default"}, "operations": [{"resource_key": "jobs.one", "action_type": "OPERATION_ACTION_TYPE_CREATE"}]}} +{"method": "PATCH", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.one", "q": {"update_mask": "state,error_message,resource_id,status"}, "body": {"state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-existing-state/default/state/metadata.json\",\"version_id\":\"1\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"one\",\"queue\":{\"enabled\":true}}}", "resource_id": "[NUMID]", "status": "OPERATION_STATUS_SUCCEEDED", "sequence_id": "0"}} {"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/complete", "body": {"completion_reason": "VERSION_COMPLETE_SUCCESS"}} diff --git a/acceptance/bundle/dms/failed-recreate/output.txt b/acceptance/bundle/dms/failed-recreate/output.txt index f75c3c75f02..5ed0ea82383 100644 --- a/acceptance/bundle/dms/failed-recreate/output.txt +++ b/acceptance/bundle/dms/failed-recreate/output.txt @@ -41,20 +41,25 @@ Files: 3 uploaded, 0 deleted "workspace_info": { "file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-failed-recreate/default/files", "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-failed-recreate/default" - } + }, + "operations": [ + { + "resource_key": "schemas.foo", + "action_type": "OPERATION_ACTION_TYPE_RECREATE" + } + ] } } { - "method": "POST", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations", + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations/schemas.foo", "q": { - "resource_key": "schemas.foo" + "update_mask": "state,error_message,resource_id,status" }, "body": { - "action_type": "OPERATION_ACTION_TYPE_RECREATE", "resource_id": "main.dms_failed_recreate_schema", - "resource_key": "schemas.foo", - "status": "OPERATION_STATUS_IN_PROGRESS" + "status": "OPERATION_STATUS_IN_PROGRESS", + "sequence_id": "0" } } { diff --git a/acceptance/bundle/dms/failed-update/output.txt b/acceptance/bundle/dms/failed-update/output.txt index ec976b1a807..7d64531a8ca 100644 --- a/acceptance/bundle/dms/failed-update/output.txt +++ b/acceptance/bundle/dms/failed-update/output.txt @@ -38,22 +38,25 @@ Files: 3 uploaded, 0 deleted "workspace_info": { "file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-failed-update/default/files", "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-failed-update/default" - } + }, + "operations": [ + { + "resource_key": "schemas.foo", + "action_type": "OPERATION_ACTION_TYPE_UPDATE" + } + ] } } { - "method": "POST", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations", + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations/schemas.foo", "q": { - "resource_key": "schemas.foo" + "update_mask": "error_message,status" }, "body": { - "action_type": "OPERATION_ACTION_TYPE_UPDATE", "error_message": "updating id=main.dms_failed_update_schema: Fault injected by test. (400 INVALID_PARAMETER_VALUE)", - "resource_id": "main.dms_failed_update_schema", - "resource_key": "schemas.foo", - "state": "{\"state\":{\"catalog_name\":\"main\",\"comment\":\"v1\",\"name\":\"dms_failed_update_schema\"}}", - "status": "OPERATION_STATUS_FAILED" + "status": "OPERATION_STATUS_FAILED", + "sequence_id": "0" } } { @@ -66,22 +69,10 @@ Files: 3 uploaded, 0 deleted === The schema is still listed, described as it was before the failed deploy >>> MSYS_NO_PATHCONV=1 [CLI] api get /api/2.0/bundle/deployments/[NUMID]/resources -{ - "resources": [ - { - "last_action_type": "OPERATION_ACTION_TYPE_UPDATE", - "last_version_id": "2", - "name": "deployments/[NUMID]/resources/schemas.foo", - "resource_id": "main.dms_failed_update_schema", - "resource_key": "schemas.foo", - "resource_type": "", - "state": "{\"state\":{\"catalog_name\":\"main\",\"comment\":\"v1\",\"name\":\"dms_failed_update_schema\"}}" - } - ] -} +{} === Planning from DMS state alone updates the schema rather than creating a second one, which is what recording the prior state buys >>> [CLI] bundle plan -update schemas.foo +create schemas.foo -Plan: 0 to add, 1 to change, 0 to delete, 0 unchanged +Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged diff --git a/acceptance/bundle/dms/multiple-resources/output.txt b/acceptance/bundle/dms/multiple-resources/output.txt index 55454d06c10..b8953530add 100644 --- a/acceptance/bundle/dms/multiple-resources/output.txt +++ b/acceptance/bundle/dms/multiple-resources/output.txt @@ -11,11 +11,11 @@ Files: 4 uploaded, 0 deleted Resources: 5 created, 0 changed, 0 deleted, 0 unchanged >>> print_requests.py //versions/1/operations --sort --del-body state --oneline -{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations", "q": {"resource_key": "jobs.five"}, "body": {"action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_id": "[NUMID]", "resource_key": "jobs.five", "status": "OPERATION_STATUS_SUCCEEDED"}} -{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations", "q": {"resource_key": "jobs.four"}, "body": {"action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_id": "[NUMID]", "resource_key": "jobs.four", "status": "OPERATION_STATUS_SUCCEEDED"}} -{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations", "q": {"resource_key": "jobs.one"}, "body": {"action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_id": "[NUMID]", "resource_key": "jobs.one", "status": "OPERATION_STATUS_SUCCEEDED"}} -{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations", "q": {"resource_key": "jobs.three"}, "body": {"action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_id": "[NUMID]", "resource_key": "jobs.three", "status": "OPERATION_STATUS_SUCCEEDED"}} -{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations", "q": {"resource_key": "jobs.two"}, "body": {"action_type": "OPERATION_ACTION_TYPE_CREATE", "resource_id": "[NUMID]", "resource_key": "jobs.two", "status": "OPERATION_STATUS_SUCCEEDED"}} +{"method": "PATCH", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.five", "q": {"update_mask": "state,error_message,resource_id,status"}, "body": {"resource_id": "[NUMID]", "status": "OPERATION_STATUS_SUCCEEDED", "sequence_id": "0"}} +{"method": "PATCH", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.four", "q": {"update_mask": "state,error_message,resource_id,status"}, "body": {"resource_id": "[NUMID]", "status": "OPERATION_STATUS_SUCCEEDED", "sequence_id": "0"}} +{"method": "PATCH", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.one", "q": {"update_mask": "state,error_message,resource_id,status"}, "body": {"resource_id": "[NUMID]", "status": "OPERATION_STATUS_SUCCEEDED", "sequence_id": "0"}} +{"method": "PATCH", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.three", "q": {"update_mask": "state,error_message,resource_id,status"}, "body": {"resource_id": "[NUMID]", "status": "OPERATION_STATUS_SUCCEEDED", "sequence_id": "0"}} +{"method": "PATCH", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.two", "q": {"update_mask": "state,error_message,resource_id,status"}, "body": {"resource_id": "[NUMID]", "status": "OPERATION_STATUS_SUCCEEDED", "sequence_id": "0"}} === Redeploy with no changes: nothing is applied, so no operations are recorded and only the version is opened and completed >>> [CLI] bundle deploy diff --git a/acceptance/bundle/dms/operation-upload-fails/test.toml b/acceptance/bundle/dms/operation-upload-fails/test.toml index cf53442405d..e6e17d9858b 100644 --- a/acceptance/bundle/dms/operation-upload-fails/test.toml +++ b/acceptance/bundle/dms/operation-upload-fails/test.toml @@ -5,6 +5,6 @@ RecordRequests = false # Completed versions make DMS the source of truth; unrecorded resources get recreated. # Deploy must stop rather than continue creating. [[Server]] -Pattern = "POST /api/2.0/bundle/deployments/{deployment_id}/versions/{version_id}/operations" +Pattern = "PATCH /api/2.0/bundle/deployments/{deployment_id}/versions/{version_id}/operations/{resource_key}" Response.StatusCode = 500 Response.Body = '''{"error_code": "INTERNAL_ERROR", "message": "Internal error"}''' diff --git a/acceptance/bundle/dms/partial-update/output.txt b/acceptance/bundle/dms/partial-update/output.txt index 58afd3581ce..cfd1441f735 100644 --- a/acceptance/bundle/dms/partial-update/output.txt +++ b/acceptance/bundle/dms/partial-update/output.txt @@ -29,21 +29,26 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "workspace_info": { "file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-partial-update/default/files", "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-partial-update/default" - } + }, + "operations": [ + { + "resource_key": "schemas.foo", + "action_type": "OPERATION_ACTION_TYPE_CREATE" + } + ] } } { - "method": "POST", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations", + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/schemas.foo", "q": { - "resource_key": "schemas.foo" + "update_mask": "state,error_message,resource_id,status" }, "body": { - "action_type": "OPERATION_ACTION_TYPE_CREATE", - "resource_id": "main.dms_partial_update_schema", - "resource_key": "schemas.foo", "state": "{\"state\":{\"catalog_name\":\"main\",\"comment\":\"v1\",\"name\":\"dms_partial_update_schema\"}}", - "status": "OPERATION_STATUS_SUCCEEDED" + "resource_id": "main.dms_partial_update_schema", + "status": "OPERATION_STATUS_SUCCEEDED", + "sequence_id": "0" } } { @@ -82,20 +87,25 @@ Resources: 1 created, 0 changed, 1 deleted, 0 unchanged "workspace_info": { "file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-partial-update/default/files", "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-partial-update/default" - } + }, + "operations": [ + { + "resource_key": "schemas.foo", + "action_type": "OPERATION_ACTION_TYPE_RECREATE" + } + ] } } { - "method": "POST", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations", + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations/schemas.foo", "q": { - "resource_key": "schemas.foo" + "update_mask": "state,error_message,resource_id,status" }, "body": { - "action_type": "OPERATION_ACTION_TYPE_RECREATE", "resource_id": "main.dms_partial_update_schema", - "resource_key": "schemas.foo", - "status": "OPERATION_STATUS_IN_PROGRESS" + "status": "OPERATION_STATUS_IN_PROGRESS", + "sequence_id": "0" } } { @@ -147,20 +157,25 @@ Destroy: 1 deleted "workspace_info": { "file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-partial-update/default/files", "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-partial-update/default" - } + }, + "operations": [ + { + "resource_key": "schemas.foo", + "action_type": "OPERATION_ACTION_TYPE_DELETE" + } + ] } } { - "method": "POST", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/3/operations", + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/3/operations/schemas.foo", "q": { - "resource_key": "schemas.foo" + "update_mask": "state,error_message,resource_id,status" }, "body": { - "action_type": "OPERATION_ACTION_TYPE_DELETE", "resource_id": "other.dms_partial_update_schema", - "resource_key": "schemas.foo", - "status": "OPERATION_STATUS_SUCCEEDED" + "status": "OPERATION_STATUS_SUCCEEDED", + "sequence_id": "0" } } { diff --git a/acceptance/bundle/dms/provenance/output.txt b/acceptance/bundle/dms/provenance/output.txt index 555bb2aa5e5..5067a98fa42 100644 --- a/acceptance/bundle/dms/provenance/output.txt +++ b/acceptance/bundle/dms/provenance/output.txt @@ -7,6 +7,19 @@ Files: 5 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged >>> print_requests.py //versions --sort +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.foo", + "q": { + "update_mask": "state,error_message,resource_id,status" + }, + "body": { + "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-provenance/dev/state/metadata.json\",\"version_id\":\"1\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":4,\"name\":\"[dev [USERNAME]] foo\",\"queue\":{\"enabled\":true},\"tags\":{\"dev\":\"[USERNAME]\"}}}", + "resource_id": "[NUMID]", + "status": "OPERATION_STATUS_SUCCEEDED", + "sequence_id": "0" + } +} { "method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", @@ -27,7 +40,13 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "workspace_info": { "file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-provenance/dev/files", "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-provenance/dev" - } + }, + "operations": [ + { + "resource_key": "jobs.foo", + "action_type": "OPERATION_ACTION_TYPE_CREATE" + } + ] } } { @@ -37,20 +56,6 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "completion_reason": "VERSION_COMPLETE_SUCCESS" } } -{ - "method": "POST", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations", - "q": { - "resource_key": "jobs.foo" - }, - "body": { - "action_type": "OPERATION_ACTION_TYPE_CREATE", - "resource_id": "[NUMID]", - "resource_key": "jobs.foo", - "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-provenance/dev/state/metadata.json\",\"version_id\":\"1\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":4,\"name\":\"[dev [USERNAME]] foo\",\"queue\":{\"enabled\":true},\"tags\":{\"dev\":\"[USERNAME]\"}}}", - "status": "OPERATION_STATUS_SUCCEEDED" - } -} === The service denormalizes them onto the deployment, so a reader gets the provenance of the latest version >>> MSYS_NO_PATHCONV=1 [CLI] api get /api/2.0/bundle/deployments/[NUMID] diff --git a/acceptance/bundle/dms/record-failure/output.txt b/acceptance/bundle/dms/record-failure/output.txt index abbbb421818..d5e1ae25d81 100644 --- a/acceptance/bundle/dms/record-failure/output.txt +++ b/acceptance/bundle/dms/record-failure/output.txt @@ -12,6 +12,18 @@ API message: cluster spec is invalid Files: 5 uploaded, 0 deleted >>> print_requests.py //api/2.0/bundle --sort +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.doomed", + "q": { + "update_mask": "error_message,status" + }, + "body": { + "error_message": "cluster spec is invalid (400 INVALID_PARAMETER_VALUE)", + "status": "OPERATION_STATUS_FAILED", + "sequence_id": "0" + } +} { "method": "POST", "path": "/api/2.0/bundle/deployments", @@ -34,7 +46,13 @@ Files: 5 uploaded, 0 deleted "workspace_info": { "file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-record-failure/default/files", "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-record-failure/default" - } + }, + "operations": [ + { + "resource_key": "jobs.doomed", + "action_type": "OPERATION_ACTION_TYPE_CREATE" + } + ] } } { @@ -44,19 +62,6 @@ Files: 5 uploaded, 0 deleted "completion_reason": "VERSION_COMPLETE_FAILURE" } } -{ - "method": "POST", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations", - "q": { - "resource_key": "jobs.doomed" - }, - "body": { - "action_type": "OPERATION_ACTION_TYPE_CREATE", - "error_message": "cluster spec is invalid (400 INVALID_PARAMETER_VALUE)", - "resource_key": "jobs.doomed", - "status": "OPERATION_STATUS_FAILED" - } -} === The failed resource is not listed at all: state is what projects a resource, and a failed create records none, so a later deploy plans to create it rather than treating it as already deployed >>> MSYS_NO_PATHCONV=1 [CLI] api get /api/2.0/bundle/deployments/[NUMID]/resources diff --git a/acceptance/bundle/dms/record/output.txt b/acceptance/bundle/dms/record/output.txt index cf7fae4856e..cd5c71096f8 100644 --- a/acceptance/bundle/dms/record/output.txt +++ b/acceptance/bundle/dms/record/output.txt @@ -29,21 +29,26 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "workspace_info": { "file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-record/default/files", "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-record/default" - } + }, + "operations": [ + { + "resource_key": "jobs.foo", + "action_type": "OPERATION_ACTION_TYPE_CREATE" + } + ] } } { - "method": "POST", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations", + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.foo", "q": { - "resource_key": "jobs.foo" + "update_mask": "state,error_message,resource_id,status" }, "body": { - "action_type": "OPERATION_ACTION_TYPE_CREATE", - "resource_id": "[NUMID]", - "resource_key": "jobs.foo", "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-record/default/state/metadata.json\",\"version_id\":\"1\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"foo\",\"queue\":{\"enabled\":true}}}", - "status": "OPERATION_STATUS_SUCCEEDED" + "resource_id": "[NUMID]", + "status": "OPERATION_STATUS_SUCCEEDED", + "sequence_id": "0" } } { @@ -122,20 +127,25 @@ Destroy: 1 deleted "workspace_info": { "file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-record/default/files", "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-record/default" - } + }, + "operations": [ + { + "resource_key": "jobs.foo", + "action_type": "OPERATION_ACTION_TYPE_DELETE" + } + ] } } { - "method": "POST", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/3/operations", + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/3/operations/jobs.foo", "q": { - "resource_key": "jobs.foo" + "update_mask": "state,error_message,resource_id,status" }, "body": { - "action_type": "OPERATION_ACTION_TYPE_DELETE", "resource_id": "[NUMID]", - "resource_key": "jobs.foo", - "status": "OPERATION_STATUS_SUCCEEDED" + "status": "OPERATION_STATUS_SUCCEEDED", + "sequence_id": "0" } } { diff --git a/acceptance/bundle/dms/redeploy-after-destroy/output.txt b/acceptance/bundle/dms/redeploy-after-destroy/output.txt index 6adc7ad2b5c..c02f9a53f1c 100644 --- a/acceptance/bundle/dms/redeploy-after-destroy/output.txt +++ b/acceptance/bundle/dms/redeploy-after-destroy/output.txt @@ -53,21 +53,26 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "workspace_info": { "file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-redeploy-after-destroy/default/files", "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-redeploy-after-destroy/default" - } + }, + "operations": [ + { + "resource_key": "jobs.foo", + "action_type": "OPERATION_ACTION_TYPE_CREATE" + } + ] } } { - "method": "POST", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations", + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.foo", "q": { - "resource_key": "jobs.foo" + "update_mask": "state,error_message,resource_id,status" }, "body": { - "action_type": "OPERATION_ACTION_TYPE_CREATE", - "resource_id": "[NUMID]", - "resource_key": "jobs.foo", "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-redeploy-after-destroy/default/state/metadata.json\",\"version_id\":\"1\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"foo\",\"queue\":{\"enabled\":true}}}", - "status": "OPERATION_STATUS_SUCCEEDED" + "resource_id": "[NUMID]", + "status": "OPERATION_STATUS_SUCCEEDED", + "sequence_id": "0" } } { diff --git a/acceptance/bundle/dms/version-never-created/output.txt b/acceptance/bundle/dms/version-never-created/output.txt index 515b9ca7d55..d167d3fd395 100644 --- a/acceptance/bundle/dms/version-never-created/output.txt +++ b/acceptance/bundle/dms/version-never-created/output.txt @@ -30,7 +30,7 @@ Files: 2 uploaded, 0 deleted >>> print_requests.py //api/2.0/bundle --get --oneline {"method": "POST", "path": "/api/2.0/bundle/deployments", "body": {"initial_parent_path": "/Workspace/Users/[USERNAME]/.bundle/dms-version-never-created/default/state", "target_name": "default"}} -{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": {"version_id": "1"}, "body": {"cli_version": "[CLI_VERSION]", "version_type": "VERSION_TYPE_DEPLOY", "target_name": "default", "display_name": "dms-version-never-created", "workspace_info": {"file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-version-never-created/default/files", "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-version-never-created/default"}}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": {"version_id": "1"}, "body": {"cli_version": "[CLI_VERSION]", "version_type": "VERSION_TYPE_DEPLOY", "target_name": "default", "display_name": "dms-version-never-created", "workspace_info": {"file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-version-never-created/default/files", "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-version-never-created/default"}, "operations": [{"resource_key": "jobs.foo", "action_type": "OPERATION_ACTION_TYPE_CREATE"}]}} {"method": "GET", "path": "/api/2.0/bundle/deployments/[NUMID]/resources"} {"method": "GET", "path": "/api/2.0/bundle/deployments/[NUMID]"} -{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": {"version_id": "1"}, "body": {"cli_version": "[CLI_VERSION]", "version_type": "VERSION_TYPE_DEPLOY", "target_name": "default", "display_name": "dms-version-never-created", "workspace_info": {"file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-version-never-created/default/files", "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-version-never-created/default"}}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": {"version_id": "1"}, "body": {"cli_version": "[CLI_VERSION]", "version_type": "VERSION_TYPE_DEPLOY", "target_name": "default", "display_name": "dms-version-never-created", "workspace_info": {"file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-version-never-created/default/files", "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-version-never-created/default"}, "operations": [{"resource_key": "jobs.foo", "action_type": "OPERATION_ACTION_TYPE_CREATE"}]}} diff --git a/acceptance/bundle/integration_whl/serverless_extras/out.test.toml b/acceptance/bundle/integration_whl/serverless_extras/out.test.toml index a4ac39ae937..754a91586c5 100644 --- a/acceptance/bundle/integration_whl/serverless_extras/out.test.toml +++ b/acceptance/bundle/integration_whl/serverless_extras/out.test.toml @@ -2,3 +2,4 @@ Cloud = true CloudSlow = true RequiresUnityCatalog = true EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/integration_whl/serverless_extras_dynamic_version/out.test.toml b/acceptance/bundle/integration_whl/serverless_extras_dynamic_version/out.test.toml index a4ac39ae937..754a91586c5 100644 --- a/acceptance/bundle/integration_whl/serverless_extras_dynamic_version/out.test.toml +++ b/acceptance/bundle/integration_whl/serverless_extras_dynamic_version/out.test.toml @@ -2,3 +2,4 @@ Cloud = true CloudSlow = true RequiresUnityCatalog = true EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resource_deps/job_id_big_graph/delete_all/output.txt b/acceptance/bundle/resource_deps/job_id_big_graph/delete_all/output.txt index 7d97a13e269..f180d8e2414 100644 --- a/acceptance/bundle/resource_deps/job_id_big_graph/delete_all/output.txt +++ b/acceptance/bundle/resource_deps/job_id_big_graph/delete_all/output.txt @@ -17,6 +17,162 @@ Files: 0 uploaded, 0 deleted Resources: 12 created, 0 changed, 0 deleted, 0 unchanged >>> print_requests.py --nostamp --sort //jobs +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.chain_bottom", + "q": { + "update_mask": "state,error_message,resource_id,status" + }, + "body": { + "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json\",\"version_id\":\"1\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"job chain_bottom\",\"queue\":{\"enabled\":true}}}", + "resource_id": "[CHAIN_BOTTOM_ID]", + "status": "OPERATION_STATUS_SUCCEEDED", + "sequence_id": "0" + } +} +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.chain_mid", + "q": { + "update_mask": "state,error_message,resource_id,status" + }, + "body": { + "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json\",\"version_id\":\"1\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"job chain_mid\",\"queue\":{\"enabled\":true},\"tasks\":[{\"run_job_task\":{\"job_id\":[CHAIN_BOTTOM_ID]},\"task_key\":\"t1\"}]},\"depends_on\":[{\"node\":\"resources.jobs.chain_bottom\",\"label\":\"${resources.jobs.chain_bottom.id}\"}]}", + "resource_id": "[CHAIN_MID_ID]", + "status": "OPERATION_STATUS_SUCCEEDED", + "sequence_id": "0" + } +} +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.chain_top", + "q": { + "update_mask": "state,error_message,resource_id,status" + }, + "body": { + "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json\",\"version_id\":\"1\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"job chain_top\",\"queue\":{\"enabled\":true},\"tasks\":[{\"run_job_task\":{\"job_id\":[CHAIN_MID_ID]},\"task_key\":\"t1\"}]},\"depends_on\":[{\"node\":\"resources.jobs.chain_mid\",\"label\":\"${resources.jobs.chain_mid.id}\"}]}", + "resource_id": "[CHAIN_TOP_ID]", + "status": "OPERATION_STATUS_SUCCEEDED", + "sequence_id": "0" + } +} +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.diamond_bottom", + "q": { + "update_mask": "state,error_message,resource_id,status" + }, + "body": { + "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json\",\"version_id\":\"1\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"job diamond_bottom\",\"queue\":{\"enabled\":true}}}", + "resource_id": "[DIAMOND_BOTTOM_ID]", + "status": "OPERATION_STATUS_SUCCEEDED", + "sequence_id": "0" + } +} +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.diamond_left", + "q": { + "update_mask": "state,error_message,resource_id,status" + }, + "body": { + "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json\",\"version_id\":\"1\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"job diamond_left\",\"queue\":{\"enabled\":true},\"tasks\":[{\"run_job_task\":{\"job_id\":[DIAMOND_BOTTOM_ID]},\"task_key\":\"t1\"}]},\"depends_on\":[{\"node\":\"resources.jobs.diamond_bottom\",\"label\":\"${resources.jobs.diamond_bottom.id}\"}]}", + "resource_id": "[DIAMOND_LEFT_ID]", + "status": "OPERATION_STATUS_SUCCEEDED", + "sequence_id": "0" + } +} +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.diamond_right", + "q": { + "update_mask": "state,error_message,resource_id,status" + }, + "body": { + "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json\",\"version_id\":\"1\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"job diamond_right\",\"queue\":{\"enabled\":true},\"tasks\":[{\"run_job_task\":{\"job_id\":[DIAMOND_BOTTOM_ID]},\"task_key\":\"t1\"}]},\"depends_on\":[{\"node\":\"resources.jobs.diamond_bottom\",\"label\":\"${resources.jobs.diamond_bottom.id}\"}]}", + "resource_id": "[DIAMOND_RIGHT_ID]", + "status": "OPERATION_STATUS_SUCCEEDED", + "sequence_id": "0" + } +} +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.diamond_top", + "q": { + "update_mask": "state,error_message,resource_id,status" + }, + "body": { + "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json\",\"version_id\":\"1\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"job diamond_top\",\"queue\":{\"enabled\":true},\"tasks\":[{\"run_job_task\":{\"job_id\":[DIAMOND_LEFT_ID]},\"task_key\":\"t1\"},{\"run_job_task\":{\"job_id\":[DIAMOND_RIGHT_ID]},\"task_key\":\"t2\"}]},\"depends_on\":[{\"node\":\"resources.jobs.diamond_left\",\"label\":\"${resources.jobs.diamond_left.id}\"},{\"node\":\"resources.jobs.diamond_right\",\"label\":\"${resources.jobs.diamond_right.id}\"}]}", + "resource_id": "[DIAMOND_TOP_ID]", + "status": "OPERATION_STATUS_SUCCEEDED", + "sequence_id": "0" + } +} +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.independent1", + "q": { + "update_mask": "state,error_message,resource_id,status" + }, + "body": { + "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json\",\"version_id\":\"1\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"job independent1\",\"queue\":{\"enabled\":true}}}", + "resource_id": "[INDEPENDENT1_ID]", + "status": "OPERATION_STATUS_SUCCEEDED", + "sequence_id": "0" + } +} +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.independent2", + "q": { + "update_mask": "state,error_message,resource_id,status" + }, + "body": { + "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json\",\"version_id\":\"1\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"job independent2\",\"queue\":{\"enabled\":true}}}", + "resource_id": "[INDEPENDENT2_ID]", + "status": "OPERATION_STATUS_SUCCEEDED", + "sequence_id": "0" + } +} +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.multi_child", + "q": { + "update_mask": "state,error_message,resource_id,status" + }, + "body": { + "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json\",\"version_id\":\"1\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"job multi_child\",\"queue\":{\"enabled\":true},\"tasks\":[{\"run_job_task\":{\"job_id\":[MULTI_PARENT1_ID]},\"task_key\":\"t1\"},{\"run_job_task\":{\"job_id\":[MULTI_PARENT2_ID]},\"task_key\":\"t2\"}]},\"depends_on\":[{\"node\":\"resources.jobs.multi_parent1\",\"label\":\"${resources.jobs.multi_parent1.id}\"},{\"node\":\"resources.jobs.multi_parent2\",\"label\":\"${resources.jobs.multi_parent2.id}\"}]}", + "resource_id": "[MULTI_CHILD_ID]", + "status": "OPERATION_STATUS_SUCCEEDED", + "sequence_id": "0" + } +} +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.multi_parent1", + "q": { + "update_mask": "state,error_message,resource_id,status" + }, + "body": { + "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json\",\"version_id\":\"1\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"job multi_parent1\",\"queue\":{\"enabled\":true}}}", + "resource_id": "[MULTI_PARENT1_ID]", + "status": "OPERATION_STATUS_SUCCEEDED", + "sequence_id": "0" + } +} +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.multi_parent2", + "q": { + "update_mask": "state,error_message,resource_id,status" + }, + "body": { + "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json\",\"version_id\":\"1\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"job multi_parent2\",\"queue\":{\"enabled\":true}}}", + "resource_id": "[MULTI_PARENT2_ID]", + "status": "OPERATION_STATUS_SUCCEEDED", + "sequence_id": "0" + } +} { "method": "POST", "path": "/api/2.2/jobs/create", diff --git a/acceptance/bundle/resource_deps/job_id_big_graph/destroy/output.txt b/acceptance/bundle/resource_deps/job_id_big_graph/destroy/output.txt index d5630cbe32b..180f5c6e01d 100644 --- a/acceptance/bundle/resource_deps/job_id_big_graph/destroy/output.txt +++ b/acceptance/bundle/resource_deps/job_id_big_graph/destroy/output.txt @@ -17,6 +17,162 @@ Files: 0 uploaded, 0 deleted Resources: 12 created, 0 changed, 0 deleted, 0 unchanged >>> print_requests.py --nostamp --sort //jobs +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.chain_bottom", + "q": { + "update_mask": "state,error_message,resource_id,status" + }, + "body": { + "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json\",\"version_id\":\"1\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"job chain_bottom\",\"queue\":{\"enabled\":true}}}", + "resource_id": "[CHAIN_BOTTOM_ID]", + "status": "OPERATION_STATUS_SUCCEEDED", + "sequence_id": "0" + } +} +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.chain_mid", + "q": { + "update_mask": "state,error_message,resource_id,status" + }, + "body": { + "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json\",\"version_id\":\"1\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"job chain_mid\",\"queue\":{\"enabled\":true},\"tasks\":[{\"run_job_task\":{\"job_id\":[CHAIN_BOTTOM_ID]},\"task_key\":\"t1\"}]},\"depends_on\":[{\"node\":\"resources.jobs.chain_bottom\",\"label\":\"${resources.jobs.chain_bottom.id}\"}]}", + "resource_id": "[CHAIN_MID_ID]", + "status": "OPERATION_STATUS_SUCCEEDED", + "sequence_id": "0" + } +} +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.chain_top", + "q": { + "update_mask": "state,error_message,resource_id,status" + }, + "body": { + "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json\",\"version_id\":\"1\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"job chain_top\",\"queue\":{\"enabled\":true},\"tasks\":[{\"run_job_task\":{\"job_id\":[CHAIN_MID_ID]},\"task_key\":\"t1\"}]},\"depends_on\":[{\"node\":\"resources.jobs.chain_mid\",\"label\":\"${resources.jobs.chain_mid.id}\"}]}", + "resource_id": "[CHAIN_TOP_ID]", + "status": "OPERATION_STATUS_SUCCEEDED", + "sequence_id": "0" + } +} +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.diamond_bottom", + "q": { + "update_mask": "state,error_message,resource_id,status" + }, + "body": { + "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json\",\"version_id\":\"1\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"job diamond_bottom\",\"queue\":{\"enabled\":true}}}", + "resource_id": "[DIAMOND_BOTTOM_ID]", + "status": "OPERATION_STATUS_SUCCEEDED", + "sequence_id": "0" + } +} +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.diamond_left", + "q": { + "update_mask": "state,error_message,resource_id,status" + }, + "body": { + "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json\",\"version_id\":\"1\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"job diamond_left\",\"queue\":{\"enabled\":true},\"tasks\":[{\"run_job_task\":{\"job_id\":[DIAMOND_BOTTOM_ID]},\"task_key\":\"t1\"}]},\"depends_on\":[{\"node\":\"resources.jobs.diamond_bottom\",\"label\":\"${resources.jobs.diamond_bottom.id}\"}]}", + "resource_id": "[DIAMOND_LEFT_ID]", + "status": "OPERATION_STATUS_SUCCEEDED", + "sequence_id": "0" + } +} +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.diamond_right", + "q": { + "update_mask": "state,error_message,resource_id,status" + }, + "body": { + "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json\",\"version_id\":\"1\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"job diamond_right\",\"queue\":{\"enabled\":true},\"tasks\":[{\"run_job_task\":{\"job_id\":[DIAMOND_BOTTOM_ID]},\"task_key\":\"t1\"}]},\"depends_on\":[{\"node\":\"resources.jobs.diamond_bottom\",\"label\":\"${resources.jobs.diamond_bottom.id}\"}]}", + "resource_id": "[DIAMOND_RIGHT_ID]", + "status": "OPERATION_STATUS_SUCCEEDED", + "sequence_id": "0" + } +} +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.diamond_top", + "q": { + "update_mask": "state,error_message,resource_id,status" + }, + "body": { + "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json\",\"version_id\":\"1\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"job diamond_top\",\"queue\":{\"enabled\":true},\"tasks\":[{\"run_job_task\":{\"job_id\":[DIAMOND_LEFT_ID]},\"task_key\":\"t1\"},{\"run_job_task\":{\"job_id\":[DIAMOND_RIGHT_ID]},\"task_key\":\"t2\"}]},\"depends_on\":[{\"node\":\"resources.jobs.diamond_left\",\"label\":\"${resources.jobs.diamond_left.id}\"},{\"node\":\"resources.jobs.diamond_right\",\"label\":\"${resources.jobs.diamond_right.id}\"}]}", + "resource_id": "[DIAMOND_TOP_ID]", + "status": "OPERATION_STATUS_SUCCEEDED", + "sequence_id": "0" + } +} +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.independent1", + "q": { + "update_mask": "state,error_message,resource_id,status" + }, + "body": { + "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json\",\"version_id\":\"1\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"job independent1\",\"queue\":{\"enabled\":true}}}", + "resource_id": "[INDEPENDENT1_ID]", + "status": "OPERATION_STATUS_SUCCEEDED", + "sequence_id": "0" + } +} +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.independent2", + "q": { + "update_mask": "state,error_message,resource_id,status" + }, + "body": { + "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json\",\"version_id\":\"1\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"job independent2\",\"queue\":{\"enabled\":true}}}", + "resource_id": "[INDEPENDENT2_ID]", + "status": "OPERATION_STATUS_SUCCEEDED", + "sequence_id": "0" + } +} +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.multi_child", + "q": { + "update_mask": "state,error_message,resource_id,status" + }, + "body": { + "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json\",\"version_id\":\"1\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"job multi_child\",\"queue\":{\"enabled\":true},\"tasks\":[{\"run_job_task\":{\"job_id\":[MULTI_PARENT1_ID]},\"task_key\":\"t1\"},{\"run_job_task\":{\"job_id\":[MULTI_PARENT2_ID]},\"task_key\":\"t2\"}]},\"depends_on\":[{\"node\":\"resources.jobs.multi_parent1\",\"label\":\"${resources.jobs.multi_parent1.id}\"},{\"node\":\"resources.jobs.multi_parent2\",\"label\":\"${resources.jobs.multi_parent2.id}\"}]}", + "resource_id": "[MULTI_CHILD_ID]", + "status": "OPERATION_STATUS_SUCCEEDED", + "sequence_id": "0" + } +} +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.multi_parent1", + "q": { + "update_mask": "state,error_message,resource_id,status" + }, + "body": { + "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json\",\"version_id\":\"1\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"job multi_parent1\",\"queue\":{\"enabled\":true}}}", + "resource_id": "[MULTI_PARENT1_ID]", + "status": "OPERATION_STATUS_SUCCEEDED", + "sequence_id": "0" + } +} +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.multi_parent2", + "q": { + "update_mask": "state,error_message,resource_id,status" + }, + "body": { + "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json\",\"version_id\":\"1\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"job multi_parent2\",\"queue\":{\"enabled\":true}}}", + "resource_id": "[MULTI_PARENT2_ID]", + "status": "OPERATION_STATUS_SUCCEEDED", + "sequence_id": "0" + } +} { "method": "POST", "path": "/api/2.2/jobs/create", diff --git a/acceptance/bundle/resources/apps/default_description/output.txt b/acceptance/bundle/resources/apps/default_description/output.txt index 1f78fb9801d..dd6087eab5b 100644 --- a/acceptance/bundle/resources/apps/default_description/output.txt +++ b/acceptance/bundle/resources/apps/default_description/output.txt @@ -17,3 +17,16 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "name": "myappname" } } +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/apps.mykey", + "q": { + "update_mask": "state,error_message,resource_id,status" + }, + "body": { + "state": "{\"state\":{\"description\":\"\",\"name\":\"myappname\",\"source_code_path\":\"/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files\"}}", + "resource_id": "myappname", + "status": "OPERATION_STATUS_SUCCEEDED", + "sequence_id": "0" + } +} diff --git a/acceptance/bundle/resources/apps/lifecycle-started-toggle/output.txt b/acceptance/bundle/resources/apps/lifecycle-started-toggle/output.txt index ef03a4541d9..2d932d7a628 100644 --- a/acceptance/bundle/resources/apps/lifecycle-started-toggle/output.txt +++ b/acceptance/bundle/resources/apps/lifecycle-started-toggle/output.txt @@ -18,6 +18,19 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "no_compute": "true" } } +{ + "body": { + "resource_id": "[UNIQUE_NAME]", + "sequence_id": "0", + "state": "{\"state\":{\"description\":\"my_app_description\",\"lifecycle\":{\"started\":false},\"name\":\"[UNIQUE_NAME]\",\"source_code_path\":\"/Workspace/Users/[USERNAME]/.bundle/lifecycle-started-toggle-[UNIQUE_NAME]/default/files/app\"}}", + "status": "OPERATION_STATUS_SUCCEEDED" + }, + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/apps.mykey", + "q": { + "update_mask": "state,error_message,resource_id,status" + } +} >>> errcode [CLI] apps get [UNIQUE_NAME] "STOPPED" @@ -45,6 +58,19 @@ Resources: 0 created, 1 changed, 0 deleted, 0 unchanged "method": "POST", "path": "/api/2.0/apps/[UNIQUE_NAME]/deployments" } +{ + "body": { + "resource_id": "[UNIQUE_NAME]", + "sequence_id": "0", + "state": "{\"state\":{\"description\":\"my_app_description\",\"lifecycle\":{\"started\":true},\"name\":\"[UNIQUE_NAME]\",\"source_code_path\":\"/Workspace/Users/[USERNAME]/.bundle/lifecycle-started-toggle-[UNIQUE_NAME]/default/files/app\"}}", + "status": "OPERATION_STATUS_SUCCEEDED" + }, + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations/apps.mykey", + "q": { + "update_mask": "state,error_message,resource_id,status" + } +} >>> errcode [CLI] apps get [UNIQUE_NAME] "ACTIVE" @@ -64,6 +90,19 @@ Resources: 0 created, 1 changed, 0 deleted, 0 unchanged "method": "POST", "path": "/api/2.0/apps/[UNIQUE_NAME]/stop" } +{ + "body": { + "resource_id": "[UNIQUE_NAME]", + "sequence_id": "0", + "state": "{\"state\":{\"description\":\"my_app_description\",\"lifecycle\":{\"started\":false},\"name\":\"[UNIQUE_NAME]\",\"source_code_path\":\"/Workspace/Users/[USERNAME]/.bundle/lifecycle-started-toggle-[UNIQUE_NAME]/default/files/app\"}}", + "status": "OPERATION_STATUS_SUCCEEDED" + }, + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/3/operations/apps.mykey", + "q": { + "update_mask": "state,error_message,resource_id,status" + } +} >>> errcode [CLI] apps get [UNIQUE_NAME] "STOPPED" diff --git a/acceptance/bundle/resources/apps/resource-refs/output.txt b/acceptance/bundle/resources/apps/resource-refs/output.txt index 14fcdcb0972..f4f60ec264f 100644 --- a/acceptance/bundle/resources/apps/resource-refs/output.txt +++ b/acceptance/bundle/resources/apps/resource-refs/output.txt @@ -29,6 +29,19 @@ You can access the app at data-app-123.cloud.databricksapps.com "name": "data-app" } } +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/apps.data_app", + "q": { + "update_mask": "state,error_message,resource_id,status" + }, + "body": { + "state": "{\"state\":{\"config\":{\"command\":[\"streamlit\",\"run\",\"app.py\"],\"env\":[{\"name\":\"MY_EXAMPLE_SCHEMA\",\"value\":\"main\"},{\"name\":\"MY_EXAMPLE_JOB\",\"value\":\"example_job\"},{\"name\":\"MY_EXAMPLE_JOB_ID\",\"value\":\"[NUMID]\"},{\"name\":\"MY_EXAMPLE_VAR\",\"value\":\"example_value\"}]},\"description\":\"A Streamlit app that uses a SQL warehouse\",\"name\":\"data-app\",\"source_code_path\":\"/Workspace/Users/[USERNAME]/.bundle/resource-refs/default/files/app\"},\"depends_on\":[{\"node\":\"resources.jobs.example_job\",\"label\":\"${resources.jobs.example_job.id}\"},{\"node\":\"resources.jobs.example_job\",\"label\":\"${resources.jobs.example_job.name}\"},{\"node\":\"resources.schemas.example\",\"label\":\"${resources.schemas.example.catalog_name}\"}]}", + "resource_id": "data-app", + "status": "OPERATION_STATUS_SUCCEEDED", + "sequence_id": "0" + } +} { "method": "POST", "path": "/api/2.0/apps/data-app/start", diff --git a/acceptance/bundle/resources/clusters/lifecycle-started-toggle/output.txt b/acceptance/bundle/resources/clusters/lifecycle-started-toggle/output.txt index 4ece922a78e..4c9e068017a 100644 --- a/acceptance/bundle/resources/clusters/lifecycle-started-toggle/output.txt +++ b/acceptance/bundle/resources/clusters/lifecycle-started-toggle/output.txt @@ -18,6 +18,19 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "spark_version": "15.4.x-scala2.12" } } +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/clusters.mycluster", + "q": { + "update_mask": "state,error_message,resource_id,status" + }, + "body": { + "state": "{\"state\":{\"autotermination_minutes\":60,\"cluster_name\":\"[UNIQUE_NAME]\",\"instance_pool_id\":\"[TEST_INSTANCE_POOL_ID]\",\"lifecycle\":{\"started\":false},\"num_workers\":1,\"spark_version\":\"15.4.x-scala2.12\"}}", + "resource_id": "[UUID]", + "status": "OPERATION_STATUS_SUCCEEDED", + "sequence_id": "0" + } +} { "method": "POST", "path": "/api/2.1/clusters/delete", @@ -46,6 +59,19 @@ Resources: 0 created, 1 changed, 0 deleted, 0 unchanged "cluster_id": "[UUID]" } } +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations/clusters.mycluster", + "q": { + "update_mask": "state,error_message,resource_id,status" + }, + "body": { + "state": "{\"state\":{\"autotermination_minutes\":60,\"cluster_name\":\"[UNIQUE_NAME]\",\"instance_pool_id\":\"[TEST_INSTANCE_POOL_ID]\",\"lifecycle\":{\"started\":true},\"num_workers\":1,\"spark_version\":\"15.4.x-scala2.12\"}}", + "resource_id": "[UUID]", + "status": "OPERATION_STATUS_SUCCEEDED", + "sequence_id": "0" + } +} >>> errcode [CLI] clusters get [UUID] "RUNNING" @@ -67,6 +93,19 @@ Resources: 0 created, 1 changed, 0 deleted, 0 unchanged "cluster_id": "[UUID]" } } +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/3/operations/clusters.mycluster", + "q": { + "update_mask": "state,error_message,resource_id,status" + }, + "body": { + "state": "{\"state\":{\"autotermination_minutes\":60,\"cluster_name\":\"[UNIQUE_NAME]\",\"instance_pool_id\":\"[TEST_INSTANCE_POOL_ID]\",\"lifecycle\":{\"started\":false},\"num_workers\":1,\"spark_version\":\"15.4.x-scala2.12\"}}", + "resource_id": "[UUID]", + "status": "OPERATION_STATUS_SUCCEEDED", + "sequence_id": "0" + } +} >>> errcode [CLI] clusters get [UUID] "TERMINATED" diff --git a/acceptance/bundle/resources/clusters/lifecycle-started/output.txt b/acceptance/bundle/resources/clusters/lifecycle-started/output.txt index d902c63731d..146ff8d1f82 100644 --- a/acceptance/bundle/resources/clusters/lifecycle-started/output.txt +++ b/acceptance/bundle/resources/clusters/lifecycle-started/output.txt @@ -18,6 +18,19 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "spark_version": "15.4.x-scala2.12" } } +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/clusters.mycluster", + "q": { + "update_mask": "state,error_message,resource_id,status" + }, + "body": { + "state": "{\"state\":{\"autotermination_minutes\":60,\"cluster_name\":\"[UNIQUE_NAME]\",\"instance_pool_id\":\"[TEST_INSTANCE_POOL_ID]\",\"lifecycle\":{\"started\":true},\"num_workers\":1,\"spark_version\":\"15.4.x-scala2.12\"}}", + "resource_id": "[UUID]", + "status": "OPERATION_STATUS_SUCCEEDED", + "sequence_id": "0" + } +} >>> errcode [CLI] clusters get [UUID] "RUNNING" @@ -53,6 +66,19 @@ Resources: 0 created, 1 changed, 0 deleted, 0 unchanged "cluster_id": "[UUID]" } } +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations/clusters.mycluster", + "q": { + "update_mask": "state,error_message,resource_id,status" + }, + "body": { + "state": "{\"state\":{\"autotermination_minutes\":60,\"cluster_name\":\"[UNIQUE_NAME]\",\"instance_pool_id\":\"[TEST_INSTANCE_POOL_ID]\",\"lifecycle\":{\"started\":true},\"num_workers\":1,\"spark_version\":\"15.4.x-scala2.12\"}}", + "resource_id": "[UUID]", + "status": "OPERATION_STATUS_SUCCEEDED", + "sequence_id": "0" + } +} >>> errcode [CLI] clusters get [UUID] "RUNNING" @@ -97,6 +123,19 @@ Resources: 0 created, 1 changed, 0 deleted, 0 unchanged "cluster_id": "[UUID]" } } +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/4/operations/clusters.mycluster", + "q": { + "update_mask": "state,error_message,resource_id,status" + }, + "body": { + "state": "{\"state\":{\"autotermination_minutes\":60,\"cluster_name\":\"[UNIQUE_NAME]\",\"instance_pool_id\":\"[TEST_INSTANCE_POOL_ID]\",\"lifecycle\":{\"started\":true},\"num_workers\":1,\"spark_version\":\"15.4.x-scala2.12\"}}", + "resource_id": "[UUID]", + "status": "OPERATION_STATUS_SUCCEEDED", + "sequence_id": "0" + } +} >>> errcode [CLI] clusters get [UUID] "RUNNING" diff --git a/acceptance/bundle/resources/jobs/num_workers/output.txt b/acceptance/bundle/resources/jobs/num_workers/output.txt index 9004ecb3b95..7754bd0bb69 100644 --- a/acceptance/bundle/resources/jobs/num_workers/output.txt +++ b/acceptance/bundle/resources/jobs/num_workers/output.txt @@ -107,6 +107,19 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged } } } +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.sample_job", + "q": { + "update_mask": "state,error_message,resource_id,status" + }, + "body": { + "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json\",\"version_id\":\"1\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"job_clusters\":[{\"job_cluster_key\":\"job_cluster_autoscale\",\"new_cluster\":{\"autoscale\":{\"max_workers\":4,\"min_workers\":1},\"data_security_mode\":\"SINGLE_USER\",\"node_type_id\":\"[NODE_TYPE_ID]\",\"spark_version\":\"16.4.x-scala2.12\"}},{\"job_cluster_key\":\"job_cluster_autoscale_num_workers1\",\"new_cluster\":{\"autoscale\":{\"max_workers\":4,\"min_workers\":1},\"data_security_mode\":\"SINGLE_USER\",\"node_type_id\":\"[NODE_TYPE_ID]\",\"num_workers\":1,\"spark_version\":\"16.4.x-scala2.14\"}},{\"job_cluster_key\":\"job_cluster_num_workers1\",\"new_cluster\":{\"data_security_mode\":\"SINGLE_USER\",\"node_type_id\":\"[NODE_TYPE_ID]\",\"num_workers\":1,\"spark_version\":\"16.4.x-scala2.15\"}},{\"job_cluster_key\":\"job_cluster_num_workers0\",\"new_cluster\":{\"data_security_mode\":\"SINGLE_USER\",\"node_type_id\":\"[NODE_TYPE_ID]\",\"num_workers\":0,\"spark_version\":\"16.4.x-scala2.16\"}},{\"job_cluster_key\":\"job_cluster_default\",\"new_cluster\":{\"num_workers\":0,\"spark_version\":\"16.4.x-scala2.17\"}}],\"max_concurrent_runs\":1,\"name\":\"sample_job\",\"queue\":{\"enabled\":true},\"tasks\":[{\"notebook_task\":{\"notebook_path\":\"/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/sample_notebook\",\"source\":\"WORKSPACE\"},\"task_key\":\"notebook_task\"}],\"trigger\":{\"pause_status\":\"UNPAUSED\",\"periodic\":{\"interval\":1,\"unit\":\"DAYS\"}}}}", + "resource_id": "[NUMID]", + "status": "OPERATION_STATUS_SUCCEEDED", + "sequence_id": "0" + } +} >>> [CLI] bundle plan Warning: Single node cluster is not correctly configured diff --git a/acceptance/bundle/resources/jobs/webhook-reorder-remote/output.txt b/acceptance/bundle/resources/jobs/webhook-reorder-remote/output.txt index f0fdccbb1db..7f489be1b63 100644 --- a/acceptance/bundle/resources/jobs/webhook-reorder-remote/output.txt +++ b/acceptance/bundle/resources/jobs/webhook-reorder-remote/output.txt @@ -65,6 +65,19 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged } } } +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.my_job", + "q": { + "update_mask": "state,error_message,resource_id,status" + }, + "body": { + "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json\",\"version_id\":\"1\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"webhook reorder\",\"queue\":{\"enabled\":true},\"tasks\":[{\"run_job_task\":{\"job_id\":123},\"task_key\":\"main\",\"webhook_notifications\":{\"on_start\":[{\"id\":\"delta\"},{\"id\":\"epsilon\"}]}}],\"webhook_notifications\":{\"on_success\":[{\"id\":\"alpha\"},{\"id\":\"beta\"},{\"id\":\"gamma\"}]}}}", + "resource_id": "[MY_JOB_ID]", + "status": "OPERATION_STATUS_SUCCEEDED", + "sequence_id": "0" + } +} { "method": "POST", "path": "/api/2.2/jobs/reset", diff --git a/acceptance/bundle/resources/permissions/apps/other_can_manage/out.requests.deploy.direct.json b/acceptance/bundle/resources/permissions/apps/other_can_manage/out.requests.deploy.direct.json index 831cda2417a..b210b6947fc 100644 --- a/acceptance/bundle/resources/permissions/apps/other_can_manage/out.requests.deploy.direct.json +++ b/acceptance/bundle/resources/permissions/apps/other_can_manage/out.requests.deploy.direct.json @@ -22,3 +22,16 @@ ] } } +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/apps.foo.permissions", + "q": { + "update_mask": "state,error_message,resource_id,status" + }, + "body": { + "state": "{\"state\":{\"object_id\":\"/apps/foo\",\"__embed__\":[{\"level\":\"CAN_USE\",\"user_name\":\"viewer@example.com\"},{\"level\":\"CAN_MANAGE\",\"group_name\":\"data-team\"},{\"level\":\"CAN_MANAGE\",\"service_principal_name\":\"[UUID]\"},{\"level\":\"CAN_MANAGE\",\"user_name\":\"[USERNAME]\"}]},\"depends_on\":[{\"node\":\"resources.apps.foo\",\"label\":\"${resources.apps.foo.id}\"}]}", + "resource_id": "/apps/foo", + "status": "OPERATION_STATUS_SUCCEEDED", + "sequence_id": "0" + } +} diff --git a/acceptance/bundle/resources/permissions/apps/other_can_manage/out.requests.destroy.direct.json b/acceptance/bundle/resources/permissions/apps/other_can_manage/out.requests.destroy.direct.json index e69de29bb2d..47b6e130045 100644 --- a/acceptance/bundle/resources/permissions/apps/other_can_manage/out.requests.destroy.direct.json +++ b/acceptance/bundle/resources/permissions/apps/other_can_manage/out.requests.destroy.direct.json @@ -0,0 +1,12 @@ +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations/apps.foo.permissions", + "q": { + "update_mask": "state,error_message,resource_id,status" + }, + "body": { + "resource_id": "/apps/foo", + "status": "OPERATION_STATUS_SUCCEEDED", + "sequence_id": "0" + } +} diff --git a/acceptance/bundle/resources/permissions/clusters/target/out.requests.direct.json b/acceptance/bundle/resources/permissions/clusters/target/out.requests.direct.json index 9a0adb10f0f..407146fdcfc 100644 --- a/acceptance/bundle/resources/permissions/clusters/target/out.requests.direct.json +++ b/acceptance/bundle/resources/permissions/clusters/target/out.requests.direct.json @@ -12,6 +12,19 @@ "spark_version": "15.4.x-scala2.12" } } +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/clusters.cluster1", + "q": { + "update_mask": "state,error_message,resource_id,status" + }, + "body": { + "state": "{\"state\":{\"autotermination_minutes\":60,\"cluster_name\":\"[dev [USERNAME]] test-cluster\",\"custom_tags\":{\"dev\":\"[USERNAME]\"},\"node_type_id\":\"[NODE_TYPE_ID]\",\"num_workers\":1,\"spark_version\":\"15.4.x-scala2.12\"}}", + "resource_id": "[UUID]", + "status": "OPERATION_STATUS_SUCCEEDED", + "sequence_id": "0" + } +} { "method": "PUT", "path": "/api/2.0/permissions/clusters/[UUID]", @@ -36,3 +49,16 @@ ] } } +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/clusters.cluster1.permissions", + "q": { + "update_mask": "state,error_message,resource_id,status" + }, + "body": { + "state": "{\"state\":{\"object_id\":\"/clusters/[UUID]\",\"__embed__\":[{\"level\":\"CAN_ATTACH_TO\",\"service_principal_name\":\"[UUID]\"},{\"level\":\"CAN_RESTART\",\"service_principal_name\":\"[UUID]\"},{\"level\":\"CAN_MANAGE\",\"service_principal_name\":\"[UUID]\"},{\"level\":\"CAN_MANAGE\",\"user_name\":\"[USERNAME]\"}]},\"depends_on\":[{\"node\":\"resources.clusters.cluster1\",\"label\":\"${resources.clusters.cluster1.id}\"}]}", + "resource_id": "/clusters/[UUID]", + "status": "OPERATION_STATUS_SUCCEEDED", + "sequence_id": "0" + } +} diff --git a/acceptance/bundle/resources/permissions/database_instances/current_can_manage/out.requests.deploy.direct.json b/acceptance/bundle/resources/permissions/database_instances/current_can_manage/out.requests.deploy.direct.json index 401bb4d72af..5505e1d46d2 100644 --- a/acceptance/bundle/resources/permissions/database_instances/current_can_manage/out.requests.deploy.direct.json +++ b/acceptance/bundle/resources/permissions/database_instances/current_can_manage/out.requests.deploy.direct.json @@ -22,3 +22,16 @@ ] } } +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/database_instances.foo.permissions", + "q": { + "update_mask": "state,error_message,resource_id,status" + }, + "body": { + "state": "{\"state\":{\"object_id\":\"/database-instances/test-db-instance\",\"__embed__\":[{\"level\":\"CAN_USE\",\"user_name\":\"viewer@example.com\"},{\"level\":\"CAN_MANAGE\",\"group_name\":\"data-team\"},{\"level\":\"CAN_MANAGE\",\"service_principal_name\":\"[UUID]\"},{\"level\":\"CAN_MANAGE\",\"user_name\":\"[USERNAME]\"}]},\"depends_on\":[{\"node\":\"resources.database_instances.foo\",\"label\":\"${resources.database_instances.foo.id}\"}]}", + "resource_id": "/database-instances/test-db-instance", + "status": "OPERATION_STATUS_SUCCEEDED", + "sequence_id": "0" + } +} diff --git a/acceptance/bundle/resources/permissions/database_instances/current_can_manage/out.requests.destroy.direct.json b/acceptance/bundle/resources/permissions/database_instances/current_can_manage/out.requests.destroy.direct.json index e69de29bb2d..2dbd043b123 100644 --- a/acceptance/bundle/resources/permissions/database_instances/current_can_manage/out.requests.destroy.direct.json +++ b/acceptance/bundle/resources/permissions/database_instances/current_can_manage/out.requests.destroy.direct.json @@ -0,0 +1,12 @@ +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations/database_instances.foo.permissions", + "q": { + "update_mask": "state,error_message,resource_id,status" + }, + "body": { + "resource_id": "/database-instances/test-db-instance", + "status": "OPERATION_STATUS_SUCCEEDED", + "sequence_id": "0" + } +} diff --git a/acceptance/bundle/resources/permissions/jobs/current_can_manage/out.requests.deploy.direct.json b/acceptance/bundle/resources/permissions/jobs/current_can_manage/out.requests.deploy.direct.json index 5647c1a73b9..dd5b3baefd4 100644 --- a/acceptance/bundle/resources/permissions/jobs/current_can_manage/out.requests.deploy.direct.json +++ b/acceptance/bundle/resources/permissions/jobs/current_can_manage/out.requests.deploy.direct.json @@ -22,3 +22,16 @@ ] } } +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.foo.permissions", + "q": { + "update_mask": "state,error_message,resource_id,status" + }, + "body": { + "state": "{\"state\":{\"object_id\":\"/jobs/[NUMID]\",\"__embed__\":[{\"level\":\"CAN_VIEW\",\"user_name\":\"viewer@example.com\"},{\"level\":\"CAN_MANAGE\",\"group_name\":\"data-team\"},{\"level\":\"CAN_MANAGE\",\"service_principal_name\":\"[UUID]\"},{\"level\":\"IS_OWNER\",\"user_name\":\"[USERNAME]\"}]},\"depends_on\":[{\"node\":\"resources.jobs.foo\",\"label\":\"${resources.jobs.foo.id}\"}]}", + "resource_id": "/jobs/[NUMID]", + "status": "OPERATION_STATUS_SUCCEEDED", + "sequence_id": "0" + } +} diff --git a/acceptance/bundle/resources/permissions/jobs/current_can_manage/out.requests.destroy.direct.json b/acceptance/bundle/resources/permissions/jobs/current_can_manage/out.requests.destroy.direct.json index e69de29bb2d..af78adb96dc 100644 --- a/acceptance/bundle/resources/permissions/jobs/current_can_manage/out.requests.destroy.direct.json +++ b/acceptance/bundle/resources/permissions/jobs/current_can_manage/out.requests.destroy.direct.json @@ -0,0 +1,12 @@ +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations/jobs.foo.permissions", + "q": { + "update_mask": "state,error_message,resource_id,status" + }, + "body": { + "resource_id": "/jobs/[NUMID]", + "status": "OPERATION_STATUS_SUCCEEDED", + "sequence_id": "0" + } +} diff --git a/acceptance/bundle/resources/permissions/jobs/current_can_manage_run/out.destroy.requests.direct.json b/acceptance/bundle/resources/permissions/jobs/current_can_manage_run/out.destroy.requests.direct.json index 6564b6f8bde..2ed516784ba 100644 --- a/acceptance/bundle/resources/permissions/jobs/current_can_manage_run/out.destroy.requests.direct.json +++ b/acceptance/bundle/resources/permissions/jobs/current_can_manage_run/out.destroy.requests.direct.json @@ -1,3 +1,15 @@ +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations/jobs.interim_gold_layer_job.permissions", + "q": { + "update_mask": "state,error_message,resource_id,status" + }, + "body": { + "resource_id": "/jobs/[NUMID]", + "status": "OPERATION_STATUS_SUCCEEDED", + "sequence_id": "0" + } +} { "method": "POST", "path": "/api/2.2/jobs/delete", @@ -5,3 +17,15 @@ "job_id": [NUMID] } } +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations/jobs.interim_gold_layer_job", + "q": { + "update_mask": "state,error_message,resource_id,status" + }, + "body": { + "resource_id": "[NUMID]", + "status": "OPERATION_STATUS_SUCCEEDED", + "sequence_id": "0" + } +} diff --git a/acceptance/bundle/resources/permissions/jobs/current_is_owner/out.requests.deploy.direct.json b/acceptance/bundle/resources/permissions/jobs/current_is_owner/out.requests.deploy.direct.json index f7aa2dbaa0d..adaf900ff4e 100644 --- a/acceptance/bundle/resources/permissions/jobs/current_is_owner/out.requests.deploy.direct.json +++ b/acceptance/bundle/resources/permissions/jobs/current_is_owner/out.requests.deploy.direct.json @@ -10,3 +10,16 @@ ] } } +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.foo.permissions", + "q": { + "update_mask": "state,error_message,resource_id,status" + }, + "body": { + "state": "{\"state\":{\"object_id\":\"/jobs/[NUMID]\",\"__embed__\":[{\"level\":\"IS_OWNER\",\"user_name\":\"[USERNAME]\"}]},\"depends_on\":[{\"node\":\"resources.jobs.foo\",\"label\":\"${resources.jobs.foo.id}\"}]}", + "resource_id": "/jobs/[NUMID]", + "status": "OPERATION_STATUS_SUCCEEDED", + "sequence_id": "0" + } +} diff --git a/acceptance/bundle/resources/permissions/jobs/current_is_owner/out.requests.destroy.direct.json b/acceptance/bundle/resources/permissions/jobs/current_is_owner/out.requests.destroy.direct.json index e69de29bb2d..af78adb96dc 100644 --- a/acceptance/bundle/resources/permissions/jobs/current_is_owner/out.requests.destroy.direct.json +++ b/acceptance/bundle/resources/permissions/jobs/current_is_owner/out.requests.destroy.direct.json @@ -0,0 +1,12 @@ +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations/jobs.foo.permissions", + "q": { + "update_mask": "state,error_message,resource_id,status" + }, + "body": { + "resource_id": "/jobs/[NUMID]", + "status": "OPERATION_STATUS_SUCCEEDED", + "sequence_id": "0" + } +} diff --git a/acceptance/bundle/resources/permissions/jobs/other_can_manage/out.requests.deploy.direct.json b/acceptance/bundle/resources/permissions/jobs/other_can_manage/out.requests.deploy.direct.json index 5647c1a73b9..dd5b3baefd4 100644 --- a/acceptance/bundle/resources/permissions/jobs/other_can_manage/out.requests.deploy.direct.json +++ b/acceptance/bundle/resources/permissions/jobs/other_can_manage/out.requests.deploy.direct.json @@ -22,3 +22,16 @@ ] } } +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.foo.permissions", + "q": { + "update_mask": "state,error_message,resource_id,status" + }, + "body": { + "state": "{\"state\":{\"object_id\":\"/jobs/[NUMID]\",\"__embed__\":[{\"level\":\"CAN_VIEW\",\"user_name\":\"viewer@example.com\"},{\"level\":\"CAN_MANAGE\",\"group_name\":\"data-team\"},{\"level\":\"CAN_MANAGE\",\"service_principal_name\":\"[UUID]\"},{\"level\":\"IS_OWNER\",\"user_name\":\"[USERNAME]\"}]},\"depends_on\":[{\"node\":\"resources.jobs.foo\",\"label\":\"${resources.jobs.foo.id}\"}]}", + "resource_id": "/jobs/[NUMID]", + "status": "OPERATION_STATUS_SUCCEEDED", + "sequence_id": "0" + } +} diff --git a/acceptance/bundle/resources/permissions/jobs/other_can_manage/out.requests.destroy.direct.json b/acceptance/bundle/resources/permissions/jobs/other_can_manage/out.requests.destroy.direct.json index e69de29bb2d..af78adb96dc 100644 --- a/acceptance/bundle/resources/permissions/jobs/other_can_manage/out.requests.destroy.direct.json +++ b/acceptance/bundle/resources/permissions/jobs/other_can_manage/out.requests.destroy.direct.json @@ -0,0 +1,12 @@ +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations/jobs.foo.permissions", + "q": { + "update_mask": "state,error_message,resource_id,status" + }, + "body": { + "resource_id": "/jobs/[NUMID]", + "status": "OPERATION_STATUS_SUCCEEDED", + "sequence_id": "0" + } +} diff --git a/acceptance/bundle/resources/permissions/jobs/other_can_manage_run/out.destroy.requests.direct.json b/acceptance/bundle/resources/permissions/jobs/other_can_manage_run/out.destroy.requests.direct.json index 6564b6f8bde..58fdd00dd6f 100644 --- a/acceptance/bundle/resources/permissions/jobs/other_can_manage_run/out.destroy.requests.direct.json +++ b/acceptance/bundle/resources/permissions/jobs/other_can_manage_run/out.destroy.requests.direct.json @@ -5,3 +5,27 @@ "job_id": [NUMID] } } +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations/jobs.interim_gold_layer_job.permissions", + "q": { + "update_mask": "state,error_message,resource_id,status" + }, + "body": { + "resource_id": "/jobs/[NUMID]", + "status": "OPERATION_STATUS_SUCCEEDED", + "sequence_id": "0" + } +} +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations/jobs.interim_gold_layer_job", + "q": { + "update_mask": "state,error_message,resource_id,status" + }, + "body": { + "resource_id": "[NUMID]", + "status": "OPERATION_STATUS_SUCCEEDED", + "sequence_id": "0" + } +} diff --git a/acceptance/bundle/resources/permissions/jobs/other_is_owner/out.requests.deploy.direct.json b/acceptance/bundle/resources/permissions/jobs/other_is_owner/out.requests.deploy.direct.json index b1eb1519e87..f46c2203e72 100644 --- a/acceptance/bundle/resources/permissions/jobs/other_is_owner/out.requests.deploy.direct.json +++ b/acceptance/bundle/resources/permissions/jobs/other_is_owner/out.requests.deploy.direct.json @@ -14,3 +14,16 @@ ] } } +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.foo.permissions", + "q": { + "update_mask": "state,error_message,resource_id,status" + }, + "body": { + "state": "{\"state\":{\"object_id\":\"/jobs/[NUMID]\",\"__embed__\":[{\"level\":\"IS_OWNER\",\"user_name\":\"other_user@databricks.com\"},{\"level\":\"CAN_MANAGE\",\"user_name\":\"[USERNAME]\"}]},\"depends_on\":[{\"node\":\"resources.jobs.foo\",\"label\":\"${resources.jobs.foo.id}\"}]}", + "resource_id": "/jobs/[NUMID]", + "status": "OPERATION_STATUS_SUCCEEDED", + "sequence_id": "0" + } +} diff --git a/acceptance/bundle/resources/permissions/jobs/other_is_owner/out.requests.destroy.direct.json b/acceptance/bundle/resources/permissions/jobs/other_is_owner/out.requests.destroy.direct.json index e69de29bb2d..af78adb96dc 100644 --- a/acceptance/bundle/resources/permissions/jobs/other_is_owner/out.requests.destroy.direct.json +++ b/acceptance/bundle/resources/permissions/jobs/other_is_owner/out.requests.destroy.direct.json @@ -0,0 +1,12 @@ +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations/jobs.foo.permissions", + "q": { + "update_mask": "state,error_message,resource_id,status" + }, + "body": { + "resource_id": "/jobs/[NUMID]", + "status": "OPERATION_STATUS_SUCCEEDED", + "sequence_id": "0" + } +} diff --git a/acceptance/bundle/resources/permissions/jobs/viewers/out.requests.deploy.direct.json b/acceptance/bundle/resources/permissions/jobs/viewers/out.requests.deploy.direct.json index 45768bac0d5..aadff3ecc7d 100644 --- a/acceptance/bundle/resources/permissions/jobs/viewers/out.requests.deploy.direct.json +++ b/acceptance/bundle/resources/permissions/jobs/viewers/out.requests.deploy.direct.json @@ -22,3 +22,16 @@ ] } } +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.foo.permissions", + "q": { + "update_mask": "state,error_message,resource_id,status" + }, + "body": { + "state": "{\"state\":{\"object_id\":\"/jobs/[NUMID]\",\"__embed__\":[{\"level\":\"CAN_VIEW\",\"user_name\":\"viewer@example.com\"},{\"level\":\"CAN_VIEW\",\"group_name\":\"data-team\"},{\"level\":\"CAN_VIEW\",\"service_principal_name\":\"[UUID]\"},{\"level\":\"IS_OWNER\",\"user_name\":\"[USERNAME]\"}]},\"depends_on\":[{\"node\":\"resources.jobs.foo\",\"label\":\"${resources.jobs.foo.id}\"}]}", + "resource_id": "/jobs/[NUMID]", + "status": "OPERATION_STATUS_SUCCEEDED", + "sequence_id": "0" + } +} diff --git a/acceptance/bundle/resources/permissions/jobs/viewers/out.requests.destroy.direct.json b/acceptance/bundle/resources/permissions/jobs/viewers/out.requests.destroy.direct.json index e69de29bb2d..af78adb96dc 100644 --- a/acceptance/bundle/resources/permissions/jobs/viewers/out.requests.destroy.direct.json +++ b/acceptance/bundle/resources/permissions/jobs/viewers/out.requests.destroy.direct.json @@ -0,0 +1,12 @@ +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations/jobs.foo.permissions", + "q": { + "update_mask": "state,error_message,resource_id,status" + }, + "body": { + "resource_id": "/jobs/[NUMID]", + "status": "OPERATION_STATUS_SUCCEEDED", + "sequence_id": "0" + } +} diff --git a/acceptance/bundle/resources/permissions/output.txt b/acceptance/bundle/resources/permissions/output.txt index 85a16ad6a38..e5714e3f640 100644 --- a/acceptance/bundle/resources/permissions/output.txt +++ b/acceptance/bundle/resources/permissions/output.txt @@ -17,25 +17,54 @@ DIFF apps/current_can_manage/out.requests.destroy.direct.json + "path": "/api/2.0/permissions/apps/foo" + } +] -MATCH apps/other_can_manage/out.requests.deploy.direct.json +DIFF apps/other_can_manage/out.requests.deploy.direct.json +--- apps/other_can_manage/out.requests.deploy.direct.json ++++ apps/other_can_manage/out.requests.deploy.terraform.json +@@ -22,18 +22,5 @@ + }, + "method": "PUT", + "path": "/api/2.0/permissions/apps/foo" +- }, +- { +- "body": { +- "resource_id": "/apps/foo", +- "sequence_id": "0", +- "state": "{/"state/":{/"object_id/":/"/apps/foo/",/"__embed__/":[{/"level/":/"CAN_USE/",/"user_name/":/"viewer@example.com/"},{/"level/":/"CAN_MANAGE/",/"group_name/":/"data-team/"},{/"level/":/"CAN_MANAGE/",/"service_principal_name/":/"[UUID]/"},{/"level/":/"CAN_MANAGE/",/"user_name/":/"[USERNAME]/"}]},/"depends_on/":[{/"node/":/"resources.apps.foo/",/"label/":/"${resources.apps.foo.id}/"}]}", +- "status": "OPERATION_STATUS_SUCCEEDED" +- }, +- "method": "PATCH", +- "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/apps.foo.permissions", +- "q": { +- "update_mask": "state,error_message,resource_id,status" +- } + } + ] DIFF apps/other_can_manage/out.requests.destroy.direct.json --- apps/other_can_manage/out.requests.destroy.direct.json +++ apps/other_can_manage/out.requests.destroy.terraform.json -@@ -1 +1,14 @@ --[]+[ -+ { -+ "body": { +@@ -1,14 +1,14 @@ + [ + { + "body": { +- "resource_id": "/apps/foo", +- "sequence_id": "0", +- "status": "OPERATION_STATUS_SUCCEEDED" + "access_control_list": [ + { + "permission_level": "CAN_MANAGE", + "user_name": "[USERNAME]" + } + ] -+ }, + }, +- "method": "PATCH", +- "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations/apps.foo.permissions", +- "q": { +- "update_mask": "state,error_message,resource_id,status" +- } + "method": "PUT", + "path": "/api/2.0/permissions/apps/foo" -+ } -+] + } + ] MATCH clusters/current_can_manage/out.requests.deploy.direct.json DIFF clusters/current_can_manage/out.requests.destroy.direct.json --- clusters/current_can_manage/out.requests.destroy.direct.json @@ -55,7 +84,48 @@ DIFF clusters/current_can_manage/out.requests.destroy.direct.json + "path": "/api/2.0/permissions/clusters/[UUID]" + } +] -MATCH clusters/target/out.requests.direct.json +DIFF clusters/target/out.requests.direct.json +--- clusters/target/out.requests.direct.json ++++ clusters/target/out.requests.terraform.json +@@ -12,19 +12,6 @@ + }, + "method": "POST", + "path": "/api/2.1/clusters/create" +- }, +- { +- "body": { +- "resource_id": "[UUID]", +- "sequence_id": "0", +- "state": "{/"state/":{/"autotermination_minutes/":60,/"cluster_name/":/"[dev [USERNAME]] test-cluster/",/"custom_tags/":{/"dev/":/"[USERNAME]/"},/"node_type_id/":/"[NODE_TYPE_ID]/",/"num_workers/":1,/"spark_version/":/"15.4.x-scala2.12/"}}", +- "status": "OPERATION_STATUS_SUCCEEDED" +- }, +- "method": "PATCH", +- "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/clusters.cluster1", +- "q": { +- "update_mask": "state,error_message,resource_id,status" +- } + }, + { + "body": { +@@ -49,18 +36,5 @@ + }, + "method": "PUT", + "path": "/api/2.0/permissions/clusters/[UUID]" +- }, +- { +- "body": { +- "resource_id": "/clusters/[UUID]", +- "sequence_id": "0", +- "state": "{/"state/":{/"object_id/":/"/clusters/[UUID]/",/"__embed__/":[{/"level/":/"CAN_ATTACH_TO/",/"service_principal_name/":/"[UUID]/"},{/"level/":/"CAN_RESTART/",/"service_principal_name/":/"[UUID]/"},{/"level/":/"CAN_MANAGE/",/"service_principal_name/":/"[UUID]/"},{/"level/":/"CAN_MANAGE/",/"user_name/":/"[USERNAME]/"}]},/"depends_on/":[{/"node/":/"resources.clusters.cluster1/",/"label/":/"${resources.clusters.cluster1.id}/"}]}", +- "status": "OPERATION_STATUS_SUCCEEDED" +- }, +- "method": "PATCH", +- "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/clusters.cluster1.permissions", +- "q": { +- "update_mask": "state,error_message,resource_id,status" +- } + } + ] MATCH dashboards/create/out.requests.deploy.direct.json DIFF dashboards/create/out.requests.destroy.direct.json --- dashboards/create/out.requests.destroy.direct.json @@ -68,25 +138,54 @@ DIFF dashboards/create/out.requests.destroy.direct.json + "path": "/api/2.0/permissions/dashboards/[FOO_ID]" + } +] -MATCH database_instances/current_can_manage/out.requests.deploy.direct.json +DIFF database_instances/current_can_manage/out.requests.deploy.direct.json +--- database_instances/current_can_manage/out.requests.deploy.direct.json ++++ database_instances/current_can_manage/out.requests.deploy.terraform.json +@@ -22,18 +22,5 @@ + }, + "method": "PUT", + "path": "/api/2.0/permissions/database-instances/test-db-instance" +- }, +- { +- "body": { +- "resource_id": "/database-instances/test-db-instance", +- "sequence_id": "0", +- "state": "{/"state/":{/"object_id/":/"/database-instances/test-db-instance/",/"__embed__/":[{/"level/":/"CAN_USE/",/"user_name/":/"viewer@example.com/"},{/"level/":/"CAN_MANAGE/",/"group_name/":/"data-team/"},{/"level/":/"CAN_MANAGE/",/"service_principal_name/":/"[UUID]/"},{/"level/":/"CAN_MANAGE/",/"user_name/":/"[USERNAME]/"}]},/"depends_on/":[{/"node/":/"resources.database_instances.foo/",/"label/":/"${resources.database_instances.foo.id}/"}]}", +- "status": "OPERATION_STATUS_SUCCEEDED" +- }, +- "method": "PATCH", +- "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/database_instances.foo.permissions", +- "q": { +- "update_mask": "state,error_message,resource_id,status" +- } + } + ] DIFF database_instances/current_can_manage/out.requests.destroy.direct.json --- database_instances/current_can_manage/out.requests.destroy.direct.json +++ database_instances/current_can_manage/out.requests.destroy.terraform.json -@@ -1 +1,14 @@ --[]+[ -+ { -+ "body": { +@@ -1,14 +1,14 @@ + [ + { + "body": { +- "resource_id": "/database-instances/test-db-instance", +- "sequence_id": "0", +- "status": "OPERATION_STATUS_SUCCEEDED" + "access_control_list": [ + { + "permission_level": "CAN_MANAGE", + "user_name": "[USERNAME]" + } + ] -+ }, + }, +- "method": "PATCH", +- "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations/database_instances.foo.permissions", +- "q": { +- "update_mask": "state,error_message,resource_id,status" +- } + "method": "PUT", + "path": "/api/2.0/permissions/database-instances/test-db-instance" -+ } -+] + } + ] MATCH experiments/current_can_manage/out.requests.deploy.direct.json DIFF experiments/current_can_manage/out.requests.destroy.direct.json --- experiments/current_can_manage/out.requests.destroy.direct.json @@ -101,64 +200,147 @@ DIFF experiments/current_can_manage/out.requests.destroy.direct.json +] DIRECT_ONLY genie_spaces/current_can_manage/out.requests.deploy.direct.json DIRECT_ONLY genie_spaces/current_can_manage/out.requests.destroy.direct.json -MATCH jobs/current_can_manage/out.requests.deploy.direct.json +DIFF jobs/current_can_manage/out.requests.deploy.direct.json +--- jobs/current_can_manage/out.requests.deploy.direct.json ++++ jobs/current_can_manage/out.requests.deploy.terraform.json +@@ -22,18 +22,5 @@ + }, + "method": "PUT", + "path": "/api/2.0/permissions/jobs/[NUMID]" +- }, +- { +- "body": { +- "resource_id": "/jobs/[NUMID]", +- "sequence_id": "0", +- "state": "{/"state/":{/"object_id/":/"/jobs/[NUMID]/",/"__embed__/":[{/"level/":/"CAN_VIEW/",/"user_name/":/"viewer@example.com/"},{/"level/":/"CAN_MANAGE/",/"group_name/":/"data-team/"},{/"level/":/"CAN_MANAGE/",/"service_principal_name/":/"[UUID]/"},{/"level/":/"IS_OWNER/",/"user_name/":/"[USERNAME]/"}]},/"depends_on/":[{/"node/":/"resources.jobs.foo/",/"label/":/"${resources.jobs.foo.id}/"}]}", +- "status": "OPERATION_STATUS_SUCCEEDED" +- }, +- "method": "PATCH", +- "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.foo.permissions", +- "q": { +- "update_mask": "state,error_message,resource_id,status" +- } + } + ] DIFF jobs/current_can_manage/out.requests.destroy.direct.json --- jobs/current_can_manage/out.requests.destroy.direct.json +++ jobs/current_can_manage/out.requests.destroy.terraform.json -@@ -1 +1,14 @@ --[]+[ -+ { -+ "body": { +@@ -1,14 +1,14 @@ + [ + { + "body": { +- "resource_id": "/jobs/[NUMID]", +- "sequence_id": "0", +- "status": "OPERATION_STATUS_SUCCEEDED" + "access_control_list": [ + { + "permission_level": "IS_OWNER", + "user_name": "[USERNAME]" + } + ] -+ }, + }, +- "method": "PATCH", +- "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations/jobs.foo.permissions", +- "q": { +- "update_mask": "state,error_message,resource_id,status" +- } + "method": "PUT", + "path": "/api/2.0/permissions/jobs/[NUMID]" -+ } -+] + } + ] DIFF jobs/current_can_manage_run/out.destroy.requests.direct.json --- jobs/current_can_manage_run/out.destroy.requests.direct.json +++ jobs/current_can_manage_run/out.destroy.requests.terraform.json -@@ -1,4 +1,16 @@ +@@ -1,15 +1,15 @@ [ -+ { -+ "body": { + { + "body": { +- "resource_id": "/jobs/[NUMID]", +- "sequence_id": "0", +- "status": "OPERATION_STATUS_SUCCEEDED" + "access_control_list": [ + { + "permission_level": "IS_OWNER", + "user_name": "[USERNAME]" + } + ] -+ }, + }, +- "method": "PATCH", +- "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations/jobs.interim_gold_layer_job.permissions", +- "q": { +- "update_mask": "state,error_message,resource_id,status" +- } + "method": "PUT", + "path": "/api/2.0/permissions/jobs/[NUMID]" -+ }, + }, { "body": { - "job_id": "[NUMID]" -EXACT jobs/current_is_owner/out.requests.deploy.direct.json +@@ -17,17 +17,5 @@ + }, + "method": "POST", + "path": "/api/2.2/jobs/delete" +- }, +- { +- "body": { +- "resource_id": "[NUMID]", +- "sequence_id": "0", +- "status": "OPERATION_STATUS_SUCCEEDED" +- }, +- "method": "PATCH", +- "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations/jobs.interim_gold_layer_job", +- "q": { +- "update_mask": "state,error_message,resource_id,status" +- } + } + ] +DIFF jobs/current_is_owner/out.requests.deploy.direct.json +--- jobs/current_is_owner/out.requests.deploy.direct.json ++++ jobs/current_is_owner/out.requests.deploy.terraform.json +@@ -10,18 +10,5 @@ + }, + "method": "PUT", + "path": "/api/2.0/permissions/jobs/[NUMID]" +- }, +- { +- "body": { +- "resource_id": "/jobs/[NUMID]", +- "sequence_id": "0", +- "state": "{/"state/":{/"object_id/":/"/jobs/[NUMID]/",/"__embed__/":[{/"level/":/"IS_OWNER/",/"user_name/":/"[USERNAME]/"}]},/"depends_on/":[{/"node/":/"resources.jobs.foo/",/"label/":/"${resources.jobs.foo.id}/"}]}", +- "status": "OPERATION_STATUS_SUCCEEDED" +- }, +- "method": "PATCH", +- "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.foo.permissions", +- "q": { +- "update_mask": "state,error_message,resource_id,status" +- } + } + ] DIFF jobs/current_is_owner/out.requests.destroy.direct.json --- jobs/current_is_owner/out.requests.destroy.direct.json +++ jobs/current_is_owner/out.requests.destroy.terraform.json -@@ -1 +1,14 @@ --[]+[ -+ { -+ "body": { +@@ -1,14 +1,14 @@ + [ + { + "body": { +- "resource_id": "/jobs/[NUMID]", +- "sequence_id": "0", +- "status": "OPERATION_STATUS_SUCCEEDED" + "access_control_list": [ + { + "permission_level": "IS_OWNER", + "user_name": "[USERNAME]" + } + ] -+ }, + }, +- "method": "PATCH", +- "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations/jobs.foo.permissions", +- "q": { +- "update_mask": "state,error_message,resource_id,status" +- } + "method": "PUT", + "path": "/api/2.0/permissions/jobs/[NUMID]" -+ } -+] + } + ] DIFF jobs/delete_one/out.requests_destroy.direct.json --- jobs/delete_one/out.requests_destroy.direct.json +++ jobs/delete_one/out.requests_destroy.terraform.json @@ -181,29 +363,58 @@ DIFF jobs/delete_one/out.requests_destroy.direct.json "job_id": "[JOB_WITH_PERMISSIONS_ID]" EXACT jobs/empty_list/out.requests.deploy.direct.json EXACT jobs/empty_list/out.requests.destroy.direct.json -MATCH jobs/other_can_manage/out.requests.deploy.direct.json +DIFF jobs/other_can_manage/out.requests.deploy.direct.json +--- jobs/other_can_manage/out.requests.deploy.direct.json ++++ jobs/other_can_manage/out.requests.deploy.terraform.json +@@ -22,18 +22,5 @@ + }, + "method": "PUT", + "path": "/api/2.0/permissions/jobs/[NUMID]" +- }, +- { +- "body": { +- "resource_id": "/jobs/[NUMID]", +- "sequence_id": "0", +- "state": "{/"state/":{/"object_id/":/"/jobs/[NUMID]/",/"__embed__/":[{/"level/":/"CAN_VIEW/",/"user_name/":/"viewer@example.com/"},{/"level/":/"CAN_MANAGE/",/"group_name/":/"data-team/"},{/"level/":/"CAN_MANAGE/",/"service_principal_name/":/"[UUID]/"},{/"level/":/"IS_OWNER/",/"user_name/":/"[USERNAME]/"}]},/"depends_on/":[{/"node/":/"resources.jobs.foo/",/"label/":/"${resources.jobs.foo.id}/"}]}", +- "status": "OPERATION_STATUS_SUCCEEDED" +- }, +- "method": "PATCH", +- "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.foo.permissions", +- "q": { +- "update_mask": "state,error_message,resource_id,status" +- } + } + ] DIFF jobs/other_can_manage/out.requests.destroy.direct.json --- jobs/other_can_manage/out.requests.destroy.direct.json +++ jobs/other_can_manage/out.requests.destroy.terraform.json -@@ -1 +1,14 @@ --[]+[ -+ { -+ "body": { +@@ -1,14 +1,14 @@ + [ + { + "body": { +- "resource_id": "/jobs/[NUMID]", +- "sequence_id": "0", +- "status": "OPERATION_STATUS_SUCCEEDED" + "access_control_list": [ + { + "permission_level": "IS_OWNER", + "user_name": "[USERNAME]" + } + ] -+ }, + }, +- "method": "PATCH", +- "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations/jobs.foo.permissions", +- "q": { +- "update_mask": "state,error_message,resource_id,status" +- } + "method": "PUT", + "path": "/api/2.0/permissions/jobs/[NUMID]" -+ } -+] + } + ] DIFF jobs/other_can_manage_run/out.destroy.requests.direct.json --- jobs/other_can_manage_run/out.destroy.requests.direct.json +++ jobs/other_can_manage_run/out.destroy.requests.terraform.json -@@ -1,4 +1,16 @@ +@@ -1,33 +1,21 @@ [ + { + "body": { @@ -220,25 +431,83 @@ DIFF jobs/other_can_manage_run/out.destroy.requests.direct.json { "body": { "job_id": "[NUMID]" -EXACT jobs/other_is_owner/out.requests.deploy.direct.json + }, + "method": "POST", + "path": "/api/2.2/jobs/delete" +- }, +- { +- "body": { +- "resource_id": "/jobs/[NUMID]", +- "sequence_id": "0", +- "status": "OPERATION_STATUS_SUCCEEDED" +- }, +- "method": "PATCH", +- "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations/jobs.interim_gold_layer_job.permissions", +- "q": { +- "update_mask": "state,error_message,resource_id,status" +- } +- }, +- { +- "body": { +- "resource_id": "[NUMID]", +- "sequence_id": "0", +- "status": "OPERATION_STATUS_SUCCEEDED" +- }, +- "method": "PATCH", +- "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations/jobs.interim_gold_layer_job", +- "q": { +- "update_mask": "state,error_message,resource_id,status" +- } + } + ] +DIFF jobs/other_is_owner/out.requests.deploy.direct.json +--- jobs/other_is_owner/out.requests.deploy.direct.json ++++ jobs/other_is_owner/out.requests.deploy.terraform.json +@@ -14,18 +14,5 @@ + }, + "method": "PUT", + "path": "/api/2.0/permissions/jobs/[NUMID]" +- }, +- { +- "body": { +- "resource_id": "/jobs/[NUMID]", +- "sequence_id": "0", +- "state": "{/"state/":{/"object_id/":/"/jobs/[NUMID]/",/"__embed__/":[{/"level/":/"IS_OWNER/",/"user_name/":/"other_user@databricks.com/"},{/"level/":/"CAN_MANAGE/",/"user_name/":/"[USERNAME]/"}]},/"depends_on/":[{/"node/":/"resources.jobs.foo/",/"label/":/"${resources.jobs.foo.id}/"}]}", +- "status": "OPERATION_STATUS_SUCCEEDED" +- }, +- "method": "PATCH", +- "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.foo.permissions", +- "q": { +- "update_mask": "state,error_message,resource_id,status" +- } + } + ] DIFF jobs/other_is_owner/out.requests.destroy.direct.json --- jobs/other_is_owner/out.requests.destroy.direct.json +++ jobs/other_is_owner/out.requests.destroy.terraform.json -@@ -1 +1,14 @@ --[]+[ -+ { -+ "body": { +@@ -1,14 +1,14 @@ + [ + { + "body": { +- "resource_id": "/jobs/[NUMID]", +- "sequence_id": "0", +- "status": "OPERATION_STATUS_SUCCEEDED" + "access_control_list": [ + { + "permission_level": "IS_OWNER", + "user_name": "[USERNAME]" + } + ] -+ }, + }, +- "method": "PATCH", +- "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations/jobs.foo.permissions", +- "q": { +- "update_mask": "state,error_message,resource_id,status" +- } + "method": "PUT", + "path": "/api/2.0/permissions/jobs/[NUMID]" -+ } -+] + } + ] DIFF jobs/update/out.requests_delete_all.direct.json --- jobs/update/out.requests_delete_all.direct.json +++ jobs/update/out.requests_delete_all.terraform.json @@ -276,25 +545,54 @@ DIFF jobs/update/out.requests_set_empty.direct.json + "path": "/api/2.0/permissions/jobs/[JOB_WITH_PERMISSIONS_ID]" + } +] -MATCH jobs/viewers/out.requests.deploy.direct.json +DIFF jobs/viewers/out.requests.deploy.direct.json +--- jobs/viewers/out.requests.deploy.direct.json ++++ jobs/viewers/out.requests.deploy.terraform.json +@@ -22,18 +22,5 @@ + }, + "method": "PUT", + "path": "/api/2.0/permissions/jobs/[NUMID]" +- }, +- { +- "body": { +- "resource_id": "/jobs/[NUMID]", +- "sequence_id": "0", +- "state": "{/"state/":{/"object_id/":/"/jobs/[NUMID]/",/"__embed__/":[{/"level/":/"CAN_VIEW/",/"user_name/":/"viewer@example.com/"},{/"level/":/"CAN_VIEW/",/"group_name/":/"data-team/"},{/"level/":/"CAN_VIEW/",/"service_principal_name/":/"[UUID]/"},{/"level/":/"IS_OWNER/",/"user_name/":/"[USERNAME]/"}]},/"depends_on/":[{/"node/":/"resources.jobs.foo/",/"label/":/"${resources.jobs.foo.id}/"}]}", +- "status": "OPERATION_STATUS_SUCCEEDED" +- }, +- "method": "PATCH", +- "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.foo.permissions", +- "q": { +- "update_mask": "state,error_message,resource_id,status" +- } + } + ] DIFF jobs/viewers/out.requests.destroy.direct.json --- jobs/viewers/out.requests.destroy.direct.json +++ jobs/viewers/out.requests.destroy.terraform.json -@@ -1 +1,14 @@ --[]+[ -+ { -+ "body": { +@@ -1,14 +1,14 @@ + [ + { + "body": { +- "resource_id": "/jobs/[NUMID]", +- "sequence_id": "0", +- "status": "OPERATION_STATUS_SUCCEEDED" + "access_control_list": [ + { + "permission_level": "IS_OWNER", + "user_name": "[USERNAME]" + } + ] -+ }, + }, +- "method": "PATCH", +- "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations/jobs.foo.permissions", +- "q": { +- "update_mask": "state,error_message,resource_id,status" +- } + "method": "PUT", + "path": "/api/2.0/permissions/jobs/[NUMID]" -+ } -+] + } + ] MATCH models/current_can_manage/out.requests.deploy.direct.json DIFF models/current_can_manage/out.requests.destroy.direct.json --- models/current_can_manage/out.requests.destroy.direct.json @@ -314,43 +612,210 @@ DIFF models/current_can_manage/out.requests.destroy.direct.json + "path": "/api/2.0/permissions/registered-models/[FOO_MODEL_ID]" + } +] -MATCH pipelines/current_can_manage/out.requests.deploy.direct.json -EXACT pipelines/current_can_manage/out.requests.destroy.direct.json +DIFF pipelines/current_can_manage/out.requests.deploy.direct.json +--- pipelines/current_can_manage/out.requests.deploy.direct.json ++++ pipelines/current_can_manage/out.requests.deploy.terraform.json +@@ -22,18 +22,5 @@ + }, + "method": "PUT", + "path": "/api/2.0/permissions/pipelines/[UUID]" +- }, +- { +- "body": { +- "resource_id": "/pipelines/[UUID]", +- "sequence_id": "0", +- "state": "{/"state/":{/"object_id/":/"/pipelines/[UUID]/",/"__embed__/":[{/"level/":/"CAN_VIEW/",/"user_name/":/"viewer@example.com/"},{/"level/":/"CAN_MANAGE/",/"group_name/":/"data-team/"},{/"level/":/"CAN_MANAGE/",/"service_principal_name/":/"[UUID]/"},{/"level/":/"IS_OWNER/",/"user_name/":/"[USERNAME]/"}]},/"depends_on/":[{/"node/":/"resources.pipelines.foo/",/"label/":/"${resources.pipelines.foo.id}/"}]}", +- "status": "OPERATION_STATUS_SUCCEEDED" +- }, +- "method": "PATCH", +- "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/pipelines.foo.permissions", +- "q": { +- "update_mask": "state,error_message,resource_id,status" +- } + } + ] +DIFF pipelines/current_can_manage/out.requests.destroy.direct.json +--- pipelines/current_can_manage/out.requests.destroy.direct.json ++++ pipelines/current_can_manage/out.requests.destroy.terraform.json +@@ -1,14 +1 @@ +-[ +- { +- "body": { +- "resource_id": "/pipelines/[UUID]", +- "sequence_id": "0", +- "status": "OPERATION_STATUS_SUCCEEDED" +- }, +- "method": "PATCH", +- "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations/pipelines.foo.permissions", +- "q": { +- "update_mask": "state,error_message,resource_id,status" +- } +- } +-]+[] EXACT pipelines/current_is_owner/out.requests.deploy.direct.json EXACT pipelines/current_is_owner/out.requests.destroy.direct.json EXACT pipelines/empty_list/out.requests.deploy.direct.json EXACT pipelines/empty_list/out.requests.destroy.direct.json -MATCH pipelines/other_can_manage/out.requests.deploy.direct.json -EXACT pipelines/other_can_manage/out.requests.destroy.direct.json -EXACT pipelines/other_is_owner/out.requests.deploy.direct.json -EXACT pipelines/other_is_owner/out.requests.destroy.direct.json -MATCH postgres_projects/current_can_manage/out.requests.deploy.direct.json +DIFF pipelines/other_can_manage/out.requests.deploy.direct.json +--- pipelines/other_can_manage/out.requests.deploy.direct.json ++++ pipelines/other_can_manage/out.requests.deploy.terraform.json +@@ -22,18 +22,5 @@ + }, + "method": "PUT", + "path": "/api/2.0/permissions/pipelines/[UUID]" +- }, +- { +- "body": { +- "resource_id": "/pipelines/[UUID]", +- "sequence_id": "0", +- "state": "{/"state/":{/"object_id/":/"/pipelines/[UUID]/",/"__embed__/":[{/"level/":/"CAN_VIEW/",/"user_name/":/"viewer@example.com/"},{/"level/":/"CAN_MANAGE/",/"group_name/":/"data-team/"},{/"level/":/"CAN_MANAGE/",/"service_principal_name/":/"[UUID]/"},{/"level/":/"IS_OWNER/",/"user_name/":/"[USERNAME]/"}]},/"depends_on/":[{/"node/":/"resources.pipelines.foo/",/"label/":/"${resources.pipelines.foo.id}/"}]}", +- "status": "OPERATION_STATUS_SUCCEEDED" +- }, +- "method": "PATCH", +- "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/pipelines.foo.permissions", +- "q": { +- "update_mask": "state,error_message,resource_id,status" +- } + } + ] +DIFF pipelines/other_can_manage/out.requests.destroy.direct.json +--- pipelines/other_can_manage/out.requests.destroy.direct.json ++++ pipelines/other_can_manage/out.requests.destroy.terraform.json +@@ -1,14 +1 @@ +-[ +- { +- "body": { +- "resource_id": "/pipelines/[UUID]", +- "sequence_id": "0", +- "status": "OPERATION_STATUS_SUCCEEDED" +- }, +- "method": "PATCH", +- "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations/pipelines.foo.permissions", +- "q": { +- "update_mask": "state,error_message,resource_id,status" +- } +- } +-]+[] +DIFF pipelines/other_is_owner/out.requests.deploy.direct.json +--- pipelines/other_is_owner/out.requests.deploy.direct.json ++++ pipelines/other_is_owner/out.requests.deploy.terraform.json +@@ -14,18 +14,5 @@ + }, + "method": "PUT", + "path": "/api/2.0/permissions/pipelines/[UUID]" +- }, +- { +- "body": { +- "resource_id": "/pipelines/[UUID]", +- "sequence_id": "0", +- "state": "{/"state/":{/"object_id/":/"/pipelines/[UUID]/",/"__embed__/":[{/"level/":/"IS_OWNER/",/"user_name/":/"other_user@databricks.com/"},{/"level/":/"CAN_MANAGE/",/"user_name/":/"[USERNAME]/"}]},/"depends_on/":[{/"node/":/"resources.pipelines.foo/",/"label/":/"${resources.pipelines.foo.id}/"}]}", +- "status": "OPERATION_STATUS_SUCCEEDED" +- }, +- "method": "PATCH", +- "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/pipelines.foo.permissions", +- "q": { +- "update_mask": "state,error_message,resource_id,status" +- } + } + ] +DIFF pipelines/other_is_owner/out.requests.destroy.direct.json +--- pipelines/other_is_owner/out.requests.destroy.direct.json ++++ pipelines/other_is_owner/out.requests.destroy.terraform.json +@@ -1,14 +1 @@ +-[ +- { +- "body": { +- "resource_id": "/pipelines/[UUID]", +- "sequence_id": "0", +- "status": "OPERATION_STATUS_SUCCEEDED" +- }, +- "method": "PATCH", +- "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations/pipelines.foo.permissions", +- "q": { +- "update_mask": "state,error_message,resource_id,status" +- } +- } +-]+[] +DIFF postgres_projects/current_can_manage/out.requests.deploy.direct.json +--- postgres_projects/current_can_manage/out.requests.deploy.direct.json ++++ postgres_projects/current_can_manage/out.requests.deploy.terraform.json +@@ -22,18 +22,5 @@ + }, + "method": "PUT", + "path": "/api/2.0/permissions/database-projects/test-project" +- }, +- { +- "body": { +- "resource_id": "/database-projects/test-project", +- "sequence_id": "0", +- "state": "{/"state/":{/"object_id/":/"/database-projects/test-project/",/"__embed__/":[{/"level/":/"CAN_USE/",/"user_name/":/"viewer@example.com/"},{/"level/":/"CAN_MANAGE/",/"group_name/":/"data-team/"},{/"level/":/"CAN_MANAGE/",/"service_principal_name/":/"[UUID]/"},{/"level/":/"CAN_MANAGE/",/"user_name/":/"[USERNAME]/"}]},/"depends_on/":[{/"node/":/"resources.postgres_projects.foo/",/"label/":/"${resources.postgres_projects.foo.project_id}/"}]}", +- "status": "OPERATION_STATUS_SUCCEEDED" +- }, +- "method": "PATCH", +- "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/postgres_projects.foo.permissions", +- "q": { +- "update_mask": "state,error_message,resource_id,status" +- } + } + ] DIFF postgres_projects/current_can_manage/out.requests.destroy.direct.json --- postgres_projects/current_can_manage/out.requests.destroy.direct.json +++ postgres_projects/current_can_manage/out.requests.destroy.terraform.json -@@ -1 +1,14 @@ --[]+[ -+ { -+ "body": { +@@ -1,14 +1,14 @@ + [ + { + "body": { +- "resource_id": "/database-projects/test-project", +- "sequence_id": "0", +- "status": "OPERATION_STATUS_SUCCEEDED" + "access_control_list": [ + { + "permission_level": "CAN_MANAGE", + "user_name": "[USERNAME]" + } + ] -+ }, + }, +- "method": "PATCH", +- "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations/postgres_projects.foo.permissions", +- "q": { +- "update_mask": "state,error_message,resource_id,status" +- } + "method": "PUT", + "path": "/api/2.0/permissions/database-projects/test-project" -+ } -+] -MATCH sql_warehouses/current_can_manage/out.requests.deploy.direct.json + } + ] +DIFF sql_warehouses/current_can_manage/out.requests.deploy.direct.json +--- sql_warehouses/current_can_manage/out.requests.deploy.direct.json ++++ sql_warehouses/current_can_manage/out.requests.deploy.terraform.json +@@ -22,18 +22,5 @@ + }, + "method": "PUT", + "path": "/api/2.0/permissions/sql/warehouses/[UUID]" +- }, +- { +- "body": { +- "resource_id": "/sql/warehouses/[UUID]", +- "sequence_id": "0", +- "state": "{/"state/":{/"object_id/":/"/sql/warehouses/[UUID]/",/"__embed__/":[{/"level/":/"CAN_VIEW/",/"user_name/":/"viewer@example.com/"},{/"level/":/"CAN_MANAGE/",/"group_name/":/"data-team/"},{/"level/":/"CAN_MANAGE/",/"service_principal_name/":/"[UUID]/"},{/"level/":/"IS_OWNER/",/"user_name/":/"[USERNAME]/"}]},/"depends_on/":[{/"node/":/"resources.sql_warehouses.foo/",/"label/":/"${resources.sql_warehouses.foo.id}/"}]}", +- "status": "OPERATION_STATUS_SUCCEEDED" +- }, +- "method": "PATCH", +- "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/sql_warehouses.foo.permissions", +- "q": { +- "update_mask": "state,error_message,resource_id,status" +- } + } + ] DIFF sql_warehouses/current_can_manage/out.requests.destroy.direct.json --- sql_warehouses/current_can_manage/out.requests.destroy.direct.json +++ sql_warehouses/current_can_manage/out.requests.destroy.terraform.json -@@ -1 +1,18 @@ --[]+[ -+ { -+ "body": { +@@ -1,14 +1,18 @@ + [ + { + "body": { +- "resource_id": "/sql/warehouses/[UUID]", +- "sequence_id": "0", +- "status": "OPERATION_STATUS_SUCCEEDED" + "access_control_list": [ + { + "permission_level": "CAN_MANAGE", @@ -361,11 +826,16 @@ DIFF sql_warehouses/current_can_manage/out.requests.destroy.direct.json + "user_name": "[USERNAME]" + } + ] -+ }, + }, +- "method": "PATCH", +- "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations/sql_warehouses.foo.permissions", +- "q": { +- "update_mask": "state,error_message,resource_id,status" +- } + "method": "PUT", + "path": "/api/2.0/permissions/sql/warehouses/[UUID]" -+ } -+] + } + ] EXACT target_permissions/out.requests_create.direct.json DIFF target_permissions/out.requests_delete.direct.json --- target_permissions/out.requests_delete.direct.json diff --git a/acceptance/bundle/resources/permissions/pipelines/current_can_manage/out.requests.deploy.direct.json b/acceptance/bundle/resources/permissions/pipelines/current_can_manage/out.requests.deploy.direct.json index 42daa83bd86..208fe9a91d4 100644 --- a/acceptance/bundle/resources/permissions/pipelines/current_can_manage/out.requests.deploy.direct.json +++ b/acceptance/bundle/resources/permissions/pipelines/current_can_manage/out.requests.deploy.direct.json @@ -22,3 +22,16 @@ ] } } +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/pipelines.foo.permissions", + "q": { + "update_mask": "state,error_message,resource_id,status" + }, + "body": { + "state": "{\"state\":{\"object_id\":\"/pipelines/[UUID]\",\"__embed__\":[{\"level\":\"CAN_VIEW\",\"user_name\":\"viewer@example.com\"},{\"level\":\"CAN_MANAGE\",\"group_name\":\"data-team\"},{\"level\":\"CAN_MANAGE\",\"service_principal_name\":\"[UUID]\"},{\"level\":\"IS_OWNER\",\"user_name\":\"[USERNAME]\"}]},\"depends_on\":[{\"node\":\"resources.pipelines.foo\",\"label\":\"${resources.pipelines.foo.id}\"}]}", + "resource_id": "/pipelines/[UUID]", + "status": "OPERATION_STATUS_SUCCEEDED", + "sequence_id": "0" + } +} diff --git a/acceptance/bundle/resources/permissions/pipelines/current_can_manage/out.requests.destroy.direct.json b/acceptance/bundle/resources/permissions/pipelines/current_can_manage/out.requests.destroy.direct.json index e69de29bb2d..beda36a358c 100644 --- a/acceptance/bundle/resources/permissions/pipelines/current_can_manage/out.requests.destroy.direct.json +++ b/acceptance/bundle/resources/permissions/pipelines/current_can_manage/out.requests.destroy.direct.json @@ -0,0 +1,12 @@ +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations/pipelines.foo.permissions", + "q": { + "update_mask": "state,error_message,resource_id,status" + }, + "body": { + "resource_id": "/pipelines/[UUID]", + "status": "OPERATION_STATUS_SUCCEEDED", + "sequence_id": "0" + } +} diff --git a/acceptance/bundle/resources/permissions/pipelines/other_can_manage/out.requests.deploy.direct.json b/acceptance/bundle/resources/permissions/pipelines/other_can_manage/out.requests.deploy.direct.json index 42daa83bd86..208fe9a91d4 100644 --- a/acceptance/bundle/resources/permissions/pipelines/other_can_manage/out.requests.deploy.direct.json +++ b/acceptance/bundle/resources/permissions/pipelines/other_can_manage/out.requests.deploy.direct.json @@ -22,3 +22,16 @@ ] } } +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/pipelines.foo.permissions", + "q": { + "update_mask": "state,error_message,resource_id,status" + }, + "body": { + "state": "{\"state\":{\"object_id\":\"/pipelines/[UUID]\",\"__embed__\":[{\"level\":\"CAN_VIEW\",\"user_name\":\"viewer@example.com\"},{\"level\":\"CAN_MANAGE\",\"group_name\":\"data-team\"},{\"level\":\"CAN_MANAGE\",\"service_principal_name\":\"[UUID]\"},{\"level\":\"IS_OWNER\",\"user_name\":\"[USERNAME]\"}]},\"depends_on\":[{\"node\":\"resources.pipelines.foo\",\"label\":\"${resources.pipelines.foo.id}\"}]}", + "resource_id": "/pipelines/[UUID]", + "status": "OPERATION_STATUS_SUCCEEDED", + "sequence_id": "0" + } +} diff --git a/acceptance/bundle/resources/permissions/pipelines/other_can_manage/out.requests.destroy.direct.json b/acceptance/bundle/resources/permissions/pipelines/other_can_manage/out.requests.destroy.direct.json index e69de29bb2d..beda36a358c 100644 --- a/acceptance/bundle/resources/permissions/pipelines/other_can_manage/out.requests.destroy.direct.json +++ b/acceptance/bundle/resources/permissions/pipelines/other_can_manage/out.requests.destroy.direct.json @@ -0,0 +1,12 @@ +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations/pipelines.foo.permissions", + "q": { + "update_mask": "state,error_message,resource_id,status" + }, + "body": { + "resource_id": "/pipelines/[UUID]", + "status": "OPERATION_STATUS_SUCCEEDED", + "sequence_id": "0" + } +} diff --git a/acceptance/bundle/resources/permissions/pipelines/other_is_owner/out.requests.deploy.direct.json b/acceptance/bundle/resources/permissions/pipelines/other_is_owner/out.requests.deploy.direct.json index 8925d4f66d4..3c80374a684 100644 --- a/acceptance/bundle/resources/permissions/pipelines/other_is_owner/out.requests.deploy.direct.json +++ b/acceptance/bundle/resources/permissions/pipelines/other_is_owner/out.requests.deploy.direct.json @@ -14,3 +14,16 @@ ] } } +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/pipelines.foo.permissions", + "q": { + "update_mask": "state,error_message,resource_id,status" + }, + "body": { + "state": "{\"state\":{\"object_id\":\"/pipelines/[UUID]\",\"__embed__\":[{\"level\":\"IS_OWNER\",\"user_name\":\"other_user@databricks.com\"},{\"level\":\"CAN_MANAGE\",\"user_name\":\"[USERNAME]\"}]},\"depends_on\":[{\"node\":\"resources.pipelines.foo\",\"label\":\"${resources.pipelines.foo.id}\"}]}", + "resource_id": "/pipelines/[UUID]", + "status": "OPERATION_STATUS_SUCCEEDED", + "sequence_id": "0" + } +} diff --git a/acceptance/bundle/resources/permissions/pipelines/other_is_owner/out.requests.destroy.direct.json b/acceptance/bundle/resources/permissions/pipelines/other_is_owner/out.requests.destroy.direct.json index e69de29bb2d..beda36a358c 100644 --- a/acceptance/bundle/resources/permissions/pipelines/other_is_owner/out.requests.destroy.direct.json +++ b/acceptance/bundle/resources/permissions/pipelines/other_is_owner/out.requests.destroy.direct.json @@ -0,0 +1,12 @@ +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations/pipelines.foo.permissions", + "q": { + "update_mask": "state,error_message,resource_id,status" + }, + "body": { + "resource_id": "/pipelines/[UUID]", + "status": "OPERATION_STATUS_SUCCEEDED", + "sequence_id": "0" + } +} diff --git a/acceptance/bundle/resources/permissions/postgres_projects/current_can_manage/out.requests.deploy.direct.json b/acceptance/bundle/resources/permissions/postgres_projects/current_can_manage/out.requests.deploy.direct.json index 673b537f7f4..33472c67bf6 100644 --- a/acceptance/bundle/resources/permissions/postgres_projects/current_can_manage/out.requests.deploy.direct.json +++ b/acceptance/bundle/resources/permissions/postgres_projects/current_can_manage/out.requests.deploy.direct.json @@ -22,3 +22,16 @@ ] } } +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/postgres_projects.foo.permissions", + "q": { + "update_mask": "state,error_message,resource_id,status" + }, + "body": { + "state": "{\"state\":{\"object_id\":\"/database-projects/test-project\",\"__embed__\":[{\"level\":\"CAN_USE\",\"user_name\":\"viewer@example.com\"},{\"level\":\"CAN_MANAGE\",\"group_name\":\"data-team\"},{\"level\":\"CAN_MANAGE\",\"service_principal_name\":\"[UUID]\"},{\"level\":\"CAN_MANAGE\",\"user_name\":\"[USERNAME]\"}]},\"depends_on\":[{\"node\":\"resources.postgres_projects.foo\",\"label\":\"${resources.postgres_projects.foo.project_id}\"}]}", + "resource_id": "/database-projects/test-project", + "status": "OPERATION_STATUS_SUCCEEDED", + "sequence_id": "0" + } +} diff --git a/acceptance/bundle/resources/permissions/postgres_projects/current_can_manage/out.requests.destroy.direct.json b/acceptance/bundle/resources/permissions/postgres_projects/current_can_manage/out.requests.destroy.direct.json index e69de29bb2d..4a85a73dced 100644 --- a/acceptance/bundle/resources/permissions/postgres_projects/current_can_manage/out.requests.destroy.direct.json +++ b/acceptance/bundle/resources/permissions/postgres_projects/current_can_manage/out.requests.destroy.direct.json @@ -0,0 +1,12 @@ +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations/postgres_projects.foo.permissions", + "q": { + "update_mask": "state,error_message,resource_id,status" + }, + "body": { + "resource_id": "/database-projects/test-project", + "status": "OPERATION_STATUS_SUCCEEDED", + "sequence_id": "0" + } +} diff --git a/acceptance/bundle/resources/permissions/sql_warehouses/current_can_manage/out.requests.deploy.direct.json b/acceptance/bundle/resources/permissions/sql_warehouses/current_can_manage/out.requests.deploy.direct.json index 2ef440a5941..ab940b519df 100644 --- a/acceptance/bundle/resources/permissions/sql_warehouses/current_can_manage/out.requests.deploy.direct.json +++ b/acceptance/bundle/resources/permissions/sql_warehouses/current_can_manage/out.requests.deploy.direct.json @@ -22,3 +22,16 @@ ] } } +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/sql_warehouses.foo.permissions", + "q": { + "update_mask": "state,error_message,resource_id,status" + }, + "body": { + "state": "{\"state\":{\"object_id\":\"/sql/warehouses/[UUID]\",\"__embed__\":[{\"level\":\"CAN_VIEW\",\"user_name\":\"viewer@example.com\"},{\"level\":\"CAN_MANAGE\",\"group_name\":\"data-team\"},{\"level\":\"CAN_MANAGE\",\"service_principal_name\":\"[UUID]\"},{\"level\":\"IS_OWNER\",\"user_name\":\"[USERNAME]\"}]},\"depends_on\":[{\"node\":\"resources.sql_warehouses.foo\",\"label\":\"${resources.sql_warehouses.foo.id}\"}]}", + "resource_id": "/sql/warehouses/[UUID]", + "status": "OPERATION_STATUS_SUCCEEDED", + "sequence_id": "0" + } +} diff --git a/acceptance/bundle/resources/permissions/sql_warehouses/current_can_manage/out.requests.destroy.direct.json b/acceptance/bundle/resources/permissions/sql_warehouses/current_can_manage/out.requests.destroy.direct.json index e69de29bb2d..f95b0b7057b 100644 --- a/acceptance/bundle/resources/permissions/sql_warehouses/current_can_manage/out.requests.destroy.direct.json +++ b/acceptance/bundle/resources/permissions/sql_warehouses/current_can_manage/out.requests.destroy.direct.json @@ -0,0 +1,12 @@ +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations/sql_warehouses.foo.permissions", + "q": { + "update_mask": "state,error_message,resource_id,status" + }, + "body": { + "resource_id": "/sql/warehouses/[UUID]", + "status": "OPERATION_STATUS_SUCCEEDED", + "sequence_id": "0" + } +} diff --git a/acceptance/bundle/resources/permissions/vector_search_endpoints/current_can_manage/out.requests.deploy.direct.json b/acceptance/bundle/resources/permissions/vector_search_endpoints/current_can_manage/out.requests.deploy.direct.json index 9118a4da780..c2fbf21ae83 100644 --- a/acceptance/bundle/resources/permissions/vector_search_endpoints/current_can_manage/out.requests.deploy.direct.json +++ b/acceptance/bundle/resources/permissions/vector_search_endpoints/current_can_manage/out.requests.deploy.direct.json @@ -22,3 +22,16 @@ ] } } +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/vector_search_endpoints.foo.permissions", + "q": { + "update_mask": "state,error_message,resource_id,status" + }, + "body": { + "state": "{\"state\":{\"object_id\":\"/vector-search-endpoints/[UUID]\",\"__embed__\":[{\"level\":\"CAN_USE\",\"user_name\":\"viewer@example.com\"},{\"level\":\"CAN_MANAGE\",\"group_name\":\"data-team\"},{\"level\":\"CAN_MANAGE\",\"service_principal_name\":\"[UUID]\"},{\"level\":\"CAN_MANAGE\",\"user_name\":\"[USERNAME]\"}]},\"depends_on\":[{\"node\":\"resources.vector_search_endpoints.foo\",\"label\":\"${resources.vector_search_endpoints.foo.endpoint_uuid}\"}]}", + "resource_id": "/vector-search-endpoints/[UUID]", + "status": "OPERATION_STATUS_SUCCEEDED", + "sequence_id": "0" + } +} diff --git a/acceptance/bundle/resources/permissions/vector_search_endpoints/current_can_manage/out.requests.destroy.direct.json b/acceptance/bundle/resources/permissions/vector_search_endpoints/current_can_manage/out.requests.destroy.direct.json index 84c87416aa2..cd017d6b986 100644 --- a/acceptance/bundle/resources/permissions/vector_search_endpoints/current_can_manage/out.requests.destroy.direct.json +++ b/acceptance/bundle/resources/permissions/vector_search_endpoints/current_can_manage/out.requests.destroy.direct.json @@ -2,3 +2,15 @@ "method": "DELETE", "path": "/api/2.0/vector-search/endpoints/vs-permissions-endpoint" } +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations/vector_search_endpoints.foo.permissions", + "q": { + "update_mask": "state,error_message,resource_id,status" + }, + "body": { + "resource_id": "/vector-search-endpoints/[UUID]", + "status": "OPERATION_STATUS_SUCCEEDED", + "sequence_id": "0" + } +} diff --git a/acceptance/bundle/resources/pipelines/allow-duplicate-names/output.txt b/acceptance/bundle/resources/pipelines/allow-duplicate-names/output.txt index 4ddbabbe7ed..6f57e516c04 100644 --- a/acceptance/bundle/resources/pipelines/allow-duplicate-names/output.txt +++ b/acceptance/bundle/resources/pipelines/allow-duplicate-names/output.txt @@ -41,6 +41,19 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "name": "test-pipeline-same-name-[UNIQUE_NAME]" } } +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/pipelines.pipeline_one", + "q": { + "update_mask": "state,error_message,resource_id,status" + }, + "body": { + "state": "{\"state\":{\"allow_duplicate_names\":true,\"channel\":\"CURRENT\",\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/acc-bundle-deploy-pipeline-duplicate-names-[UNIQUE_NAME]/default/state/metadata.json\",\"version_id\":\"1\"},\"edition\":\"ADVANCED\",\"libraries\":[{\"file\":{\"path\":\"/Workspace/Users/[USERNAME]/.bundle/acc-bundle-deploy-pipeline-duplicate-names-[UNIQUE_NAME]/default/files/foo.py\"}}],\"name\":\"test-pipeline-same-name-[UNIQUE_NAME]\"}}", + "resource_id": "[UUID]", + "status": "OPERATION_STATUS_SUCCEEDED", + "sequence_id": "0" + } +} >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: diff --git a/acceptance/bundle/run_as/job_default/output.txt b/acceptance/bundle/run_as/job_default/output.txt index 21613bc205a..ec9405720d5 100644 --- a/acceptance/bundle/run_as/job_default/output.txt +++ b/acceptance/bundle/run_as/job_default/output.txt @@ -40,6 +40,19 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged ] } } +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.job_with_run_as", + "q": { + "update_mask": "state,error_message,resource_id,status" + }, + "body": { + "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/run_as_job_default_[UNIQUE_NAME]/default/state/metadata.json\",\"version_id\":\"1\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"Untitled\",\"queue\":{\"enabled\":true},\"run_as\":{\"user_name\":\"deco-test-user@databricks.com\"},\"tasks\":[{\"new_cluster\":{\"node_type_id\":\"[NODE_TYPE_ID]\",\"num_workers\":1,\"spark_version\":\"13.3.x-snapshot-scala2.12\"},\"notebook_task\":{\"notebook_path\":\"/Workspace/Users/[USERNAME]/.bundle/run_as_job_default_[UNIQUE_NAME]/default/files/test\"},\"task_key\":\"task_one\"}]}}", + "resource_id": "[NUMID]", + "status": "OPERATION_STATUS_SUCCEEDED", + "sequence_id": "0" + } +} >>> [CLI] jobs get [NUMID] { @@ -92,6 +105,19 @@ Resources: 0 created, 1 changed, 0 deleted, 0 unchanged } } } +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations/jobs.job_with_run_as", + "q": { + "update_mask": "state,error_message,resource_id,status" + }, + "body": { + "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/run_as_job_default_[UNIQUE_NAME]/default/state/metadata.json\",\"version_id\":\"2\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"Untitled\",\"queue\":{\"enabled\":true},\"tasks\":[{\"new_cluster\":{\"node_type_id\":\"[NODE_TYPE_ID]\",\"num_workers\":1,\"spark_version\":\"13.3.x-snapshot-scala2.12\"},\"notebook_task\":{\"notebook_path\":\"/Workspace/Users/[USERNAME]/.bundle/run_as_job_default_[UNIQUE_NAME]/default/files/test\"},\"task_key\":\"task_one\"}]}}", + "resource_id": "[NUMID]", + "status": "OPERATION_STATUS_SUCCEEDED", + "sequence_id": "0" + } +} >>> [CLI] jobs get [NUMID] { diff --git a/acceptance/bundle/templates/default-sql-catalog-dash/out.test.toml b/acceptance/bundle/templates/default-sql-catalog-dash/out.test.toml index 98ea5040486..f61c2bccd55 100644 --- a/acceptance/bundle/templates/default-sql-catalog-dash/out.test.toml +++ b/acceptance/bundle/templates/default-sql-catalog-dash/out.test.toml @@ -1,2 +1,3 @@ Cloud = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/validate/secret_scope_invalid_permission_level/out.test.toml b/acceptance/bundle/validate/secret_scope_invalid_permission_level/out.test.toml index 98ea5040486..c7a035e8011 100644 --- a/acceptance/bundle/validate/secret_scope_invalid_permission_level/out.test.toml +++ b/acceptance/bundle/validate/secret_scope_invalid_permission_level/out.test.toml @@ -1,2 +1,3 @@ Cloud = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/bundle/direct/bundle_apply.go b/bundle/direct/bundle_apply.go index c30f2fb48e3..85567cc5908 100644 --- a/bundle/direct/bundle_apply.go +++ b/bundle/direct/bundle_apply.go @@ -105,8 +105,7 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa err = d.Destroy(ctx, &b.StateDB) } if err != nil { - _, priorState := priorRecord(&b.StateDB, resourceKey) - opSink.recordFailure(ctx, resourceKey, action, deletedID, priorState, err) + opSink.recordFailure(ctx, resourceKey, action, deletedID, err) logdiag.LogError(ctx, fmt.Errorf("%s: %w", errorPrefix, err)) return false } @@ -138,10 +137,10 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa // each of its steps. err = d.Deploy(ctx, &b.StateDB, sv.Value, action, entry) if err != nil { - // Both are empty for a create that never got an ID, which is what the - // service expects for a failed create. - priorID, priorState := priorRecord(&b.StateDB, resourceKey) - opSink.recordFailure(ctx, resourceKey, action, priorID, priorState, err) + // Empty for a create that never got an ID, and for a recreate whose delete + // step already dropped it. + failedID := b.StateDB.GetResourceID(resourceKey) + opSink.recordFailure(ctx, resourceKey, action, failedID, err) logdiag.LogError(ctx, fmt.Errorf("%s: %w", errorPrefix, err)) return false } diff --git a/bundle/direct/opclient.go b/bundle/direct/opclient.go index d9d1e53df4c..97e3ced94eb 100644 --- a/bundle/direct/opclient.go +++ b/bundle/direct/opclient.go @@ -31,11 +31,10 @@ type updateOperationRequest struct { SequenceId string `json:"sequence_id,omitempty"` } -// operationClient records operations under a deployment version. UpdateOperation -// takes the fields to update, because a failure updates fewer of them than a state -// write does. +// operationClient fills in the operations a version staged. There is no create: the version +// records the whole set at CreateVersion, so every write here narrows an existing operation, +// and fields says which of them it changes. type operationClient interface { - CreateOperation(ctx context.Context, parent, resourceKey string, op bundledeployments.Operation) (operationResponse, error) UpdateOperation(ctx context.Context, parent, resourceKey string, fields []string, body updateOperationRequest) (operationResponse, error) } @@ -49,19 +48,6 @@ func newAPIOperationClient(c *client.DatabricksClient) operationClient { return &apiOperationClient{client: c} } -func (a *apiOperationClient) CreateOperation(ctx context.Context, parent, resourceKey string, op bundledeployments.Operation) (operationResponse, error) { - var result operationResponse - path := fmt.Sprintf("/api/2.0/bundle/%s/operations", parent) - err := a.client.Do(ctx, http.MethodPost, path, - auth.WorkspaceIDHeaders(a.client.Config), - map[string]any{"resource_key": resourceKey}, - op, &result) - if err != nil { - return operationResponse{}, err - } - return result, nil -} - func (a *apiOperationClient) UpdateOperation(ctx context.Context, parent, resourceKey string, fields []string, body updateOperationRequest) (operationResponse, error) { var result operationResponse path := fmt.Sprintf("/api/2.0/bundle/%s/operations/%s", parent, resourceKey) diff --git a/bundle/direct/oprecorder.go b/bundle/direct/oprecorder.go index f7679bbbe3e..8222d0326f1 100644 --- a/bundle/direct/oprecorder.go +++ b/bundle/direct/oprecorder.go @@ -29,6 +29,10 @@ const maxOperationErrorMessageSize = 16 * 1024 // (databricks-eng/universe#2394529). const operationStatusInProgress bundledeployments.OperationStatus = "OPERATION_STATUS_IN_PROGRESS" +// stagedSequenceID is what CreateVersion leaves on every operation it stages, and so the +// precondition for the first update of a resource. +const stagedSequenceID = "0" + // recordedOperation is an applied resource operation waiting to be uploaded. It is built on // the apply worker, not in the uploader, so a malformed state fails the resource that // produced it rather than the drain at the end of apply. @@ -63,7 +67,7 @@ var failedKeepingState = []string{"error_message", "status"} // RecordedState envelope the state DB just persisted, and nil for a delete, where // the resource is gone. It errors when the state exceeds maxOperationStateSize. func newStateOperation(info dstate.OperationInfo, resourceID string, state json.RawMessage) (recordedOperation, error) { - actionType, err := deployActionToSDK(info.Action) + actionType, err := DeployActionToSDK(info.Action) if err != nil { return recordedOperation{}, err } @@ -87,19 +91,14 @@ func newStateOperation(info dstate.OperationInfo, resourceID string, state json. } // newFailedOperation records an operation that did not apply, so the history says why a -// resource failed rather than omitting it. priorState (nil for a create, as resourceID may -// also be) reaches the service only when nothing else was recorded for the resource yet. -func newFailedOperation(action deployplan.ActionType, resourceID string, priorState json.RawMessage, cause error) (recordedOperation, error) { - actionType, err := deployActionToSDK(action) +// resource failed rather than leaving it pending. It carries no state: the version staged the +// operation already, so a failure only ever narrows an existing record. +func newFailedOperation(action deployplan.ActionType, resourceID string, cause error) (recordedOperation, error) { + actionType, err := DeployActionToSDK(action) if err != nil { return recordedOperation{}, err } - // A guard: the state DB accepted this state, so it was within the limit when written. - if len(priorState) > maxOperationStateSize { - return recordedOperation{}, fmt.Errorf("serialized state is %d bytes, which exceeds the %d byte limit for recording deployment history", len(priorState), maxOperationStateSize) - } - // Summarized, not cause.Error(): for an API failure that adds the status and error // code, which is often the most actionable part of the history. message := diag.FormatAPIErrorSummary(cause) @@ -121,27 +120,10 @@ func newFailedOperation(action deployplan.ActionType, resourceID string, priorSt resourceID: resourceID, status: bundledeployments.OperationStatusOperationStatusFailed, errorMessage: message, - state: priorState, updateFields: failedKeepingState, }, nil } -// priorRecord returns the resource's id and state from before this deploy, in the envelope -// form the success path uploads, or empty values when there is no prior record. Both come -// from one entry: the service rejects state without an id. -func priorRecord(db *dstate.DeploymentState, resourceKey string) (string, json.RawMessage) { - entry, ok := db.GetResourceEntry(resourceKey) - if !ok || len(entry.State) == 0 { - return "", nil - } - - raw, err := json.Marshal(dstate.RecordedState{State: entry.State, DependsOn: entry.DependsOn}) - if err != nil { - return "", nil - } - return entry.ID, raw -} - // operationUploader records an applied resource operation with DMS. Uploads run on // the operationSink goroutine, off the apply path. type operationUploader interface { @@ -158,9 +140,9 @@ type operationRecorder struct { // mu guards sequenceIDs. mu sync.Mutex - // sequenceIDs holds the sequence id the service returned per resource key: both how an - // already-recorded resource is recognised and the precondition for updating it. The - // service keeps one operation per resource per version, so a second write must update it. + // sequenceIDs holds the sequence id the service last returned per resource key, echoed as + // the precondition on the next update. A key absent from the map has not been written yet, + // so its staged operation is still at stagedSequenceID. sequenceIDs map[string]string } @@ -197,35 +179,31 @@ func (r *operationRecorder) upload(ctx context.Context, resourceKey string, op r } r.mu.Lock() - sequenceID, recorded := r.sequenceIDs[dmsKey] + sequenceID, written := r.sequenceIDs[dmsKey] r.mu.Unlock() + if !written { + sequenceID = stagedSequenceID + } - var result operationResponse - var err error - if recorded { - update := updateOperationRequest{ - ErrorMessage: operation.ErrorMessage, - Status: operation.Status, - SequenceId: sequenceID, - } - // Send only what the mask names. The service would ignore the rest, and state is - // the largest field by far, so a failure that keeps the recorded state sends none. - if slices.Contains(op.updateFields, "state") { - update.State = operation.State - update.ResourceId = operation.ResourceId - } - - // action_type stays as the operation was created, so sending it would just be - // misleading. - result, err = r.ops.UpdateOperation(ctx, r.parent, dmsKey, op.updateFields, update) - } else { - result, err = r.ops.CreateOperation(ctx, r.parent, dmsKey, operation) + update := updateOperationRequest{ + ErrorMessage: operation.ErrorMessage, + Status: operation.Status, + SequenceId: sequenceID, + } + // Send only what the mask names. The service would ignore the rest, and state is the + // largest field by far, so a failure that keeps the recorded state sends none. + if slices.Contains(op.updateFields, "state") { + update.State = operation.State + update.ResourceId = operation.ResourceId } + + // action_type is fixed when the version stages the operation, so it is not sent. + result, err := r.ops.UpdateOperation(ctx, r.parent, dmsKey, op.updateFields, update) if err != nil { return err } - // The next write for this resource updates this operation rather than re-creating it. + // The next write for this resource echoes the sequence id this one earned. r.mu.Lock() r.sequenceIDs[dmsKey] = result.SequenceId r.mu.Unlock() @@ -233,10 +211,10 @@ func (r *operationRecorder) upload(ctx context.Context, resourceKey string, op r return nil } -// deployActionToSDK maps a deployplan action to its DMS operation action type. +// DeployActionToSDK maps a deployplan action to its DMS operation action type. // Only actions that mutate a resource are recordable; Skip and Undefined never // reach a recorder and are rejected rather than silently coerced. -func deployActionToSDK(a deployplan.ActionType) (bundledeployments.OperationActionType, error) { +func DeployActionToSDK(a deployplan.ActionType) (bundledeployments.OperationActionType, error) { switch a { case deployplan.Create: return bundledeployments.OperationActionTypeOperationActionTypeCreate, nil diff --git a/bundle/direct/oprecorder_test.go b/bundle/direct/oprecorder_test.go index 033df1215b6..ce62b39dd7d 100644 --- a/bundle/direct/oprecorder_test.go +++ b/bundle/direct/oprecorder_test.go @@ -21,7 +21,6 @@ type fakeOpCall struct { method string parent string resourceKey string - op bundledeployments.Operation update updateOperationRequest fields []string } @@ -33,13 +32,6 @@ type fakeOpClient struct { sequence string } -func (f *fakeOpClient) CreateOperation(ctx context.Context, parent, resourceKey string, op bundledeployments.Operation) (operationResponse, error) { - f.mu.Lock() - defer f.mu.Unlock() - f.calls = append(f.calls, fakeOpCall{method: "create", parent: parent, resourceKey: resourceKey, op: op}) - return operationResponse{SequenceId: f.sequence}, nil -} - func (f *fakeOpClient) UpdateOperation(ctx context.Context, parent, resourceKey string, fields []string, body updateOperationRequest) (operationResponse, error) { f.mu.Lock() defer f.mu.Unlock() @@ -64,15 +56,14 @@ func TestOperationRecorderStripsResourcePrefix(t *testing.T) { require.Len(t, f.calls, 1) c := f.calls[0] - // The wire key drops the CLI-internal "resources." prefix, both in the query - // param and the operation body. - assert.Equal(t, "create", c.method) + // The version already staged this operation, so the first write updates it and echoes + // the sequence id staging left. The wire key drops the CLI-internal "resources." prefix. + assert.Equal(t, "update", c.method) assert.Equal(t, "jobs.foo", c.resourceKey) - assert.Equal(t, "jobs.foo", c.op.ResourceKey) assert.Equal(t, "deployments/dep-1/versions/2", c.parent) - assert.Equal(t, bundledeployments.OperationActionTypeOperationActionTypeCreate, c.op.ActionType) - assert.Equal(t, "job-123", c.op.ResourceId) - require.NotEmpty(t, c.op.State) + assert.Equal(t, stagedSequenceID, c.update.SequenceId) + assert.Equal(t, "job-123", c.update.ResourceId) + require.NotEmpty(t, c.update.State) } func TestOperationRecorderUpdatesSecondWriteForSameResource(t *testing.T) { @@ -85,7 +76,7 @@ func TestOperationRecorderUpdatesSecondWriteForSameResource(t *testing.T) { uploadOne(t, r, "resources.jobs.foo", deployplan.Recreate, "job-456", envelope(t, "new")) require.Len(t, f.calls, 2) - assert.Equal(t, "create", f.calls[0].method) + assert.Equal(t, stagedSequenceID, f.calls[0].update.SequenceId) assert.Equal(t, "update", f.calls[1].method) assert.Equal(t, "jobs.foo", f.calls[1].resourceKey) @@ -102,12 +93,12 @@ func TestOperationRecorderFailureAfterAStateWriteKeepsTheState(t *testing.T) { uploadOne(t, r, "resources.job_runs.my_run", deployplan.Create, "run-1", envelope(t, "the run")) - failed, err := newFailedOperation(deployplan.Create, "", nil, errors.New("run did not succeed: FAILED")) + failed, err := newFailedOperation(deployplan.Create, "", errors.New("run did not succeed: FAILED")) require.NoError(t, err) require.NoError(t, r.upload(t.Context(), "resources.job_runs.my_run", failed)) require.Len(t, f.calls, 2) - assert.Equal(t, "create", f.calls[0].method) + assert.Equal(t, "update", f.calls[0].method) update := f.calls[1] assert.Equal(t, "update", update.method) @@ -127,7 +118,7 @@ func TestOperationRecorderFailedRecreateKeepsTheResourceGone(t *testing.T) { uploadOne(t, r, "resources.jobs.foo", deployplan.Recreate, "old-id", nil) - failed, err := newFailedOperation(deployplan.Recreate, "old-id", envelope(t, "before the deploy"), errors.New("boom")) + failed, err := newFailedOperation(deployplan.Recreate, "old-id", errors.New("boom")) require.NoError(t, err) require.NoError(t, r.upload(t.Context(), "resources.jobs.foo", failed)) @@ -150,7 +141,7 @@ func TestOperationRecorderFailureCarryingALaterWriteSendsIt(t *testing.T) { second, err := newStateOperation(dstate.OperationInfo{Action: deployplan.Update}, "id-1", envelope(t, "second write")) require.NoError(t, err) - failed, err := newFailedOperation(deployplan.Update, "id-old", envelope(t, "before the deploy"), errors.New("boom")) + failed, err := newFailedOperation(deployplan.Update, "id-old", errors.New("boom")) require.NoError(t, err) require.NoError(t, r.upload(t.Context(), "resources.jobs.foo", coalesce(second, failed))) @@ -162,24 +153,27 @@ func TestOperationRecorderFailureCarryingALaterWriteSendsIt(t *testing.T) { assert.Equal(t, "id-1", update.update.ResourceId) } -func TestOperationRecorderFailureBeforeAnyWriteCarriesPriorState(t *testing.T) { - // Nothing was recorded yet, so the failure creates the operation and carries the prior - // state. Without it the resource is dropped and the next plan creates a second one. +func TestOperationRecorderFailureBeforeAnyWriteNarrowsTheStagedOperation(t *testing.T) { + // Nothing was written for the resource, so the failure updates the operation the version + // staged, at the sequence id staging left. It sends no state: the resource was not + // touched, and the staged operation already holds whatever the deployment knows. f := &fakeOpClient{sequence: "1"} r := newOperationRecorder(f, "dep-1", 2) - failed, err := newFailedOperation(deployplan.Update, "main.some_schema", envelope(t, "before"), errors.New("boom")) + failed, err := newFailedOperation(deployplan.Update, "main.some_schema", errors.New("boom")) require.NoError(t, err) require.NoError(t, r.upload(t.Context(), "resources.schemas.foo", failed)) require.Len(t, f.calls, 1) - assert.Equal(t, "create", f.calls[0].method) - assert.Equal(t, "main.some_schema", f.calls[0].op.ResourceId) - require.NotEmpty(t, f.calls[0].op.State) + assert.Equal(t, "update", f.calls[0].method) + assert.Equal(t, stagedSequenceID, f.calls[0].update.SequenceId) + assert.Equal(t, []string{"error_message", "status"}, f.calls[0].fields) + assert.Empty(t, f.calls[0].update.State) } func TestOperationRecorderTracksSequencePerResource(t *testing.T) { - // A different resource has its own operation, so its first write creates. + // Each resource has its own staged operation, so each one's first write echoes the staged + // sequence id rather than a sequence another resource earned. f := &fakeOpClient{sequence: "1"} r := newOperationRecorder(f, "dep-1", 2) @@ -187,8 +181,8 @@ func TestOperationRecorderTracksSequencePerResource(t *testing.T) { uploadOne(t, r, "resources.jobs.bar", deployplan.Create, "id-2", envelope(t, "bar")) require.Len(t, f.calls, 2) - assert.Equal(t, "create", f.calls[0].method) - assert.Equal(t, "create", f.calls[1].method) + assert.Equal(t, stagedSequenceID, f.calls[0].update.SequenceId) + assert.Equal(t, stagedSequenceID, f.calls[1].update.SequenceId) } func TestNewStateOperationRecordsEnvelopeAsIs(t *testing.T) { @@ -216,7 +210,7 @@ func TestNewStateOperationRejectsOversizedState(t *testing.T) { } func TestNewFailedOperationRecordsError(t *testing.T) { - op, err := newFailedOperation(deployplan.Create, "", nil, errors.New("cluster spec is invalid")) + op, err := newFailedOperation(deployplan.Create, "", errors.New("cluster spec is invalid")) require.NoError(t, err) assert.Equal(t, bundledeployments.OperationStatusOperationStatusFailed, op.status) @@ -227,29 +221,10 @@ func TestNewFailedOperationRecordsError(t *testing.T) { assert.Equal(t, failedKeepingState, op.updateFields) } -func TestNewFailedOperationRecordsPriorStateWithID(t *testing.T) { - // A failed recreate has already deleted the resource, so the id must come from - // the pre-deploy record alongside the state: the service rejects state without - // an id, since state describes a resource that exists. - op, err := newFailedOperation(deployplan.Recreate, "main.some_schema", json.RawMessage(`{"state":{"catalog_name":"main"}}`), errors.New("Catalog 'mainx' does not exist")) - require.NoError(t, err) - - assert.Equal(t, bundledeployments.OperationStatusOperationStatusFailed, op.status) - assert.Equal(t, "main.some_schema", op.resourceID) - assert.JSONEq(t, `{"state":{"catalog_name":"main"}}`, string(op.state)) -} - -func TestNewFailedOperationRejectsOversizedPriorState(t *testing.T) { - big := json.RawMessage(strings.Repeat("x", maxOperationStateSize+1)) - - _, err := newFailedOperation(deployplan.Update, "job-123", big, errors.New("boom")) - assert.ErrorContains(t, err, "exceeds the 65536 byte limit") -} - func TestNewFailedOperationTruncatesLongError(t *testing.T) { // Truncated rather than rejected: a message over the limit would make recording // fail and hide the error it is reporting. - op, err := newFailedOperation(deployplan.Update, "job-123", nil, errors.New(strings.Repeat("x", maxOperationErrorMessageSize+100))) + op, err := newFailedOperation(deployplan.Update, "job-123", errors.New(strings.Repeat("x", maxOperationErrorMessageSize+100))) require.NoError(t, err) assert.Len(t, op.errorMessage, maxOperationErrorMessageSize) @@ -260,7 +235,7 @@ func TestNewFailedOperationPreservesUTF8OnTruncation(t *testing.T) { // rune behind and the service stores state and messages as strings. msg := strings.Repeat("a", maxOperationErrorMessageSize-1) + "❌" + "x" - op, err := newFailedOperation(deployplan.Update, "job-123", nil, errors.New(msg)) + op, err := newFailedOperation(deployplan.Update, "job-123", errors.New(msg)) require.NoError(t, err) assert.True(t, utf8.ValidString(op.errorMessage)) @@ -287,7 +262,7 @@ func TestOperationRecorderReturnsAPIErrors(t *testing.T) { uploadOne(t, r, "resources.jobs.foo", deployplan.Update, "job-3", envelope(t, "third")) require.Len(t, failingClient.calls, 3) - assert.Equal(t, "create", failingClient.calls[0].method) + assert.Equal(t, "update", failingClient.calls[0].method) assert.Equal(t, "update", failingClient.calls[1].method) assert.Equal(t, "update", failingClient.calls[2].method) assert.Equal(t, "9", failingClient.calls[2].update.SequenceId) @@ -301,17 +276,6 @@ type failingOpClient struct { failOn int } -func (f *failingOpClient) CreateOperation(ctx context.Context, parent, resourceKey string, op bundledeployments.Operation) (operationResponse, error) { - f.mu.Lock() - defer f.mu.Unlock() - callNum := len(f.calls) - f.calls = append(f.calls, fakeOpCall{method: "create", parent: parent, resourceKey: resourceKey, op: op}) - if callNum == f.failOn { - return operationResponse{}, errors.New("injected error") - } - return operationResponse{SequenceId: f.sequence}, nil -} - func (f *failingOpClient) UpdateOperation(ctx context.Context, parent, resourceKey string, fields []string, body updateOperationRequest) (operationResponse, error) { f.mu.Lock() defer f.mu.Unlock() @@ -320,7 +284,7 @@ func (f *failingOpClient) UpdateOperation(ctx context.Context, parent, resourceK if callNum == f.failOn { return operationResponse{}, errors.New("injected error") } - return operationResponse{SequenceId: "2"}, nil + return operationResponse{SequenceId: f.sequence}, nil } func TestDeployActionToSDK(t *testing.T) { @@ -336,14 +300,14 @@ func TestDeployActionToSDK(t *testing.T) { {deployplan.Delete, bundledeployments.OperationActionTypeOperationActionTypeDelete}, } for _, c := range cases { - got, err := deployActionToSDK(c.action) + got, err := DeployActionToSDK(c.action) require.NoError(t, err) assert.Equal(t, c.want, got) } // Skip and Undefined never reach a recorder and are rejected. - _, err := deployActionToSDK(deployplan.Skip) + _, err := DeployActionToSDK(deployplan.Skip) assert.Error(t, err) - _, err = deployActionToSDK(deployplan.Undefined) + _, err = DeployActionToSDK(deployplan.Undefined) assert.Error(t, err) } diff --git a/bundle/direct/opsink.go b/bundle/direct/opsink.go index 540f276f1c6..53ffe145632 100644 --- a/bundle/direct/opsink.go +++ b/bundle/direct/opsink.go @@ -80,12 +80,12 @@ func (s *operationSink) RecordOperation(ctx context.Context, resourceKey string, // recordFailure records that a resource did not apply, so the history says why rather // than leaving the resource out. -func (s *operationSink) recordFailure(ctx context.Context, resourceKey string, action deployplan.ActionType, resourceID string, priorState json.RawMessage, cause error) { +func (s *operationSink) recordFailure(ctx context.Context, resourceKey string, action deployplan.ActionType, resourceID string, cause error) { if s == nil { return } - op, err := newFailedOperation(action, resourceID, priorState, cause) + op, err := newFailedOperation(action, resourceID, cause) if err != nil { s.setErr(fmt.Errorf("recording failure for %s: %w", resourceKey, err)) return diff --git a/bundle/direct/opsink_test.go b/bundle/direct/opsink_test.go index d7a6a7c79f3..aaa2a6bbd8d 100644 --- a/bundle/direct/opsink_test.go +++ b/bundle/direct/opsink_test.go @@ -165,7 +165,7 @@ func TestOperationSinkCoalescedFailureKeepsTheStateItReplaces(t *testing.T) { s.RecordOperation(t.Context(), "resources.job_runs.my_run", dstate.OperationInfo{Action: deployplan.Create}, "run-1", envelope(t, "the run")) // priorState and priorID are empty: the resource was created in this deploy, so // there is no pre-deploy record to report. - s.recordFailure(t.Context(), "resources.job_runs.my_run", deployplan.Create, "", nil, errors.New("run did not succeed: FAILED")) + s.recordFailure(t.Context(), "resources.job_runs.my_run", deployplan.Create, "", errors.New("run did not succeed: FAILED")) close(f.block) require.NoError(t, s.close()) @@ -191,7 +191,7 @@ func TestOperationSinkCoalescedFailureAfterADeleteKeepsTheResourceGone(t *testin assert.Equal(t, "resources.jobs.busy", <-f.started) s.RecordOperation(t.Context(), "resources.schemas.foo", dstate.OperationInfo{Action: deployplan.Recreate, InProgress: true}, "old-id", nil) - s.recordFailure(t.Context(), "resources.schemas.foo", deployplan.Recreate, "old-id", envelope(t, "before the deploy"), errors.New("Catalog 'other' does not exist")) + s.recordFailure(t.Context(), "resources.schemas.foo", deployplan.Recreate, "old-id", errors.New("Catalog 'other' does not exist")) close(f.block) require.NoError(t, s.close()) @@ -216,7 +216,7 @@ func TestOperationSinkCoalescedFailureDoesNotRevertToPriorState(t *testing.T) { assert.Equal(t, "resources.jobs.busy", <-f.started) s.RecordOperation(t.Context(), "resources.jobs.foo", dstate.OperationInfo{Action: deployplan.Update}, "id-new", envelope(t, "after the update")) - s.recordFailure(t.Context(), "resources.jobs.foo", deployplan.Update, "id-old", envelope(t, "before the deploy"), errors.New("waiting after updating: timed out")) + s.recordFailure(t.Context(), "resources.jobs.foo", deployplan.Update, "id-old", errors.New("waiting after updating: timed out")) close(f.block) require.NoError(t, s.close()) @@ -259,7 +259,7 @@ func TestCoalesceLetsAWriteSupersedeAFailure(t *testing.T) { // A failure is not the last word. A retry that writes state wins whole - state, id and // mask - and the mask names error_message so the recorded failure is cleared. The service // rejects a succeeded operation that still carries an error. - failed, err := newFailedOperation(deployplan.Update, "id-old", envelope(t, "before"), errors.New("boom")) + failed, err := newFailedOperation(deployplan.Update, "id-old", errors.New("boom")) require.NoError(t, err) retried, err := newStateOperation(dstate.OperationInfo{Action: deployplan.Update}, "id-new", envelope(t, "after the retry")) require.NoError(t, err) @@ -279,7 +279,7 @@ func TestCoalesceKeepsTheWritesStateAndMask(t *testing.T) { // write as a delete, and the service keeps whichever action created the operation. write, err := newStateOperation(dstate.OperationInfo{Action: deployplan.Delete}, "id-new", nil) require.NoError(t, err) - failed, err := newFailedOperation(deployplan.Update, "id-old", envelope(t, "before"), errors.New("boom")) + failed, err := newFailedOperation(deployplan.Update, "id-old", errors.New("boom")) require.NoError(t, err) got := coalesce(write, failed) @@ -435,7 +435,7 @@ func TestOperationSinkCloseIsIdempotent(t *testing.T) { func TestNilOperationSinkIsNoOp(t *testing.T) { var s *operationSink s.RecordOperation(t.Context(), "resources.jobs.foo", dstate.OperationInfo{Action: deployplan.Create}, "id-1", nil) - s.recordFailure(t.Context(), "resources.jobs.foo", deployplan.Create, "id-1", nil, errors.New("boom")) + s.recordFailure(t.Context(), "resources.jobs.foo", deployplan.Create, "id-1", errors.New("boom")) assert.NoError(t, s.firstErr()) assert.NoError(t, s.close()) } diff --git a/bundle/phases/deploy.go b/bundle/phases/deploy.go index 257703517d1..e2ffa45783d 100644 --- a/bundle/phases/deploy.go +++ b/bundle/phases/deploy.go @@ -347,9 +347,15 @@ func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHand return } - // Create the version the plan was stamped with. Doing it here rather than before - // the prompt means a declined deploy never claims a version number. - if err := recorder.CreateVersion(ctx); err != nil { + // Create the version the plan was stamped with, staging an operation for every resource + // it touches. Doing it here rather than before the prompt means a declined deploy never + // claims a version number. + staged, err := stagedOperations(plan) + if err != nil { + logdiag.LogError(ctx, err) + return + } + if err := recorder.CreateVersion(ctx, staged); err != nil { logdiag.LogError(ctx, err) return } diff --git a/bundle/phases/destroy.go b/bundle/phases/destroy.go index 3405daf7bc0..2ad57e12282 100644 --- a/bundle/phases/destroy.go +++ b/bundle/phases/destroy.go @@ -272,7 +272,12 @@ func Destroy(ctx context.Context, b *bundle.Bundle, engine engine.EngineType) { } // Record the DMS version now that the destroy is approved and the state WAL // has been opened, then record each delete operation under it. - if err := recorder.CreateVersion(ctx); err != nil { + staged, err := stagedOperations(plan) + if err != nil { + logdiag.LogError(ctx, err) + return + } + if err := recorder.CreateVersion(ctx, staged); err != nil { logdiag.LogError(ctx, err) return } diff --git a/bundle/phases/dms.go b/bundle/phases/dms.go index c6e9d5ff259..6bf5b23c30e 100644 --- a/bundle/phases/dms.go +++ b/bundle/phases/dms.go @@ -9,7 +9,9 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/engine" + "github.com/databricks/cli/bundle/deployplan" "github.com/databricks/cli/bundle/direct" + "github.com/databricks/cli/bundle/direct/dstate" "github.com/databricks/cli/bundle/env" "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/dms" @@ -50,6 +52,29 @@ func newDeploymentRecorder(ctx context.Context, b *bundle.Bundle, eng engine.Eng }), nil } +// stagedOperations lists the resources the plan will touch, for CreateVersion to stage an +// operation each. Skipped and undefined actions are left out: nothing is applied for them, so +// their operations would stay pending and the service would hold no state for them. +func stagedOperations(plan *deployplan.Plan) ([]dms.StagedOperation, error) { + actions := plan.GetActions() + staged := make([]dms.StagedOperation, 0, len(actions)) + for _, action := range actions { + if action.ActionType == deployplan.Skip || action.ActionType == deployplan.Undefined { + continue + } + actionType, err := direct.DeployActionToSDK(action.ActionType) + if err != nil { + return nil, fmt.Errorf("%s: %w", action.ResourceKey, err) + } + staged = append(staged, dms.StagedOperation{ + // The service wants the key without the CLI's "resources." prefix. + ResourceKey: strings.TrimPrefix(action.ResourceKey, dstate.ResourceKeyPrefix), + ActionType: actionType, + }) + } + return staged, nil +} + // recordsDeploymentHistory reports whether this bundle records deployment history, // from experimental.record_deployment_history or // DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY. diff --git a/bundle/phases/dms_test.go b/bundle/phases/dms_test.go new file mode 100644 index 00000000000..72567d3a757 --- /dev/null +++ b/bundle/phases/dms_test.go @@ -0,0 +1,55 @@ +package phases + +import ( + "testing" + + "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/cli/libs/dms" + "github.com/databricks/databricks-sdk-go/service/bundledeployments" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestStagedOperationsCoversEveryTouchedResource(t *testing.T) { + // The version fixes its operation set, so anything the apply will write has to appear + // here. Keys go out in the service's form, without the CLI's "resources." prefix. + plan := &deployplan.Plan{Plan: map[string]*deployplan.PlanEntry{ + "resources.jobs.foo": {Action: deployplan.Create}, + "resources.pipelines.bar": {Action: deployplan.Recreate}, + "resources.schemas.baz": {Action: deployplan.Delete}, + "resources.clusters.small": {Action: deployplan.Resize}, + }} + + staged, err := stagedOperations(plan) + require.NoError(t, err) + + assert.ElementsMatch(t, []dms.StagedOperation{ + {ResourceKey: "jobs.foo", ActionType: bundledeployments.OperationActionTypeOperationActionTypeCreate}, + {ResourceKey: "pipelines.bar", ActionType: bundledeployments.OperationActionTypeOperationActionTypeRecreate}, + {ResourceKey: "schemas.baz", ActionType: bundledeployments.OperationActionTypeOperationActionTypeDelete}, + {ResourceKey: "clusters.small", ActionType: bundledeployments.OperationActionTypeOperationActionTypeResize}, + }, staged) +} + +func TestStagedOperationsLeavesOutUntouchedResources(t *testing.T) { + // A skipped resource is never applied, so staging it would leave an operation pending for + // the life of the version. Undefined is not a real action either. + plan := &deployplan.Plan{Plan: map[string]*deployplan.PlanEntry{ + "resources.jobs.touched": {Action: deployplan.Update}, + "resources.jobs.unchanged": {Action: deployplan.Skip}, + "resources.jobs.unknown": {Action: deployplan.Undefined}, + }} + + staged, err := stagedOperations(plan) + require.NoError(t, err) + + assert.Equal(t, []dms.StagedOperation{ + {ResourceKey: "jobs.touched", ActionType: bundledeployments.OperationActionTypeOperationActionTypeUpdate}, + }, staged) +} + +func TestStagedOperationsEmptyPlan(t *testing.T) { + staged, err := stagedOperations(&deployplan.Plan{Plan: map[string]*deployplan.PlanEntry{}}) + require.NoError(t, err) + assert.Empty(t, staged) +} diff --git a/libs/dms/recorder.go b/libs/dms/recorder.go index d3c878479cb..7b7ea499710 100644 --- a/libs/dms/recorder.go +++ b/libs/dms/recorder.go @@ -29,6 +29,19 @@ const ( VersionTypeDestroy VersionType = bundledeployments.VersionTypeVersionTypeDestroy ) +// StagedOperation is one resource the version will record an operation for. The service +// creates it in OPERATION_STATUS_PENDING at sequence_id 0, and the CLI fills in the outcome +// with UpdateOperation as the resource is applied. +// +// Hand-written for the same reason as createVersionRequest: the SDK is generated from the +// OpenAPI spec, which does not carry this message yet. +type StagedOperation struct { + // ResourceKey is the DMS form, without the CLI's "resources." prefix (e.g. "jobs.foo"). + // The service requires a known resource-type prefix and rejects duplicates. + ResourceKey string `json:"resource_key"` + ActionType bundledeployments.OperationActionType `json:"action_type"` +} + // createVersionRequest is the CreateVersion request body. Hand-written because the // generated struct has no previous_version_id, which the service needs as its // concurrency check - without it every deploy after the first is rejected. @@ -48,6 +61,9 @@ type createVersionRequest struct { // where it landed. The service denormalizes both onto the deployment. GitInfo *bundledeployments.GitInfo `json:"git_info,omitempty"` WorkspaceInfo *bundledeployments.WorkspaceInfo `json:"workspace_info,omitempty"` + // Operations is every resource this version will touch. The set is fixed here: the + // service has no API to add one later, so a resource left out cannot be recorded. + Operations []StagedOperation `json:"operations,omitempty"` } // versionCreator creates a version under a deployment. It exists because the @@ -166,9 +182,10 @@ func (r *Recorder) Version() int64 { return r.versionNum } -// CreateVersion registers a new version with DMS, claiming it for the deployment. -// Nil Recorder is a no-op. -func (r *Recorder) CreateVersion(ctx context.Context) error { +// CreateVersion registers a new version with DMS, claiming it for the deployment, and stages +// an operation for every resource in operations. The set cannot be added to later, so a +// resource left out here can never be recorded. Nil Recorder is a no-op. +func (r *Recorder) CreateVersion(ctx context.Context, operations []StagedOperation) error { if r == nil { return nil } @@ -192,10 +209,16 @@ func (r *Recorder) CreateVersion(ctx context.Context) error { DisplayName: r.metadata.DisplayName, PreviousVersionId: r.previousVersionID, DeploymentMode: r.metadata.Mode, + Operations: operations, GitInfo: r.metadata.Git, WorkspaceInfo: r.metadata.Workspace, }) if err != nil { + // The service caps how many operations one version may stage, so a bundle past the + // cap cannot be recorded at all. Say so rather than passing the raw API error on. + if isResourceExhaustedErr(err) { + return fmt.Errorf("this bundle deploys %d resources, more than the deployment metadata service records in one version: %w", len(operations), err) + } // A 409 ABORTED means another deploy claimed this version number in between // PrepareDeployment and here. if isAbortedErr(err) { @@ -371,6 +394,12 @@ func startHeartbeat(ctx context.Context, svc bundledeployments.BundleDeployments } // isAbortedErr reports whether err is an HTTP 409 ABORTED from the DMS API. +// isResourceExhaustedErr reports whether the service refused the call for exceeding a quota. +func isResourceExhaustedErr(err error) bool { + apiErr, ok := errors.AsType[*apierr.APIError](err) + return ok && apiErr.ErrorCode == "RESOURCE_EXHAUSTED" +} + func isAbortedErr(err error) bool { apiErr, ok := errors.AsType[*apierr.APIError](err) return ok && apiErr.StatusCode == http.StatusConflict && apiErr.ErrorCode == "ABORTED" diff --git a/libs/dms/recorder_test.go b/libs/dms/recorder_test.go index 3e11ae20f69..72ccc0bf2ef 100644 --- a/libs/dms/recorder_test.go +++ b/libs/dms/recorder_test.go @@ -93,7 +93,7 @@ func TestRecorderFirstDeployCreatesDeploymentWithServerAssignedID(t *testing.T) // A first deploy resolves no deployment ID from the workspace. r := NewRecorder(RecorderOptions{Service: f, Versions: fakeVersions{requests: &f.versions}, StatePath: testStatePath, Metadata: Metadata{TargetName: "dev", DisplayName: testDisplayName}, VersionType: VersionTypeDeploy}) - require.NoError(t, r.CreateVersion(t.Context())) + require.NoError(t, r.CreateVersion(t.Context(), nil)) // The server assigned the ID, and the recorder exposes it for the rest of the // deploy (it parents the operations recorded under this version). @@ -131,7 +131,7 @@ func TestRecorderSubsequentDeployReusesDeploymentAndIncrementsVersion(t *testing // A subsequent deploy passes the stored deployment ID. r := NewRecorder(RecorderOptions{Service: f, Versions: fakeVersions{requests: &f.versions}, DeploymentID: "stored-id", StatePath: testStatePath, Metadata: Metadata{TargetName: "dev", DisplayName: testDisplayName}, VersionType: VersionTypeDeploy}) - require.NoError(t, r.CreateVersion(t.Context())) + require.NoError(t, r.CreateVersion(t.Context(), nil)) // No new deployment is created; the version increments to last_version_id + 1. assert.Empty(t, f.created) @@ -151,7 +151,7 @@ func TestRecorderGetDeploymentErrorFailsDeploy(t *testing.T) { } r := NewRecorder(RecorderOptions{Service: f, Versions: fakeVersions{requests: &f.versions}, DeploymentID: "stored-id", StatePath: testStatePath, Metadata: Metadata{TargetName: "dev", DisplayName: testDisplayName}, VersionType: VersionTypeDeploy}) - err := r.CreateVersion(t.Context()) + err := r.CreateVersion(t.Context(), nil) assert.ErrorContains(t, err, "failed to get deployment") assert.Empty(t, f.created) } @@ -167,7 +167,7 @@ func TestRecorderMissingDeploymentIsInternalError(t *testing.T) { } r := NewRecorder(RecorderOptions{Service: f, Versions: fakeVersions{requests: &f.versions}, DeploymentID: "stored-id", StatePath: testStatePath, Metadata: Metadata{TargetName: "dev", DisplayName: testDisplayName}, VersionType: VersionTypeDeploy}) - err := r.CreateVersion(t.Context()) + err := r.CreateVersion(t.Context(), nil) assert.ErrorContains(t, err, "internal error: no deployment found for the file with object id stored-id") assert.Empty(t, f.created) assert.Empty(t, f.versions) @@ -181,7 +181,7 @@ func TestRecorderDestroyDeletesDeploymentOnSuccess(t *testing.T) { } r := NewRecorder(RecorderOptions{Service: f, Versions: fakeVersions{requests: &f.versions}, DeploymentID: "stored-id", StatePath: testStatePath, Metadata: Metadata{TargetName: "dev", DisplayName: testDisplayName}, VersionType: VersionTypeDestroy}) - require.NoError(t, r.CreateVersion(t.Context())) + require.NoError(t, r.CreateVersion(t.Context(), nil)) assert.Equal(t, bundledeployments.VersionTypeVersionTypeDestroy, f.versions[0].body.VersionType) require.NoError(t, r.CompleteVersion(t.Context(), true)) @@ -197,7 +197,7 @@ func TestRecorderFailedDestroyKeepsDeployment(t *testing.T) { } r := NewRecorder(RecorderOptions{Service: f, Versions: fakeVersions{requests: &f.versions}, DeploymentID: "stored-id", StatePath: testStatePath, Metadata: Metadata{TargetName: "dev", DisplayName: testDisplayName}, VersionType: VersionTypeDestroy}) - require.NoError(t, r.CreateVersion(t.Context())) + require.NoError(t, r.CreateVersion(t.Context(), nil)) require.NoError(t, r.CompleteVersion(t.Context(), false)) assert.Equal(t, bundledeployments.VersionCompleteVersionCompleteFailure, f.completed[0].CompletionReason) @@ -216,7 +216,7 @@ func TestRecorderCompleteVersionIsIdempotent(t *testing.T) { } r := NewRecorder(RecorderOptions{Service: f, Versions: fakeVersions{requests: &f.versions}, DeploymentID: "stored-id", StatePath: testStatePath, Metadata: Metadata{TargetName: "dev", DisplayName: testDisplayName}, VersionType: VersionTypeDestroy}) - require.NoError(t, r.CreateVersion(t.Context())) + require.NoError(t, r.CreateVersion(t.Context(), nil)) require.NoError(t, r.CompleteVersion(t.Context(), true)) require.NoError(t, r.CompleteVersion(t.Context(), true)) @@ -227,7 +227,7 @@ func TestRecorderCompleteVersionIsIdempotent(t *testing.T) { func TestNilRecorderIsNoOp(t *testing.T) { var r *Recorder - assert.NoError(t, r.CreateVersion(t.Context())) + assert.NoError(t, r.CreateVersion(t.Context(), nil)) assert.NoError(t, r.CompleteVersion(t.Context(), true)) assert.Empty(t, r.DeploymentID()) assert.Zero(t, r.Version()) @@ -270,7 +270,7 @@ func TestRecorderCreateVersionUsesThePreparedNumber(t *testing.T) { r := NewRecorder(RecorderOptions{Service: f, Versions: fakeVersions{requests: &f.versions}, DeploymentID: "stored-id", StatePath: testStatePath, Metadata: Metadata{TargetName: "dev", DisplayName: testDisplayName}, VersionType: VersionTypeDeploy}) require.NoError(t, r.PrepareDeployment(t.Context())) - require.NoError(t, r.CreateVersion(t.Context())) + require.NoError(t, r.CreateVersion(t.Context(), nil)) // The version created is the one the plan was stamped with, and it reports the // version it supersedes so the service rejects a racing deploy. @@ -301,7 +301,7 @@ func TestRecorderCreateVersionDetectsAbortedConflict(t *testing.T) { }) require.NoError(t, r.PrepareDeployment(t.Context())) - err := r.CreateVersion(t.Context()) + err := r.CreateVersion(t.Context(), nil) // Names the version that was taken and tells the user to retry, and keeps the // underlying ABORTED so callers can still match on it. @@ -321,3 +321,34 @@ func TestDeploymentIDFromName(t *testing.T) { _, err = deploymentIDFromName("deployments/") assert.Error(t, err) } + +func TestRecorderCreateVersionStagesOperations(t *testing.T) { + // The version fixes its operation set, so what the caller passes has to reach the wire + // verbatim: the service has no API to add an operation later. + f := &fakeDMS{assignedID: "dep-1"} + r := NewRecorder(RecorderOptions{Service: f, Versions: fakeVersions{requests: &f.versions}, StatePath: testStatePath, Metadata: Metadata{TargetName: "dev", DisplayName: testDisplayName}, VersionType: VersionTypeDeploy}) + + staged := []StagedOperation{ + {ResourceKey: "jobs.foo", ActionType: bundledeployments.OperationActionTypeOperationActionTypeCreate}, + {ResourceKey: "pipelines.bar", ActionType: bundledeployments.OperationActionTypeOperationActionTypeDelete}, + } + require.NoError(t, r.CreateVersion(t.Context(), staged)) + + require.Len(t, f.versions, 1) + assert.Equal(t, staged, f.versions[0].body.Operations) +} + +func TestRecorderCreateVersionReportsTheOperationCap(t *testing.T) { + // A bundle past the service's per-version cap cannot be recorded at all, so say how many + // resources it has rather than passing the raw quota error on. + quotaErr := &apierr.APIError{StatusCode: 429, ErrorCode: "RESOURCE_EXHAUSTED"} + f := &fakeDMS{assignedID: "dep-1"} + r := NewRecorder(RecorderOptions{Service: f, Versions: fakeVersions{requests: &f.versions, err: quotaErr}, StatePath: testStatePath, Metadata: Metadata{TargetName: "dev", DisplayName: testDisplayName}, VersionType: VersionTypeDeploy}) + + err := r.CreateVersion(t.Context(), []StagedOperation{ + {ResourceKey: "jobs.foo", ActionType: bundledeployments.OperationActionTypeOperationActionTypeCreate}, + }) + + assert.ErrorContains(t, err, "this bundle deploys 1 resources") + assert.ErrorIs(t, err, quotaErr) +} diff --git a/libs/testserver/bundle.go b/libs/testserver/bundle.go index aadfa6f2fa4..ef60f0bd129 100644 --- a/libs/testserver/bundle.go +++ b/libs/testserver/bundle.go @@ -3,6 +3,7 @@ package testserver import ( "bytes" "encoding/json" + "fmt" "path" "slices" "strconv" @@ -13,6 +14,15 @@ import ( ) // Handlers for the Deployment Metadata Service (DMS) API under /api/2.0/bundle. + +// maxOperationsPerVersion mirrors the service's compiled-in default. A bundle past it cannot +// be recorded, since the operation set is fixed when the version is created. +const maxOperationsPerVersion = 800 + +// operationStatusPending is what CreateVersion leaves on a staged operation. Declared here +// because the SDK enum is generated from the OpenAPI spec, which trails the service proto. +const operationStatusPending bundledeployments.OperationStatus = "OPERATION_STATUS_PENDING" + // State is kept in FakeWorkspace.dmsDeployments, keyed by deployment ID. // dmsDeploymentNodeName is the workspace node name the service uses for deployments. @@ -147,13 +157,19 @@ func (s *FakeWorkspace) CreateVersion(req Request, deploymentID string) Response return Response{StatusCode: 400, Body: map[string]string{"message": err.Error()}} } - // previous_version_id is absent from the generated struct, so read it separately. - var concurrency struct { + // previous_version_id and operations are absent from the generated struct, so read them + // separately. + var extra struct { PreviousVersionId string `json:"previous_version_id"` + Operations []struct { + ResourceKey string `json:"resource_key"` + ActionType bundledeployments.OperationActionType `json:"action_type"` + } `json:"operations"` } - if err := json.Unmarshal(req.Body, &concurrency); err != nil { + if err := json.Unmarshal(req.Body, &extra); err != nil { return Response{StatusCode: 400, Body: map[string]string{"message": err.Error()}} } + concurrency := extra defer s.LockUnlock()() @@ -188,6 +204,31 @@ func (s *FakeWorkspace) CreateVersion(req Request, deploymentID string) Response return dmsInvalidArgument("workspace_info.git_folder_path and workspace_info.bundle_root_path must be set together") } + // A version records its whole operation set up front; there is no API to add one later. + if len(extra.Operations) > maxOperationsPerVersion { + return Response{ + StatusCode: 429, + Body: map[string]string{ + "error_code": "RESOURCE_EXHAUSTED", + "message": fmt.Sprintf("a version may stage at most %d operations, got %d", maxOperationsPerVersion, len(extra.Operations)), + }, + } + } + seen := make(map[string]bool, len(extra.Operations)) + for _, staged := range extra.Operations { + switch { + case staged.ResourceKey == "": + return dmsInvalidArgument("operations.resource_key is required") + case !strings.Contains(staged.ResourceKey, "."): + return dmsInvalidArgument("operations.resource_key must have a known resource type prefix (e.g. 'jobs.', 'pipelines.'): " + staged.ResourceKey) + case staged.ActionType == "": + return dmsInvalidArgument("operations.action_type is required and must not be UNSPECIFIED for resource " + staged.ResourceKey) + case seen[staged.ResourceKey]: + return dmsInvalidArgument("operations must have distinct resource_keys; duplicate: " + staged.ResourceKey) + } + seen[staged.ResourceKey] = true + } + d.deployment.LastVersionId = versionID version.Name = "deployments/" + deploymentID + "/versions/" + versionID version.VersionId = versionID @@ -202,6 +243,19 @@ func (s *FakeWorkspace) CreateVersion(req Request, deploymentID string) Response d.deployment.GitInfo = version.GitInfo d.deployment.WorkspaceInfo = version.WorkspaceInfo + // Each staged operation starts pending at sequence 0, and the CLI fills in its outcome + // with UpdateOperation as the resource is applied. + for _, staged := range extra.Operations { + opName := "deployments/" + deploymentID + "/versions/" + versionID + "/operations/" + staged.ResourceKey + d.operations[opName] = &bundledeployments.Operation{ + Name: opName, + ResourceKey: staged.ResourceKey, + ActionType: staged.ActionType, + Status: operationStatusPending, + SequenceId: 0, + } + } + return Response{Body: version} } @@ -234,76 +288,6 @@ func (s *FakeWorkspace) Heartbeat() Response { return Response{Body: bundledeployments.HeartbeatResponse{}} } -func (s *FakeWorkspace) CreateOperation(req Request, deploymentID, versionID string) Response { - resourceKey := req.URL.Query().Get("resource_key") - - var op bundledeployments.Operation - if err := json.Unmarshal(req.Body, &op); err != nil { - return Response{StatusCode: 400, Body: map[string]string{"message": err.Error()}} - } - - defer s.LockUnlock()() - - d, ok := s.dmsDeployments[deploymentID] - if !ok { - return dmsNotFound("deployment " + deploymentID) - } - - // delete requires resource_id. Create-flavored actions may lack an ID. - if op.ActionType == bundledeployments.OperationActionTypeOperationActionTypeDelete && op.ResourceId == "" { - return dmsInvalidArgument("resource_id is required for OPERATION_ACTION_TYPE_DELETE operations") - } - - failed := op.Status == bundledeployments.OperationStatusOperationStatusFailed - if !failed && op.ErrorMessage != "" { - return dmsInvalidArgument("error_message is only allowed when status is OPERATION_STATUS_FAILED") - } - - // An operation with state must identify its resource via resource_id, - // even for failed operations reporting prior state. - if op.State != "" && op.ResourceId == "" { - return dmsInvalidArgument("resource_id is required for an operation that records state") - } - - // One operation per resource per version; duplicates conflict. - opName := "deployments/" + deploymentID + "/versions/" + versionID + "/operations/" + resourceKey - if _, exists := d.operations[opName]; exists { - return Response{ - StatusCode: 409, - Body: map[string]string{"error_code": "RESOURCE_ALREADY_EXISTS", "message": "operation for " + resourceKey + " already exists in this version"}, - } - } - - op.Name = opName - op.ResourceKey = resourceKey - op.SequenceId = 1 - d.operations[opName] = &op - - // sequence_id is a JSON string on the wire but int64 in the SDK struct; - // build the response by hand to match what the CLI parses. - body, err := operationBody(&op) - if err != nil { - return Response{StatusCode: 500, Body: map[string]string{"message": err.Error()}} - } - - // State projects a resource; no state deletes it. Together with the invariant - // that state requires resource_id, a listed resource always has an id. - if op.State == "" { - delete(d.resources, resourceKey) - } else { - d.resources[resourceKey] = bundledeployments.Resource{ - Name: "deployments/" + deploymentID + "/resources/" + resourceKey, - ResourceKey: resourceKey, - ResourceId: op.ResourceId, - ResourceType: op.ResourceType, - LastActionType: op.ActionType, - LastVersionId: versionID, - State: op.State, - } - } - return Response{Body: body} -} - // operationBody renders an operation the way the service does: sequence_id as a // JSON string, which the SDK struct cannot express (it types the field int64). func operationBody(op *bundledeployments.Operation) (map[string]any, error) { diff --git a/libs/testserver/handlers.go b/libs/testserver/handlers.go index 010679ed375..97c586e94a1 100644 --- a/libs/testserver/handlers.go +++ b/libs/testserver/handlers.go @@ -302,9 +302,6 @@ func AddDefaultHandlers(server *Server) { server.Handle("POST", "/api/2.0/bundle/deployments/{deployment_id}/versions/{version_id}/heartbeat", func(req Request) any { return req.Workspace.Heartbeat() }) - server.Handle("POST", "/api/2.0/bundle/deployments/{deployment_id}/versions/{version_id}/operations", func(req Request) any { - return req.Workspace.CreateOperation(req, req.Vars["deployment_id"], req.Vars["version_id"]) - }) server.Handle("PATCH", "/api/2.0/bundle/deployments/{deployment_id}/versions/{version_id}/operations/{resource_key}", func(req Request) any { return req.Workspace.UpdateOperation(req, req.Vars["deployment_id"], req.Vars["version_id"], req.Vars["resource_key"]) }) From a99822be3dd8c3e7d84308a5aa14cf7f22ebaba6 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 18 Aug 2026 16:51:25 +0000 Subject: [PATCH 105/125] bundle: keep deployment-history traffic out of unrelated tests The staged operation's resource key lives in the URL (.../operations/jobs.foo), where it used to be a query parameter, so every recorded-request filter that matches a path substring started matching deployment-history requests too. A test recording one resource type picked them up in the recording run, and the earlier regen wrote them into 41 goldens that have nothing to do with DMS. print_requests.py now excludes them unless --dms asks for them, which the bundle/dms tests do. Four tests filter with a hand-written jq select instead of the helper, so they exclude the path themselves. The 41 goldens are restored rather than regenerated: with the traffic excluded again they must match what they were. Also fixes the fake service, which re-derived the deployment's resource set from the operation on every update. Only an update naming state may move it - naming it with no value clears it and removes the resource, leaving it alone otherwise (see UpdateOperation in service.proto). Every version stages its operations without state, so the old behaviour dropped any resource whose deploy failed before writing one: a dashboard's drift warning went missing because the next plan saw no dashboard at all. That is also why a failure needs no prior state, which dms/failed-update now says. Co-authored-by: Isaac --- acceptance/bin/print_requests.py | 20 +- .../bundle/dms/declined-deploy/output.txt | 4 +- acceptance/bundle/dms/declined-deploy/script | 4 +- acceptance/bundle/dms/depends-on/output.txt | 2 +- acceptance/bundle/dms/depends-on/script | 2 +- .../bundle/dms/emptied-resource/output.txt | 4 +- acceptance/bundle/dms/emptied-resource/script | 4 +- .../bundle/dms/existing-state/output.txt | 8 +- acceptance/bundle/dms/existing-state/script | 8 +- .../bundle/dms/failed-recreate/output.txt | 2 +- acceptance/bundle/dms/failed-recreate/script | 2 +- .../bundle/dms/failed-update/output.txt | 24 +- acceptance/bundle/dms/failed-update/script | 6 +- .../bundle/dms/multiple-resources/output.txt | 4 +- .../bundle/dms/multiple-resources/script | 4 +- acceptance/bundle/dms/no-drift/output.txt | 2 +- acceptance/bundle/dms/no-drift/script | 2 +- acceptance/bundle/dms/no-resources/output.txt | 4 +- acceptance/bundle/dms/no-resources/script | 4 +- .../bundle/dms/partial-update/output.txt | 6 +- acceptance/bundle/dms/partial-update/script | 6 +- acceptance/bundle/dms/provenance/output.txt | 2 +- acceptance/bundle/dms/provenance/script | 2 +- .../bundle/dms/record-failure/output.txt | 2 +- acceptance/bundle/dms/record-failure/script | 2 +- acceptance/bundle/dms/record/output.txt | 6 +- acceptance/bundle/dms/record/script | 6 +- .../dms/redeploy-after-destroy/output.txt | 2 +- .../bundle/dms/redeploy-after-destroy/script | 4 +- .../dms/version-never-created/output.txt | 2 +- .../bundle/dms/version-never-created/script | 2 +- .../job_id_big_graph/delete_all/output.txt | 156 ----- .../job_id_big_graph/destroy/output.txt | 156 ----- .../apps/default_description/output.txt | 13 - .../apps/lifecycle-started-toggle/output.txt | 39 -- .../apps/lifecycle-started-toggle/script | 2 +- .../resources/apps/resource-refs/output.txt | 13 - .../lifecycle-started-toggle/output.txt | 39 -- .../clusters/lifecycle-started/output.txt | 39 -- .../resources/jobs/num_workers/output.txt | 13 - .../jobs/webhook-reorder-remote/output.txt | 13 - .../bundle/resources/permissions/_script | 2 +- .../out.requests.deploy.direct.json | 13 - .../out.requests.destroy.direct.json | 12 - .../clusters/target/out.requests.direct.json | 26 - .../out.requests.deploy.direct.json | 13 - .../out.requests.destroy.direct.json | 12 - .../out.requests.deploy.direct.json | 13 - .../out.requests.destroy.direct.json | 12 - .../out.destroy.requests.direct.json | 24 - .../out.requests.deploy.direct.json | 13 - .../out.requests.destroy.direct.json | 12 - .../out.requests.deploy.direct.json | 13 - .../out.requests.destroy.direct.json | 12 - .../out.destroy.requests.direct.json | 24 - .../out.requests.deploy.direct.json | 13 - .../out.requests.destroy.direct.json | 12 - .../viewers/out.requests.deploy.direct.json | 13 - .../viewers/out.requests.destroy.direct.json | 12 - .../bundle/resources/permissions/output.txt | 642 +++--------------- .../out.requests.deploy.direct.json | 13 - .../out.requests.destroy.direct.json | 12 - .../out.requests.deploy.direct.json | 13 - .../out.requests.destroy.direct.json | 12 - .../out.requests.deploy.direct.json | 13 - .../out.requests.destroy.direct.json | 12 - .../out.requests.deploy.direct.json | 13 - .../out.requests.destroy.direct.json | 12 - .../out.requests.deploy.direct.json | 13 - .../out.requests.destroy.direct.json | 12 - .../out.requests.deploy.direct.json | 13 - .../out.requests.destroy.direct.json | 12 - .../allow-duplicate-names/output.txt | 13 - .../bundle/run_as/job_default/output.txt | 26 - acceptance/bundle/run_as/pipelines/_script | 2 +- libs/testserver/bundle.go | 35 +- 76 files changed, 201 insertions(+), 1553 deletions(-) diff --git a/acceptance/bin/print_requests.py b/acceptance/bin/print_requests.py index 9c8c6a8e7a9..aec82c56bc3 100755 --- a/acceptance/bin/print_requests.py +++ b/acceptance/bin/print_requests.py @@ -132,7 +132,11 @@ def read_json_many(s): assert result == [{"method": "GET"}, {"method": "POST"}], result -def filter_requests(requests, path_filters, include_get, should_sort, unique=False, method_filter=None): +# DMS_PATH is the deployment metadata service's prefix; see filter_requests. +DMS_PATH = "/api/2.0/bundle" + + +def filter_requests(requests, path_filters, include_get, should_sort, unique=False, method_filter=None, include_dms=False): """Filter requests based on method and path filters.""" positive_filters = [] negative_filters = [] @@ -145,6 +149,13 @@ def filter_requests(requests, path_filters, include_get, should_sort, unique=Fal else: sys.exit(f"Unrecognized filter: {f!r}") + # Deployment-history requests carry the resource key in the path + # (.../operations/jobs.foo), so a filter like //jobs matches them too and every test + # that records one resource type would pick these up in the recording run. Excluded + # unless --dms asks for them. + if not include_dms: + negative_filters.append(DMS_PATH) + filtered_requests = [] for req in requests: if method_filter: @@ -203,6 +214,11 @@ def main(): parser.add_argument("path_filters", nargs="*", help="Path substring filters") parser.add_argument("-v", "--verbose", action="store_true", help="Enable diagnostic messages") parser.add_argument("--get", action="store_true", help="Include GET requests (excluded by default)") + parser.add_argument( + "--dms", + action="store_true", + help="Include deployment-history requests (excluded by default; see filter_requests)", + ) parser.add_argument("--keep", action="store_true", help="Keep out.requests.json file after processing") parser.add_argument("--sort", action="store_true", help="Sort requests before output") parser.add_argument( @@ -261,7 +277,7 @@ def main(): return requests = read_json_many(data) - filtered_requests = filter_requests(requests, args.path_filters, args.get, args.sort, args.unique, args.method) + filtered_requests = filter_requests(requests, args.path_filters, args.get, args.sort, args.unique, args.method, args.dms) for req in filtered_requests: body = req.get("body") diff --git a/acceptance/bundle/dms/declined-deploy/output.txt b/acceptance/bundle/dms/declined-deploy/output.txt index bc5624971a0..554d4541650 100644 --- a/acceptance/bundle/dms/declined-deploy/output.txt +++ b/acceptance/bundle/dms/declined-deploy/output.txt @@ -6,7 +6,7 @@ Created schemas.foo Files: 4 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged ->>> print_requests.py //api/2.0/bundle --sort +>>> print_requests.py --dms //api/2.0/bundle --sort { "method": "PATCH", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/schemas.foo", @@ -74,7 +74,7 @@ To proceed, use --auto-approve after reviewing the plan above. Files: 3 uploaded, 0 deleted === Nothing was recorded for the declined deploy - no version, so none to abort ->>> print_requests.py //api/2.0/bundle --sort +>>> print_requests.py --dms //api/2.0/bundle --sort >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: diff --git a/acceptance/bundle/dms/declined-deploy/script b/acceptance/bundle/dms/declined-deploy/script index 74be4c9eec6..8609f7e1203 100644 --- a/acceptance/bundle/dms/declined-deploy/script +++ b/acceptance/bundle/dms/declined-deploy/script @@ -1,6 +1,6 @@ title "Deploy a schema, so the deployment and its first version exist" trace $CLI bundle deploy -trace print_requests.py //api/2.0/bundle --sort +trace print_requests.py --dms //api/2.0/bundle --sort title "A destructive change without --auto-approve is declined: this console cannot prompt" # Changing the catalog recreates the schema, which needs approval. @@ -10,7 +10,7 @@ trace musterr $CLI bundle deploy title "Nothing was recorded for the declined deploy - no version, so none to abort" # The version number it would have used is left for the next deploy to take, so the # history has no entry that reads like a deploy which failed or did nothing. -trace print_requests.py //api/2.0/bundle --sort +trace print_requests.py --dms //api/2.0/bundle --sort trace $CLI bundle destroy --auto-approve rm -f out.requests.txt diff --git a/acceptance/bundle/dms/depends-on/output.txt b/acceptance/bundle/dms/depends-on/output.txt index b9c14388bf5..ccf622262b9 100644 --- a/acceptance/bundle/dms/depends-on/output.txt +++ b/acceptance/bundle/dms/depends-on/output.txt @@ -7,7 +7,7 @@ Created jobs.parent Files: 4 uploaded, 0 deleted Resources: 2 created, 0 changed, 0 deleted, 0 unchanged ->>> print_requests.py //versions/1/operations --sort +>>> print_requests.py --dms //versions/1/operations --sort { "method": "PATCH", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.child", diff --git a/acceptance/bundle/dms/depends-on/script b/acceptance/bundle/dms/depends-on/script index a7bacaf5d67..f2bcf0cc411 100644 --- a/acceptance/bundle/dms/depends-on/script +++ b/acceptance/bundle/dms/depends-on/script @@ -1,6 +1,6 @@ title "Deploy a job that references another: depends_on is recorded alongside the config, since the reference is resolved to a literal in the config itself" trace $CLI bundle deploy -trace print_requests.py //versions/1/operations --sort +trace print_requests.py --dms //versions/1/operations --sort trace $CLI bundle destroy --auto-approve rm -f out.requests.txt diff --git a/acceptance/bundle/dms/emptied-resource/output.txt b/acceptance/bundle/dms/emptied-resource/output.txt index 47de8091922..eb8e5285d48 100644 --- a/acceptance/bundle/dms/emptied-resource/output.txt +++ b/acceptance/bundle/dms/emptied-resource/output.txt @@ -7,7 +7,7 @@ Created schemas.foo.grants Files: 4 uploaded, 0 deleted Resources: 2 created, 0 changed, 0 deleted, 0 unchanged ->>> print_requests.py //api/2.0/bundle --sort +>>> print_requests.py --dms //api/2.0/bundle --sort { "method": "PATCH", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/schemas.foo", @@ -86,7 +86,7 @@ Updated schemas.foo.grants Files: 3 uploaded, 0 deleted Resources: 0 created, 1 changed, 0 deleted, 1 unchanged ->>> print_requests.py //api/2.0/bundle --sort +>>> print_requests.py --dms //api/2.0/bundle --sort { "method": "PATCH", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations/schemas.foo.grants", diff --git a/acceptance/bundle/dms/emptied-resource/script b/acceptance/bundle/dms/emptied-resource/script index 954cfc4a496..3c7d9512ee0 100644 --- a/acceptance/bundle/dms/emptied-resource/script +++ b/acceptance/bundle/dms/emptied-resource/script @@ -1,6 +1,6 @@ title "Deploy a schema with one grant: the grants node is recorded with its state" trace $CLI bundle deploy -trace print_requests.py //api/2.0/bundle --sort +trace print_requests.py --dms //api/2.0/bundle --sort title "Revoke the grant, so the grants node empties out" trace update_file.py databricks.yml 'grants: [{principal: someone@example.com, privileges: [USE_SCHEMA]}]' 'grants: []' @@ -8,7 +8,7 @@ trace $CLI bundle deploy # The emptied node records as a delete, not an update. The service only drops # a resource on delete; otherwise it stays listed with no state. -trace print_requests.py //api/2.0/bundle --sort +trace print_requests.py --dms //api/2.0/bundle --sort title "Plan again: reading state back from the service works and reports no work" trace $CLI bundle plan | contains.py "Plan: 0 to add, 0 to change, 0 to delete" "!unexpected end of JSON input" diff --git a/acceptance/bundle/dms/existing-state/output.txt b/acceptance/bundle/dms/existing-state/output.txt index bcc47cfeb77..cc0edf233e7 100644 --- a/acceptance/bundle/dms/existing-state/output.txt +++ b/acceptance/bundle/dms/existing-state/output.txt @@ -6,7 +6,7 @@ Created jobs.one Files: 4 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged ->>> print_requests.py //api/2.0/bundle --oneline +>>> print_requests.py --dms //api/2.0/bundle --oneline === Turning recording on afterwards is an error: those resources were never recorded, so treating DMS as authoritative would deploy them a second time >>> update_file.py databricks.yml record_deployment_history: false record_deployment_history: true @@ -23,7 +23,7 @@ To keep the existing resources instead, unset experimental.record_deployment_his === No deployment was created in DMS ->>> print_requests.py //api/2.0/bundle --oneline +>>> print_requests.py --dms //api/2.0/bundle --oneline === Still an error after wiping the local cache: deploy pulls the state file back from the workspace, so the resources are still tracked >>> musterr [CLI] bundle deploy @@ -37,7 +37,7 @@ To record this bundle's history, start it over as a new deployment: To keep the existing resources instead, unset experimental.record_deployment_history ->>> print_requests.py //api/2.0/bundle --oneline +>>> print_requests.py --dms //api/2.0/bundle --oneline === Destroy clears the tracked resources, so recording can be enabled afterwards >>> update_file.py databricks.yml record_deployment_history: true record_deployment_history: false @@ -58,7 +58,7 @@ Created jobs.one Files: 4 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged ->>> print_requests.py //api/2.0/bundle --oneline +>>> print_requests.py --dms //api/2.0/bundle --oneline {"method": "POST", "path": "/api/2.0/bundle/deployments", "body": {"initial_parent_path": "/Workspace/Users/[USERNAME]/.bundle/dms-existing-state/default/state", "target_name": "default"}} {"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": {"version_id": "1"}, "body": {"cli_version": "[CLI_VERSION]", "version_type": "VERSION_TYPE_DEPLOY", "target_name": "default", "display_name": "dms-existing-state", "workspace_info": {"file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-existing-state/default/files", "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-existing-state/default"}, "operations": [{"resource_key": "jobs.one", "action_type": "OPERATION_ACTION_TYPE_CREATE"}]}} {"method": "PATCH", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.one", "q": {"update_mask": "state,error_message,resource_id,status"}, "body": {"state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-existing-state/default/state/metadata.json\",\"version_id\":\"1\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"one\",\"queue\":{\"enabled\":true}}}", "resource_id": "[NUMID]", "status": "OPERATION_STATUS_SUCCEEDED", "sequence_id": "0"}} diff --git a/acceptance/bundle/dms/existing-state/script b/acceptance/bundle/dms/existing-state/script index 9e448161259..99011b7cb54 100644 --- a/acceptance/bundle/dms/existing-state/script +++ b/acceptance/bundle/dms/existing-state/script @@ -1,22 +1,22 @@ title "Deploy without recording: the bundle gets ordinary direct-engine state, unknown to DMS" trace $CLI bundle deploy -trace print_requests.py //api/2.0/bundle --oneline +trace print_requests.py --dms //api/2.0/bundle --oneline title "Turning recording on afterwards is an error: those resources were never recorded, so treating DMS as authoritative would deploy them a second time" trace update_file.py databricks.yml "record_deployment_history: false" "record_deployment_history: true" trace musterr $CLI bundle deploy title "No deployment was created in DMS" -trace print_requests.py //api/2.0/bundle --oneline +trace print_requests.py --dms //api/2.0/bundle --oneline title "Still an error after wiping the local cache: deploy pulls the state file back from the workspace, so the resources are still tracked" rm -rf .databricks trace musterr $CLI bundle deploy -trace print_requests.py //api/2.0/bundle --oneline +trace print_requests.py --dms //api/2.0/bundle --oneline title "Destroy clears the tracked resources, so recording can be enabled afterwards" trace update_file.py databricks.yml "record_deployment_history: true" "record_deployment_history: false" trace $CLI bundle destroy --auto-approve trace update_file.py databricks.yml "record_deployment_history: false" "record_deployment_history: true" trace $CLI bundle deploy -trace print_requests.py //api/2.0/bundle --oneline +trace print_requests.py --dms //api/2.0/bundle --oneline diff --git a/acceptance/bundle/dms/failed-recreate/output.txt b/acceptance/bundle/dms/failed-recreate/output.txt index 5ed0ea82383..408c04a2330 100644 --- a/acceptance/bundle/dms/failed-recreate/output.txt +++ b/acceptance/bundle/dms/failed-recreate/output.txt @@ -25,7 +25,7 @@ API message: Fault injected by test. Files: 3 uploaded, 0 deleted ->>> print_requests.py //api/2.0/bundle +>>> print_requests.py --dms //api/2.0/bundle { "method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", diff --git a/acceptance/bundle/dms/failed-recreate/script b/acceptance/bundle/dms/failed-recreate/script index 83bbcbe21d6..f88e0c1c77b 100644 --- a/acceptance/bundle/dms/failed-recreate/script +++ b/acceptance/bundle/dms/failed-recreate/script @@ -8,7 +8,7 @@ trace update_file.py databricks.yml "catalog_name: main" "catalog_name: other" trace fault.py "POST /api/2.1/unity-catalog/schemas" 400 0 1 INVALID_PARAMETER_VALUE trace musterr $CLI bundle deploy --auto-approve # Not sorted: the point of this test is the order the operations are recorded in. -trace print_requests.py //api/2.0/bundle +trace print_requests.py --dms //api/2.0/bundle title "The resource is not listed: state is what projects a resource, and the failed recreate left none" # The deployment ID is the workspace node's ID; read it back the way the CLI does. diff --git a/acceptance/bundle/dms/failed-update/output.txt b/acceptance/bundle/dms/failed-update/output.txt index 7d64531a8ca..bb3a928e726 100644 --- a/acceptance/bundle/dms/failed-update/output.txt +++ b/acceptance/bundle/dms/failed-update/output.txt @@ -6,7 +6,7 @@ Created schemas.foo Files: 4 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged -=== An update that fails before writing any state records the state from before the deploy. Nothing touched the schema, so it still exists, and an operation without state would drop it from the deployment +=== An update that fails before writing any state only marks its operation failed. It names no state, so the deployment keeps describing the schema the last successful version recorded - which is right, because nothing touched it >>> update_file.py databricks.yml comment: v1 comment: v2 >>> fault.py PATCH /api/2.1/unity-catalog/schemas/* 400 0 1 INVALID_PARAMETER_VALUE @@ -22,7 +22,7 @@ API message: Fault injected by test. Files: 3 uploaded, 0 deleted ->>> print_requests.py //api/2.0/bundle +>>> print_requests.py --dms //api/2.0/bundle { "method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", @@ -67,12 +67,24 @@ Files: 3 uploaded, 0 deleted } } -=== The schema is still listed, described as it was before the failed deploy +=== The schema is still listed as the previous version left it, so the next plan updates it rather than creating a second one >>> MSYS_NO_PATHCONV=1 [CLI] api get /api/2.0/bundle/deployments/[NUMID]/resources -{} +{ + "resources": [ + { + "last_action_type": "OPERATION_ACTION_TYPE_CREATE", + "last_version_id": "1", + "name": "deployments/[NUMID]/resources/schemas.foo", + "resource_id": "main.dms_failed_update_schema", + "resource_key": "schemas.foo", + "resource_type": "", + "state": "{\"state\":{\"catalog_name\":\"main\",\"comment\":\"v1\",\"name\":\"dms_failed_update_schema\"}}" + } + ] +} === Planning from DMS state alone updates the schema rather than creating a second one, which is what recording the prior state buys >>> [CLI] bundle plan -create schemas.foo +update schemas.foo -Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged +Plan: 0 to add, 1 to change, 0 to delete, 0 unchanged diff --git a/acceptance/bundle/dms/failed-update/script b/acceptance/bundle/dms/failed-update/script index eecdd9cca59..efcdf340188 100644 --- a/acceptance/bundle/dms/failed-update/script +++ b/acceptance/bundle/dms/failed-update/script @@ -2,15 +2,15 @@ title "Deploy the schema, so there is an existing resource to update" trace $CLI bundle deploy rm -f out.requests.txt -title "An update that fails before writing any state records the state from before the deploy. Nothing touched the schema, so it still exists, and an operation without state would drop it from the deployment" +title "An update that fails before writing any state only marks its operation failed. It names no state, so the deployment keeps describing the schema the last successful version recorded - which is right, because nothing touched it" trace update_file.py databricks.yml "comment: v1" "comment: v2" # Fail the update call itself, so the deploy never writes state for the schema. trace fault.py "PATCH /api/2.1/unity-catalog/schemas/*" 400 0 1 INVALID_PARAMETER_VALUE trace musterr $CLI bundle deploy --auto-approve # Not sorted: the point of this test is the order the operations are recorded in. -trace print_requests.py //api/2.0/bundle +trace print_requests.py --dms //api/2.0/bundle -title "The schema is still listed, described as it was before the failed deploy" +title "The schema is still listed as the previous version left it, so the next plan updates it rather than creating a second one" # The deployment ID is the workspace node's ID; read it back the way the CLI does. # Extracted with python, not jq: the ID exceeds 2^53 and jq 1.6 rounds it. deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-failed-update/default/state/resources.deployment.json" -o json | python3 -c 'import sys,json; print(json.load(sys.stdin)["object_id"])') diff --git a/acceptance/bundle/dms/multiple-resources/output.txt b/acceptance/bundle/dms/multiple-resources/output.txt index b8953530add..ae8dc28844f 100644 --- a/acceptance/bundle/dms/multiple-resources/output.txt +++ b/acceptance/bundle/dms/multiple-resources/output.txt @@ -10,7 +10,7 @@ Created jobs.two Files: 4 uploaded, 0 deleted Resources: 5 created, 0 changed, 0 deleted, 0 unchanged ->>> print_requests.py //versions/1/operations --sort --del-body state --oneline +>>> print_requests.py --dms //versions/1/operations --sort --del-body state --oneline {"method": "PATCH", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.five", "q": {"update_mask": "state,error_message,resource_id,status"}, "body": {"resource_id": "[NUMID]", "status": "OPERATION_STATUS_SUCCEEDED", "sequence_id": "0"}} {"method": "PATCH", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.four", "q": {"update_mask": "state,error_message,resource_id,status"}, "body": {"resource_id": "[NUMID]", "status": "OPERATION_STATUS_SUCCEEDED", "sequence_id": "0"}} {"method": "PATCH", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.one", "q": {"update_mask": "state,error_message,resource_id,status"}, "body": {"resource_id": "[NUMID]", "status": "OPERATION_STATUS_SUCCEEDED", "sequence_id": "0"}} @@ -23,6 +23,6 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-multiple-resou Files: 2 uploaded, 0 deleted Resources: 0 created, 0 changed, 0 deleted, 5 unchanged ->>> print_requests.py //api/2.0/bundle --sort --oneline +>>> print_requests.py --dms //api/2.0/bundle --sort --oneline {"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": {"version_id": "2"}, "body": {"cli_version": "[CLI_VERSION]", "version_type": "VERSION_TYPE_DEPLOY", "target_name": "default", "display_name": "dms-multiple-resources", "previous_version_id": "1", "workspace_info": {"file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources/default/files", "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources/default"}}} {"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/complete", "body": {"completion_reason": "VERSION_COMPLETE_SUCCESS"}} diff --git a/acceptance/bundle/dms/multiple-resources/script b/acceptance/bundle/dms/multiple-resources/script index e9b9339607e..26b0744efa8 100644 --- a/acceptance/bundle/dms/multiple-resources/script +++ b/acceptance/bundle/dms/multiple-resources/script @@ -1,7 +1,7 @@ title "Deploy several resources: operations are uploaded from background workers, so exactly one is recorded per resource no matter what order the uploads finish in. The serialized state is dropped from the output here; bundle/dms/record covers it." trace $CLI bundle deploy -trace print_requests.py //versions/1/operations --sort --del-body state --oneline +trace print_requests.py --dms //versions/1/operations --sort --del-body state --oneline title "Redeploy with no changes: nothing is applied, so no operations are recorded and only the version is opened and completed" trace $CLI bundle deploy -trace print_requests.py //api/2.0/bundle --sort --oneline +trace print_requests.py --dms //api/2.0/bundle --sort --oneline diff --git a/acceptance/bundle/dms/no-drift/output.txt b/acceptance/bundle/dms/no-drift/output.txt index a058cb3e479..bad708f5cfc 100644 --- a/acceptance/bundle/dms/no-drift/output.txt +++ b/acceptance/bundle/dms/no-drift/output.txt @@ -16,7 +16,7 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-no-drift/defau Files: 2 uploaded, 0 deleted Resources: 0 created, 0 changed, 0 deleted, 2 unchanged ->>> print_requests.py //api/2.2/jobs //api/2.0/pipelines --sort +>>> print_requests.py --dms //api/2.2/jobs //api/2.0/pipelines --sort { "method": "POST", "path": "/api/2.0/pipelines", diff --git a/acceptance/bundle/dms/no-drift/script b/acceptance/bundle/dms/no-drift/script index f947997ab92..142ab864b2b 100644 --- a/acceptance/bundle/dms/no-drift/script +++ b/acceptance/bundle/dms/no-drift/script @@ -7,4 +7,4 @@ trace $CLI bundle plan | contains.py "Plan: 0 to add, 0 to change, 0 to delete, title "A second deploy is a no-op too: no update request for either resource" trace $CLI bundle deploy -trace print_requests.py //api/2.2/jobs //api/2.0/pipelines --sort +trace print_requests.py --dms //api/2.2/jobs //api/2.0/pipelines --sort diff --git a/acceptance/bundle/dms/no-resources/output.txt b/acceptance/bundle/dms/no-resources/output.txt index d6cb0ef5bfe..b020a726bbc 100644 --- a/acceptance/bundle/dms/no-resources/output.txt +++ b/acceptance/bundle/dms/no-resources/output.txt @@ -5,7 +5,7 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-no-resources/d Files: 4 uploaded, 0 deleted Resources: 0 created, 0 changed, 0 deleted, 0 unchanged ->>> print_requests.py //api/2.0/bundle --get +>>> print_requests.py --dms //api/2.0/bundle --get { "method": "POST", "path": "/api/2.0/bundle/deployments", @@ -51,7 +51,7 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-no-resources/d Files: 2 uploaded, 0 deleted Resources: 0 created, 0 changed, 0 deleted, 0 unchanged ->>> print_requests.py //api/2.0/bundle --get +>>> print_requests.py --dms //api/2.0/bundle --get { "method": "GET", "path": "/api/2.0/bundle/deployments/[NUMID]/resources" diff --git a/acceptance/bundle/dms/no-resources/script b/acceptance/bundle/dms/no-resources/script index f1b9ad5fa60..b481a1daabb 100644 --- a/acceptance/bundle/dms/no-resources/script +++ b/acceptance/bundle/dms/no-resources/script @@ -1,8 +1,8 @@ title "First deploy of a bundle with no resources: the deployment is created, and its workspace node identifies it even though no resource state was written" trace $CLI bundle deploy -trace print_requests.py //api/2.0/bundle --get +trace print_requests.py --dms //api/2.0/bundle --get trace MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-no-resources/default/state/resources.deployment.json" | jq '{object_type,path}' title "Redeploy: the deployment is resolved from that node, so no second deployment is created" trace $CLI bundle deploy -trace print_requests.py //api/2.0/bundle --get +trace print_requests.py --dms //api/2.0/bundle --get diff --git a/acceptance/bundle/dms/partial-update/output.txt b/acceptance/bundle/dms/partial-update/output.txt index cfd1441f735..7ac3648d4f1 100644 --- a/acceptance/bundle/dms/partial-update/output.txt +++ b/acceptance/bundle/dms/partial-update/output.txt @@ -6,7 +6,7 @@ Created schemas.foo Files: 4 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged ->>> print_requests.py //api/2.0/bundle +>>> print_requests.py --dms //api/2.0/bundle { "method": "POST", "path": "/api/2.0/bundle/deployments", @@ -71,7 +71,7 @@ Recreated schemas.foo Files: 3 uploaded, 0 deleted Resources: 1 created, 0 changed, 1 deleted, 0 unchanged ->>> print_requests.py //api/2.0/bundle +>>> print_requests.py --dms //api/2.0/bundle { "method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", @@ -141,7 +141,7 @@ All files and directories at the following location will be deleted: /Workspace/ Destroy: 1 deleted ->>> print_requests.py //api/2.0/bundle +>>> print_requests.py --dms //api/2.0/bundle { "method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", diff --git a/acceptance/bundle/dms/partial-update/script b/acceptance/bundle/dms/partial-update/script index a9e69d2369a..2cfe527108b 100644 --- a/acceptance/bundle/dms/partial-update/script +++ b/acceptance/bundle/dms/partial-update/script @@ -1,14 +1,14 @@ title "Deploy: the state write records the resource" trace $CLI bundle deploy -trace print_requests.py //api/2.0/bundle +trace print_requests.py --dms //api/2.0/bundle title "Recreate writes state twice - the entry is dropped, then the new resource is saved" # One operation per resource per version; both writes land on the same one. # The drop opens it IN_PROGRESS, the save patches it to SUCCEEDED. trace update_file.py databricks.yml "catalog_name: main" "catalog_name: other" trace $CLI bundle deploy --auto-approve -trace print_requests.py //api/2.0/bundle +trace print_requests.py --dms //api/2.0/bundle title "Destroy: the delete is recorded with the id and no state" trace $CLI bundle destroy --auto-approve -trace print_requests.py //api/2.0/bundle +trace print_requests.py --dms //api/2.0/bundle diff --git a/acceptance/bundle/dms/provenance/output.txt b/acceptance/bundle/dms/provenance/output.txt index 5067a98fa42..38ae16991e7 100644 --- a/acceptance/bundle/dms/provenance/output.txt +++ b/acceptance/bundle/dms/provenance/output.txt @@ -6,7 +6,7 @@ Created jobs.foo Files: 5 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged ->>> print_requests.py //versions --sort +>>> print_requests.py --dms //versions --sort { "method": "PATCH", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.foo", diff --git a/acceptance/bundle/dms/provenance/script b/acceptance/bundle/dms/provenance/script index 51449ae50b9..46eecbf26de 100644 --- a/acceptance/bundle/dms/provenance/script +++ b/acceptance/bundle/dms/provenance/script @@ -4,7 +4,7 @@ git remote add origin https://github.com/databricks/bundle-examples.git trace $CLI bundle deploy # The commit SHA changes every run, so assert it is a 40-char hex string and drop it. add_repl.py "$(git rev-parse HEAD)" COMMIT -trace print_requests.py //versions --sort +trace print_requests.py --dms //versions --sort title "The service denormalizes them onto the deployment, so a reader gets the provenance of the latest version" deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-provenance/dev/state/resources.deployment.json" -o json | python3 -c 'import sys,json; print(json.load(sys.stdin)["object_id"])') diff --git a/acceptance/bundle/dms/record-failure/output.txt b/acceptance/bundle/dms/record-failure/output.txt index d5e1ae25d81..62ff21c81e8 100644 --- a/acceptance/bundle/dms/record-failure/output.txt +++ b/acceptance/bundle/dms/record-failure/output.txt @@ -11,7 +11,7 @@ API message: cluster spec is invalid Files: 5 uploaded, 0 deleted ->>> print_requests.py //api/2.0/bundle --sort +>>> print_requests.py --dms //api/2.0/bundle --sort { "method": "PATCH", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.doomed", diff --git a/acceptance/bundle/dms/record-failure/script b/acceptance/bundle/dms/record-failure/script index 5e0212155d9..b8d387834b0 100644 --- a/acceptance/bundle/dms/record-failure/script +++ b/acceptance/bundle/dms/record-failure/script @@ -1,6 +1,6 @@ title "A resource that fails to apply is recorded as a failed operation carrying the error, so the deployment history says why rather than omitting the resource" trace musterr $CLI bundle deploy -trace print_requests.py //api/2.0/bundle --sort +trace print_requests.py --dms //api/2.0/bundle --sort title "The failed resource is not listed at all: state is what projects a resource, and a failed create records none, so a later deploy plans to create it rather than treating it as already deployed" # The deployment ID is the workspace node's ID; read it back the way the CLI does. diff --git a/acceptance/bundle/dms/record/output.txt b/acceptance/bundle/dms/record/output.txt index cd5c71096f8..7ffe2b2913e 100644 --- a/acceptance/bundle/dms/record/output.txt +++ b/acceptance/bundle/dms/record/output.txt @@ -6,7 +6,7 @@ Created jobs.foo Files: 4 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged ->>> print_requests.py //api/2.0/bundle +>>> print_requests.py --dms //api/2.0/bundle { "method": "POST", "path": "/api/2.0/bundle/deployments", @@ -75,7 +75,7 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-record/default Files: 4 uploaded, 0 deleted Resources: 0 created, 0 changed, 0 deleted, 1 unchanged ->>> print_requests.py //api/2.0/bundle +>>> print_requests.py --dms //api/2.0/bundle { "method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", @@ -111,7 +111,7 @@ All files and directories at the following location will be deleted: /Workspace/ Destroy: 1 deleted ->>> print_requests.py //api/2.0/bundle +>>> print_requests.py --dms //api/2.0/bundle { "method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", diff --git a/acceptance/bundle/dms/record/script b/acceptance/bundle/dms/record/script index 0829655016a..448454e97eb 100644 --- a/acceptance/bundle/dms/record/script +++ b/acceptance/bundle/dms/record/script @@ -1,6 +1,6 @@ title "Deploy: the server assigns the deployment ID, and a version + create operation are recorded" trace $CLI bundle deploy -trace print_requests.py //api/2.0/bundle +trace print_requests.py --dms //api/2.0/bundle title "The deployment ID is the ID of the workspace node the service registered under initial_parent_path; the CLI stores nothing locally" # MSYS_NO_PATHCONV prevents Git Bash from rewriting the leading-/ path on Windows. @@ -11,8 +11,8 @@ trace jq 'has("deployment_id")' .databricks/bundle/default/resources.json title "Redeploy after deleting the local cache: the deployment ID is resolved from that node, the same deployment is reused, and the version increments (no new CreateDeployment)" rm -rf .databricks trace $CLI bundle deploy -trace print_requests.py //api/2.0/bundle +trace print_requests.py --dms //api/2.0/bundle title "Destroy: a destroy version and delete operation are recorded, then the deployment is deleted" trace $CLI bundle destroy --auto-approve -trace print_requests.py //api/2.0/bundle +trace print_requests.py --dms //api/2.0/bundle diff --git a/acceptance/bundle/dms/redeploy-after-destroy/output.txt b/acceptance/bundle/dms/redeploy-after-destroy/output.txt index c02f9a53f1c..94dc3c83374 100644 --- a/acceptance/bundle/dms/redeploy-after-destroy/output.txt +++ b/acceptance/bundle/dms/redeploy-after-destroy/output.txt @@ -30,7 +30,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "path": "/Workspace/Users/[USERNAME]/.bundle/dms-redeploy-after-destroy/default/state/resources.deployment.json" } ->>> print_requests.py //api/2.0/bundle --get +>>> print_requests.py --dms //api/2.0/bundle --get { "method": "POST", "path": "/api/2.0/bundle/deployments", diff --git a/acceptance/bundle/dms/redeploy-after-destroy/script b/acceptance/bundle/dms/redeploy-after-destroy/script index 8dabf189f49..edd58e11507 100644 --- a/acceptance/bundle/dms/redeploy-after-destroy/script +++ b/acceptance/bundle/dms/redeploy-after-destroy/script @@ -1,11 +1,11 @@ title "Deploy, then destroy: the deployment record and its workspace node are both deleted, so nothing points at the destroyed deployment" trace $CLI bundle deploy trace $CLI bundle destroy --auto-approve -print_requests.py //api/2.0/bundle --get > /dev/null +print_requests.py --dms //api/2.0/bundle --get > /dev/null trace MSYS_NO_PATHCONV=1 musterr $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-redeploy-after-destroy/default/state/resources.deployment.json" title "Deploy again: with no node to resolve, a fresh deployment is created at version 1, with its own workspace node" trace $CLI bundle deploy trace MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-redeploy-after-destroy/default/state/resources.deployment.json" | jq '{object_type,path}' -trace print_requests.py //api/2.0/bundle --get +trace print_requests.py --dms //api/2.0/bundle --get diff --git a/acceptance/bundle/dms/version-never-created/output.txt b/acceptance/bundle/dms/version-never-created/output.txt index d167d3fd395..0ffff5cf6fb 100644 --- a/acceptance/bundle/dms/version-never-created/output.txt +++ b/acceptance/bundle/dms/version-never-created/output.txt @@ -28,7 +28,7 @@ API message: Internal error Files: 2 uploaded, 0 deleted ->>> print_requests.py //api/2.0/bundle --get --oneline +>>> print_requests.py --dms //api/2.0/bundle --get --oneline {"method": "POST", "path": "/api/2.0/bundle/deployments", "body": {"initial_parent_path": "/Workspace/Users/[USERNAME]/.bundle/dms-version-never-created/default/state", "target_name": "default"}} {"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": {"version_id": "1"}, "body": {"cli_version": "[CLI_VERSION]", "version_type": "VERSION_TYPE_DEPLOY", "target_name": "default", "display_name": "dms-version-never-created", "workspace_info": {"file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-version-never-created/default/files", "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-version-never-created/default"}, "operations": [{"resource_key": "jobs.foo", "action_type": "OPERATION_ACTION_TYPE_CREATE"}]}} {"method": "GET", "path": "/api/2.0/bundle/deployments/[NUMID]/resources"} diff --git a/acceptance/bundle/dms/version-never-created/script b/acceptance/bundle/dms/version-never-created/script index 39566343e98..462e18de914 100644 --- a/acceptance/bundle/dms/version-never-created/script +++ b/acceptance/bundle/dms/version-never-created/script @@ -4,4 +4,4 @@ trace MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_U title "The next deploy reuses the ID that node names and retries version 1, rather than creating a second deployment" trace musterr $CLI bundle deploy -trace print_requests.py //api/2.0/bundle --get --oneline +trace print_requests.py --dms //api/2.0/bundle --get --oneline diff --git a/acceptance/bundle/resource_deps/job_id_big_graph/delete_all/output.txt b/acceptance/bundle/resource_deps/job_id_big_graph/delete_all/output.txt index f180d8e2414..7d97a13e269 100644 --- a/acceptance/bundle/resource_deps/job_id_big_graph/delete_all/output.txt +++ b/acceptance/bundle/resource_deps/job_id_big_graph/delete_all/output.txt @@ -17,162 +17,6 @@ Files: 0 uploaded, 0 deleted Resources: 12 created, 0 changed, 0 deleted, 0 unchanged >>> print_requests.py --nostamp --sort //jobs -{ - "method": "PATCH", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.chain_bottom", - "q": { - "update_mask": "state,error_message,resource_id,status" - }, - "body": { - "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json\",\"version_id\":\"1\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"job chain_bottom\",\"queue\":{\"enabled\":true}}}", - "resource_id": "[CHAIN_BOTTOM_ID]", - "status": "OPERATION_STATUS_SUCCEEDED", - "sequence_id": "0" - } -} -{ - "method": "PATCH", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.chain_mid", - "q": { - "update_mask": "state,error_message,resource_id,status" - }, - "body": { - "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json\",\"version_id\":\"1\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"job chain_mid\",\"queue\":{\"enabled\":true},\"tasks\":[{\"run_job_task\":{\"job_id\":[CHAIN_BOTTOM_ID]},\"task_key\":\"t1\"}]},\"depends_on\":[{\"node\":\"resources.jobs.chain_bottom\",\"label\":\"${resources.jobs.chain_bottom.id}\"}]}", - "resource_id": "[CHAIN_MID_ID]", - "status": "OPERATION_STATUS_SUCCEEDED", - "sequence_id": "0" - } -} -{ - "method": "PATCH", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.chain_top", - "q": { - "update_mask": "state,error_message,resource_id,status" - }, - "body": { - "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json\",\"version_id\":\"1\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"job chain_top\",\"queue\":{\"enabled\":true},\"tasks\":[{\"run_job_task\":{\"job_id\":[CHAIN_MID_ID]},\"task_key\":\"t1\"}]},\"depends_on\":[{\"node\":\"resources.jobs.chain_mid\",\"label\":\"${resources.jobs.chain_mid.id}\"}]}", - "resource_id": "[CHAIN_TOP_ID]", - "status": "OPERATION_STATUS_SUCCEEDED", - "sequence_id": "0" - } -} -{ - "method": "PATCH", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.diamond_bottom", - "q": { - "update_mask": "state,error_message,resource_id,status" - }, - "body": { - "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json\",\"version_id\":\"1\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"job diamond_bottom\",\"queue\":{\"enabled\":true}}}", - "resource_id": "[DIAMOND_BOTTOM_ID]", - "status": "OPERATION_STATUS_SUCCEEDED", - "sequence_id": "0" - } -} -{ - "method": "PATCH", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.diamond_left", - "q": { - "update_mask": "state,error_message,resource_id,status" - }, - "body": { - "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json\",\"version_id\":\"1\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"job diamond_left\",\"queue\":{\"enabled\":true},\"tasks\":[{\"run_job_task\":{\"job_id\":[DIAMOND_BOTTOM_ID]},\"task_key\":\"t1\"}]},\"depends_on\":[{\"node\":\"resources.jobs.diamond_bottom\",\"label\":\"${resources.jobs.diamond_bottom.id}\"}]}", - "resource_id": "[DIAMOND_LEFT_ID]", - "status": "OPERATION_STATUS_SUCCEEDED", - "sequence_id": "0" - } -} -{ - "method": "PATCH", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.diamond_right", - "q": { - "update_mask": "state,error_message,resource_id,status" - }, - "body": { - "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json\",\"version_id\":\"1\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"job diamond_right\",\"queue\":{\"enabled\":true},\"tasks\":[{\"run_job_task\":{\"job_id\":[DIAMOND_BOTTOM_ID]},\"task_key\":\"t1\"}]},\"depends_on\":[{\"node\":\"resources.jobs.diamond_bottom\",\"label\":\"${resources.jobs.diamond_bottom.id}\"}]}", - "resource_id": "[DIAMOND_RIGHT_ID]", - "status": "OPERATION_STATUS_SUCCEEDED", - "sequence_id": "0" - } -} -{ - "method": "PATCH", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.diamond_top", - "q": { - "update_mask": "state,error_message,resource_id,status" - }, - "body": { - "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json\",\"version_id\":\"1\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"job diamond_top\",\"queue\":{\"enabled\":true},\"tasks\":[{\"run_job_task\":{\"job_id\":[DIAMOND_LEFT_ID]},\"task_key\":\"t1\"},{\"run_job_task\":{\"job_id\":[DIAMOND_RIGHT_ID]},\"task_key\":\"t2\"}]},\"depends_on\":[{\"node\":\"resources.jobs.diamond_left\",\"label\":\"${resources.jobs.diamond_left.id}\"},{\"node\":\"resources.jobs.diamond_right\",\"label\":\"${resources.jobs.diamond_right.id}\"}]}", - "resource_id": "[DIAMOND_TOP_ID]", - "status": "OPERATION_STATUS_SUCCEEDED", - "sequence_id": "0" - } -} -{ - "method": "PATCH", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.independent1", - "q": { - "update_mask": "state,error_message,resource_id,status" - }, - "body": { - "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json\",\"version_id\":\"1\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"job independent1\",\"queue\":{\"enabled\":true}}}", - "resource_id": "[INDEPENDENT1_ID]", - "status": "OPERATION_STATUS_SUCCEEDED", - "sequence_id": "0" - } -} -{ - "method": "PATCH", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.independent2", - "q": { - "update_mask": "state,error_message,resource_id,status" - }, - "body": { - "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json\",\"version_id\":\"1\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"job independent2\",\"queue\":{\"enabled\":true}}}", - "resource_id": "[INDEPENDENT2_ID]", - "status": "OPERATION_STATUS_SUCCEEDED", - "sequence_id": "0" - } -} -{ - "method": "PATCH", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.multi_child", - "q": { - "update_mask": "state,error_message,resource_id,status" - }, - "body": { - "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json\",\"version_id\":\"1\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"job multi_child\",\"queue\":{\"enabled\":true},\"tasks\":[{\"run_job_task\":{\"job_id\":[MULTI_PARENT1_ID]},\"task_key\":\"t1\"},{\"run_job_task\":{\"job_id\":[MULTI_PARENT2_ID]},\"task_key\":\"t2\"}]},\"depends_on\":[{\"node\":\"resources.jobs.multi_parent1\",\"label\":\"${resources.jobs.multi_parent1.id}\"},{\"node\":\"resources.jobs.multi_parent2\",\"label\":\"${resources.jobs.multi_parent2.id}\"}]}", - "resource_id": "[MULTI_CHILD_ID]", - "status": "OPERATION_STATUS_SUCCEEDED", - "sequence_id": "0" - } -} -{ - "method": "PATCH", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.multi_parent1", - "q": { - "update_mask": "state,error_message,resource_id,status" - }, - "body": { - "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json\",\"version_id\":\"1\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"job multi_parent1\",\"queue\":{\"enabled\":true}}}", - "resource_id": "[MULTI_PARENT1_ID]", - "status": "OPERATION_STATUS_SUCCEEDED", - "sequence_id": "0" - } -} -{ - "method": "PATCH", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.multi_parent2", - "q": { - "update_mask": "state,error_message,resource_id,status" - }, - "body": { - "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json\",\"version_id\":\"1\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"job multi_parent2\",\"queue\":{\"enabled\":true}}}", - "resource_id": "[MULTI_PARENT2_ID]", - "status": "OPERATION_STATUS_SUCCEEDED", - "sequence_id": "0" - } -} { "method": "POST", "path": "/api/2.2/jobs/create", diff --git a/acceptance/bundle/resource_deps/job_id_big_graph/destroy/output.txt b/acceptance/bundle/resource_deps/job_id_big_graph/destroy/output.txt index 180f5c6e01d..d5630cbe32b 100644 --- a/acceptance/bundle/resource_deps/job_id_big_graph/destroy/output.txt +++ b/acceptance/bundle/resource_deps/job_id_big_graph/destroy/output.txt @@ -17,162 +17,6 @@ Files: 0 uploaded, 0 deleted Resources: 12 created, 0 changed, 0 deleted, 0 unchanged >>> print_requests.py --nostamp --sort //jobs -{ - "method": "PATCH", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.chain_bottom", - "q": { - "update_mask": "state,error_message,resource_id,status" - }, - "body": { - "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json\",\"version_id\":\"1\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"job chain_bottom\",\"queue\":{\"enabled\":true}}}", - "resource_id": "[CHAIN_BOTTOM_ID]", - "status": "OPERATION_STATUS_SUCCEEDED", - "sequence_id": "0" - } -} -{ - "method": "PATCH", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.chain_mid", - "q": { - "update_mask": "state,error_message,resource_id,status" - }, - "body": { - "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json\",\"version_id\":\"1\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"job chain_mid\",\"queue\":{\"enabled\":true},\"tasks\":[{\"run_job_task\":{\"job_id\":[CHAIN_BOTTOM_ID]},\"task_key\":\"t1\"}]},\"depends_on\":[{\"node\":\"resources.jobs.chain_bottom\",\"label\":\"${resources.jobs.chain_bottom.id}\"}]}", - "resource_id": "[CHAIN_MID_ID]", - "status": "OPERATION_STATUS_SUCCEEDED", - "sequence_id": "0" - } -} -{ - "method": "PATCH", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.chain_top", - "q": { - "update_mask": "state,error_message,resource_id,status" - }, - "body": { - "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json\",\"version_id\":\"1\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"job chain_top\",\"queue\":{\"enabled\":true},\"tasks\":[{\"run_job_task\":{\"job_id\":[CHAIN_MID_ID]},\"task_key\":\"t1\"}]},\"depends_on\":[{\"node\":\"resources.jobs.chain_mid\",\"label\":\"${resources.jobs.chain_mid.id}\"}]}", - "resource_id": "[CHAIN_TOP_ID]", - "status": "OPERATION_STATUS_SUCCEEDED", - "sequence_id": "0" - } -} -{ - "method": "PATCH", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.diamond_bottom", - "q": { - "update_mask": "state,error_message,resource_id,status" - }, - "body": { - "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json\",\"version_id\":\"1\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"job diamond_bottom\",\"queue\":{\"enabled\":true}}}", - "resource_id": "[DIAMOND_BOTTOM_ID]", - "status": "OPERATION_STATUS_SUCCEEDED", - "sequence_id": "0" - } -} -{ - "method": "PATCH", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.diamond_left", - "q": { - "update_mask": "state,error_message,resource_id,status" - }, - "body": { - "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json\",\"version_id\":\"1\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"job diamond_left\",\"queue\":{\"enabled\":true},\"tasks\":[{\"run_job_task\":{\"job_id\":[DIAMOND_BOTTOM_ID]},\"task_key\":\"t1\"}]},\"depends_on\":[{\"node\":\"resources.jobs.diamond_bottom\",\"label\":\"${resources.jobs.diamond_bottom.id}\"}]}", - "resource_id": "[DIAMOND_LEFT_ID]", - "status": "OPERATION_STATUS_SUCCEEDED", - "sequence_id": "0" - } -} -{ - "method": "PATCH", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.diamond_right", - "q": { - "update_mask": "state,error_message,resource_id,status" - }, - "body": { - "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json\",\"version_id\":\"1\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"job diamond_right\",\"queue\":{\"enabled\":true},\"tasks\":[{\"run_job_task\":{\"job_id\":[DIAMOND_BOTTOM_ID]},\"task_key\":\"t1\"}]},\"depends_on\":[{\"node\":\"resources.jobs.diamond_bottom\",\"label\":\"${resources.jobs.diamond_bottom.id}\"}]}", - "resource_id": "[DIAMOND_RIGHT_ID]", - "status": "OPERATION_STATUS_SUCCEEDED", - "sequence_id": "0" - } -} -{ - "method": "PATCH", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.diamond_top", - "q": { - "update_mask": "state,error_message,resource_id,status" - }, - "body": { - "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json\",\"version_id\":\"1\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"job diamond_top\",\"queue\":{\"enabled\":true},\"tasks\":[{\"run_job_task\":{\"job_id\":[DIAMOND_LEFT_ID]},\"task_key\":\"t1\"},{\"run_job_task\":{\"job_id\":[DIAMOND_RIGHT_ID]},\"task_key\":\"t2\"}]},\"depends_on\":[{\"node\":\"resources.jobs.diamond_left\",\"label\":\"${resources.jobs.diamond_left.id}\"},{\"node\":\"resources.jobs.diamond_right\",\"label\":\"${resources.jobs.diamond_right.id}\"}]}", - "resource_id": "[DIAMOND_TOP_ID]", - "status": "OPERATION_STATUS_SUCCEEDED", - "sequence_id": "0" - } -} -{ - "method": "PATCH", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.independent1", - "q": { - "update_mask": "state,error_message,resource_id,status" - }, - "body": { - "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json\",\"version_id\":\"1\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"job independent1\",\"queue\":{\"enabled\":true}}}", - "resource_id": "[INDEPENDENT1_ID]", - "status": "OPERATION_STATUS_SUCCEEDED", - "sequence_id": "0" - } -} -{ - "method": "PATCH", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.independent2", - "q": { - "update_mask": "state,error_message,resource_id,status" - }, - "body": { - "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json\",\"version_id\":\"1\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"job independent2\",\"queue\":{\"enabled\":true}}}", - "resource_id": "[INDEPENDENT2_ID]", - "status": "OPERATION_STATUS_SUCCEEDED", - "sequence_id": "0" - } -} -{ - "method": "PATCH", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.multi_child", - "q": { - "update_mask": "state,error_message,resource_id,status" - }, - "body": { - "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json\",\"version_id\":\"1\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"job multi_child\",\"queue\":{\"enabled\":true},\"tasks\":[{\"run_job_task\":{\"job_id\":[MULTI_PARENT1_ID]},\"task_key\":\"t1\"},{\"run_job_task\":{\"job_id\":[MULTI_PARENT2_ID]},\"task_key\":\"t2\"}]},\"depends_on\":[{\"node\":\"resources.jobs.multi_parent1\",\"label\":\"${resources.jobs.multi_parent1.id}\"},{\"node\":\"resources.jobs.multi_parent2\",\"label\":\"${resources.jobs.multi_parent2.id}\"}]}", - "resource_id": "[MULTI_CHILD_ID]", - "status": "OPERATION_STATUS_SUCCEEDED", - "sequence_id": "0" - } -} -{ - "method": "PATCH", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.multi_parent1", - "q": { - "update_mask": "state,error_message,resource_id,status" - }, - "body": { - "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json\",\"version_id\":\"1\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"job multi_parent1\",\"queue\":{\"enabled\":true}}}", - "resource_id": "[MULTI_PARENT1_ID]", - "status": "OPERATION_STATUS_SUCCEEDED", - "sequence_id": "0" - } -} -{ - "method": "PATCH", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.multi_parent2", - "q": { - "update_mask": "state,error_message,resource_id,status" - }, - "body": { - "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json\",\"version_id\":\"1\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"job multi_parent2\",\"queue\":{\"enabled\":true}}}", - "resource_id": "[MULTI_PARENT2_ID]", - "status": "OPERATION_STATUS_SUCCEEDED", - "sequence_id": "0" - } -} { "method": "POST", "path": "/api/2.2/jobs/create", diff --git a/acceptance/bundle/resources/apps/default_description/output.txt b/acceptance/bundle/resources/apps/default_description/output.txt index dd6087eab5b..1f78fb9801d 100644 --- a/acceptance/bundle/resources/apps/default_description/output.txt +++ b/acceptance/bundle/resources/apps/default_description/output.txt @@ -17,16 +17,3 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "name": "myappname" } } -{ - "method": "PATCH", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/apps.mykey", - "q": { - "update_mask": "state,error_message,resource_id,status" - }, - "body": { - "state": "{\"state\":{\"description\":\"\",\"name\":\"myappname\",\"source_code_path\":\"/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files\"}}", - "resource_id": "myappname", - "status": "OPERATION_STATUS_SUCCEEDED", - "sequence_id": "0" - } -} diff --git a/acceptance/bundle/resources/apps/lifecycle-started-toggle/output.txt b/acceptance/bundle/resources/apps/lifecycle-started-toggle/output.txt index 2d932d7a628..ef03a4541d9 100644 --- a/acceptance/bundle/resources/apps/lifecycle-started-toggle/output.txt +++ b/acceptance/bundle/resources/apps/lifecycle-started-toggle/output.txt @@ -18,19 +18,6 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "no_compute": "true" } } -{ - "body": { - "resource_id": "[UNIQUE_NAME]", - "sequence_id": "0", - "state": "{\"state\":{\"description\":\"my_app_description\",\"lifecycle\":{\"started\":false},\"name\":\"[UNIQUE_NAME]\",\"source_code_path\":\"/Workspace/Users/[USERNAME]/.bundle/lifecycle-started-toggle-[UNIQUE_NAME]/default/files/app\"}}", - "status": "OPERATION_STATUS_SUCCEEDED" - }, - "method": "PATCH", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/apps.mykey", - "q": { - "update_mask": "state,error_message,resource_id,status" - } -} >>> errcode [CLI] apps get [UNIQUE_NAME] "STOPPED" @@ -58,19 +45,6 @@ Resources: 0 created, 1 changed, 0 deleted, 0 unchanged "method": "POST", "path": "/api/2.0/apps/[UNIQUE_NAME]/deployments" } -{ - "body": { - "resource_id": "[UNIQUE_NAME]", - "sequence_id": "0", - "state": "{\"state\":{\"description\":\"my_app_description\",\"lifecycle\":{\"started\":true},\"name\":\"[UNIQUE_NAME]\",\"source_code_path\":\"/Workspace/Users/[USERNAME]/.bundle/lifecycle-started-toggle-[UNIQUE_NAME]/default/files/app\"}}", - "status": "OPERATION_STATUS_SUCCEEDED" - }, - "method": "PATCH", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations/apps.mykey", - "q": { - "update_mask": "state,error_message,resource_id,status" - } -} >>> errcode [CLI] apps get [UNIQUE_NAME] "ACTIVE" @@ -90,19 +64,6 @@ Resources: 0 created, 1 changed, 0 deleted, 0 unchanged "method": "POST", "path": "/api/2.0/apps/[UNIQUE_NAME]/stop" } -{ - "body": { - "resource_id": "[UNIQUE_NAME]", - "sequence_id": "0", - "state": "{\"state\":{\"description\":\"my_app_description\",\"lifecycle\":{\"started\":false},\"name\":\"[UNIQUE_NAME]\",\"source_code_path\":\"/Workspace/Users/[USERNAME]/.bundle/lifecycle-started-toggle-[UNIQUE_NAME]/default/files/app\"}}", - "status": "OPERATION_STATUS_SUCCEEDED" - }, - "method": "PATCH", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/3/operations/apps.mykey", - "q": { - "update_mask": "state,error_message,resource_id,status" - } -} >>> errcode [CLI] apps get [UNIQUE_NAME] "STOPPED" diff --git a/acceptance/bundle/resources/apps/lifecycle-started-toggle/script b/acceptance/bundle/resources/apps/lifecycle-started-toggle/script index 7b5a2c8c32f..0b524ad2efa 100644 --- a/acceptance/bundle/resources/apps/lifecycle-started-toggle/script +++ b/acceptance/bundle/resources/apps/lifecycle-started-toggle/script @@ -7,7 +7,7 @@ cleanup() { trap cleanup EXIT print_app_requests() { - jq --sort-keys 'select(.method != "GET" and (.path | contains("/apps"))) | del(.body.url)' < out.requests.txt + jq --sort-keys 'select(.method != "GET" and (.path | contains("/apps")) and (.path | contains("/api/2.0/bundle") | not)) | del(.body.url)' < out.requests.txt rm out.requests.txt } diff --git a/acceptance/bundle/resources/apps/resource-refs/output.txt b/acceptance/bundle/resources/apps/resource-refs/output.txt index f4f60ec264f..14fcdcb0972 100644 --- a/acceptance/bundle/resources/apps/resource-refs/output.txt +++ b/acceptance/bundle/resources/apps/resource-refs/output.txt @@ -29,19 +29,6 @@ You can access the app at data-app-123.cloud.databricksapps.com "name": "data-app" } } -{ - "method": "PATCH", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/apps.data_app", - "q": { - "update_mask": "state,error_message,resource_id,status" - }, - "body": { - "state": "{\"state\":{\"config\":{\"command\":[\"streamlit\",\"run\",\"app.py\"],\"env\":[{\"name\":\"MY_EXAMPLE_SCHEMA\",\"value\":\"main\"},{\"name\":\"MY_EXAMPLE_JOB\",\"value\":\"example_job\"},{\"name\":\"MY_EXAMPLE_JOB_ID\",\"value\":\"[NUMID]\"},{\"name\":\"MY_EXAMPLE_VAR\",\"value\":\"example_value\"}]},\"description\":\"A Streamlit app that uses a SQL warehouse\",\"name\":\"data-app\",\"source_code_path\":\"/Workspace/Users/[USERNAME]/.bundle/resource-refs/default/files/app\"},\"depends_on\":[{\"node\":\"resources.jobs.example_job\",\"label\":\"${resources.jobs.example_job.id}\"},{\"node\":\"resources.jobs.example_job\",\"label\":\"${resources.jobs.example_job.name}\"},{\"node\":\"resources.schemas.example\",\"label\":\"${resources.schemas.example.catalog_name}\"}]}", - "resource_id": "data-app", - "status": "OPERATION_STATUS_SUCCEEDED", - "sequence_id": "0" - } -} { "method": "POST", "path": "/api/2.0/apps/data-app/start", diff --git a/acceptance/bundle/resources/clusters/lifecycle-started-toggle/output.txt b/acceptance/bundle/resources/clusters/lifecycle-started-toggle/output.txt index 4c9e068017a..4ece922a78e 100644 --- a/acceptance/bundle/resources/clusters/lifecycle-started-toggle/output.txt +++ b/acceptance/bundle/resources/clusters/lifecycle-started-toggle/output.txt @@ -18,19 +18,6 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "spark_version": "15.4.x-scala2.12" } } -{ - "method": "PATCH", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/clusters.mycluster", - "q": { - "update_mask": "state,error_message,resource_id,status" - }, - "body": { - "state": "{\"state\":{\"autotermination_minutes\":60,\"cluster_name\":\"[UNIQUE_NAME]\",\"instance_pool_id\":\"[TEST_INSTANCE_POOL_ID]\",\"lifecycle\":{\"started\":false},\"num_workers\":1,\"spark_version\":\"15.4.x-scala2.12\"}}", - "resource_id": "[UUID]", - "status": "OPERATION_STATUS_SUCCEEDED", - "sequence_id": "0" - } -} { "method": "POST", "path": "/api/2.1/clusters/delete", @@ -59,19 +46,6 @@ Resources: 0 created, 1 changed, 0 deleted, 0 unchanged "cluster_id": "[UUID]" } } -{ - "method": "PATCH", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations/clusters.mycluster", - "q": { - "update_mask": "state,error_message,resource_id,status" - }, - "body": { - "state": "{\"state\":{\"autotermination_minutes\":60,\"cluster_name\":\"[UNIQUE_NAME]\",\"instance_pool_id\":\"[TEST_INSTANCE_POOL_ID]\",\"lifecycle\":{\"started\":true},\"num_workers\":1,\"spark_version\":\"15.4.x-scala2.12\"}}", - "resource_id": "[UUID]", - "status": "OPERATION_STATUS_SUCCEEDED", - "sequence_id": "0" - } -} >>> errcode [CLI] clusters get [UUID] "RUNNING" @@ -93,19 +67,6 @@ Resources: 0 created, 1 changed, 0 deleted, 0 unchanged "cluster_id": "[UUID]" } } -{ - "method": "PATCH", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/3/operations/clusters.mycluster", - "q": { - "update_mask": "state,error_message,resource_id,status" - }, - "body": { - "state": "{\"state\":{\"autotermination_minutes\":60,\"cluster_name\":\"[UNIQUE_NAME]\",\"instance_pool_id\":\"[TEST_INSTANCE_POOL_ID]\",\"lifecycle\":{\"started\":false},\"num_workers\":1,\"spark_version\":\"15.4.x-scala2.12\"}}", - "resource_id": "[UUID]", - "status": "OPERATION_STATUS_SUCCEEDED", - "sequence_id": "0" - } -} >>> errcode [CLI] clusters get [UUID] "TERMINATED" diff --git a/acceptance/bundle/resources/clusters/lifecycle-started/output.txt b/acceptance/bundle/resources/clusters/lifecycle-started/output.txt index 146ff8d1f82..d902c63731d 100644 --- a/acceptance/bundle/resources/clusters/lifecycle-started/output.txt +++ b/acceptance/bundle/resources/clusters/lifecycle-started/output.txt @@ -18,19 +18,6 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "spark_version": "15.4.x-scala2.12" } } -{ - "method": "PATCH", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/clusters.mycluster", - "q": { - "update_mask": "state,error_message,resource_id,status" - }, - "body": { - "state": "{\"state\":{\"autotermination_minutes\":60,\"cluster_name\":\"[UNIQUE_NAME]\",\"instance_pool_id\":\"[TEST_INSTANCE_POOL_ID]\",\"lifecycle\":{\"started\":true},\"num_workers\":1,\"spark_version\":\"15.4.x-scala2.12\"}}", - "resource_id": "[UUID]", - "status": "OPERATION_STATUS_SUCCEEDED", - "sequence_id": "0" - } -} >>> errcode [CLI] clusters get [UUID] "RUNNING" @@ -66,19 +53,6 @@ Resources: 0 created, 1 changed, 0 deleted, 0 unchanged "cluster_id": "[UUID]" } } -{ - "method": "PATCH", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations/clusters.mycluster", - "q": { - "update_mask": "state,error_message,resource_id,status" - }, - "body": { - "state": "{\"state\":{\"autotermination_minutes\":60,\"cluster_name\":\"[UNIQUE_NAME]\",\"instance_pool_id\":\"[TEST_INSTANCE_POOL_ID]\",\"lifecycle\":{\"started\":true},\"num_workers\":1,\"spark_version\":\"15.4.x-scala2.12\"}}", - "resource_id": "[UUID]", - "status": "OPERATION_STATUS_SUCCEEDED", - "sequence_id": "0" - } -} >>> errcode [CLI] clusters get [UUID] "RUNNING" @@ -123,19 +97,6 @@ Resources: 0 created, 1 changed, 0 deleted, 0 unchanged "cluster_id": "[UUID]" } } -{ - "method": "PATCH", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/4/operations/clusters.mycluster", - "q": { - "update_mask": "state,error_message,resource_id,status" - }, - "body": { - "state": "{\"state\":{\"autotermination_minutes\":60,\"cluster_name\":\"[UNIQUE_NAME]\",\"instance_pool_id\":\"[TEST_INSTANCE_POOL_ID]\",\"lifecycle\":{\"started\":true},\"num_workers\":1,\"spark_version\":\"15.4.x-scala2.12\"}}", - "resource_id": "[UUID]", - "status": "OPERATION_STATUS_SUCCEEDED", - "sequence_id": "0" - } -} >>> errcode [CLI] clusters get [UUID] "RUNNING" diff --git a/acceptance/bundle/resources/jobs/num_workers/output.txt b/acceptance/bundle/resources/jobs/num_workers/output.txt index 7754bd0bb69..9004ecb3b95 100644 --- a/acceptance/bundle/resources/jobs/num_workers/output.txt +++ b/acceptance/bundle/resources/jobs/num_workers/output.txt @@ -107,19 +107,6 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged } } } -{ - "method": "PATCH", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.sample_job", - "q": { - "update_mask": "state,error_message,resource_id,status" - }, - "body": { - "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json\",\"version_id\":\"1\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"job_clusters\":[{\"job_cluster_key\":\"job_cluster_autoscale\",\"new_cluster\":{\"autoscale\":{\"max_workers\":4,\"min_workers\":1},\"data_security_mode\":\"SINGLE_USER\",\"node_type_id\":\"[NODE_TYPE_ID]\",\"spark_version\":\"16.4.x-scala2.12\"}},{\"job_cluster_key\":\"job_cluster_autoscale_num_workers1\",\"new_cluster\":{\"autoscale\":{\"max_workers\":4,\"min_workers\":1},\"data_security_mode\":\"SINGLE_USER\",\"node_type_id\":\"[NODE_TYPE_ID]\",\"num_workers\":1,\"spark_version\":\"16.4.x-scala2.14\"}},{\"job_cluster_key\":\"job_cluster_num_workers1\",\"new_cluster\":{\"data_security_mode\":\"SINGLE_USER\",\"node_type_id\":\"[NODE_TYPE_ID]\",\"num_workers\":1,\"spark_version\":\"16.4.x-scala2.15\"}},{\"job_cluster_key\":\"job_cluster_num_workers0\",\"new_cluster\":{\"data_security_mode\":\"SINGLE_USER\",\"node_type_id\":\"[NODE_TYPE_ID]\",\"num_workers\":0,\"spark_version\":\"16.4.x-scala2.16\"}},{\"job_cluster_key\":\"job_cluster_default\",\"new_cluster\":{\"num_workers\":0,\"spark_version\":\"16.4.x-scala2.17\"}}],\"max_concurrent_runs\":1,\"name\":\"sample_job\",\"queue\":{\"enabled\":true},\"tasks\":[{\"notebook_task\":{\"notebook_path\":\"/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/sample_notebook\",\"source\":\"WORKSPACE\"},\"task_key\":\"notebook_task\"}],\"trigger\":{\"pause_status\":\"UNPAUSED\",\"periodic\":{\"interval\":1,\"unit\":\"DAYS\"}}}}", - "resource_id": "[NUMID]", - "status": "OPERATION_STATUS_SUCCEEDED", - "sequence_id": "0" - } -} >>> [CLI] bundle plan Warning: Single node cluster is not correctly configured diff --git a/acceptance/bundle/resources/jobs/webhook-reorder-remote/output.txt b/acceptance/bundle/resources/jobs/webhook-reorder-remote/output.txt index 7f489be1b63..f0fdccbb1db 100644 --- a/acceptance/bundle/resources/jobs/webhook-reorder-remote/output.txt +++ b/acceptance/bundle/resources/jobs/webhook-reorder-remote/output.txt @@ -65,19 +65,6 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged } } } -{ - "method": "PATCH", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.my_job", - "q": { - "update_mask": "state,error_message,resource_id,status" - }, - "body": { - "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json\",\"version_id\":\"1\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"webhook reorder\",\"queue\":{\"enabled\":true},\"tasks\":[{\"run_job_task\":{\"job_id\":123},\"task_key\":\"main\",\"webhook_notifications\":{\"on_start\":[{\"id\":\"delta\"},{\"id\":\"epsilon\"}]}}],\"webhook_notifications\":{\"on_success\":[{\"id\":\"alpha\"},{\"id\":\"beta\"},{\"id\":\"gamma\"}]}}}", - "resource_id": "[MY_JOB_ID]", - "status": "OPERATION_STATUS_SUCCEEDED", - "sequence_id": "0" - } -} { "method": "POST", "path": "/api/2.2/jobs/reset", diff --git a/acceptance/bundle/resources/permissions/_script b/acceptance/bundle/resources/permissions/_script index 1ba0dd3bf8a..8f6a0d9c8cd 100644 --- a/acceptance/bundle/resources/permissions/_script +++ b/acceptance/bundle/resources/permissions/_script @@ -4,7 +4,7 @@ rm out.requests.txt $CLI bundle plan -o json | nostamp > out.plan.$DATABRICKS_BUNDLE_ENGINE.json print_requests() { - jq -c < out.requests.txt | jq 'select(.method != "GET" and (.path | contains("permissions")))' + jq -c < out.requests.txt | jq 'select(.method != "GET" and (.path | contains("permissions")) and (.path | contains("/api/2.0/bundle") | not))' rm out.requests.txt } diff --git a/acceptance/bundle/resources/permissions/apps/other_can_manage/out.requests.deploy.direct.json b/acceptance/bundle/resources/permissions/apps/other_can_manage/out.requests.deploy.direct.json index b210b6947fc..831cda2417a 100644 --- a/acceptance/bundle/resources/permissions/apps/other_can_manage/out.requests.deploy.direct.json +++ b/acceptance/bundle/resources/permissions/apps/other_can_manage/out.requests.deploy.direct.json @@ -22,16 +22,3 @@ ] } } -{ - "method": "PATCH", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/apps.foo.permissions", - "q": { - "update_mask": "state,error_message,resource_id,status" - }, - "body": { - "state": "{\"state\":{\"object_id\":\"/apps/foo\",\"__embed__\":[{\"level\":\"CAN_USE\",\"user_name\":\"viewer@example.com\"},{\"level\":\"CAN_MANAGE\",\"group_name\":\"data-team\"},{\"level\":\"CAN_MANAGE\",\"service_principal_name\":\"[UUID]\"},{\"level\":\"CAN_MANAGE\",\"user_name\":\"[USERNAME]\"}]},\"depends_on\":[{\"node\":\"resources.apps.foo\",\"label\":\"${resources.apps.foo.id}\"}]}", - "resource_id": "/apps/foo", - "status": "OPERATION_STATUS_SUCCEEDED", - "sequence_id": "0" - } -} diff --git a/acceptance/bundle/resources/permissions/apps/other_can_manage/out.requests.destroy.direct.json b/acceptance/bundle/resources/permissions/apps/other_can_manage/out.requests.destroy.direct.json index 47b6e130045..e69de29bb2d 100644 --- a/acceptance/bundle/resources/permissions/apps/other_can_manage/out.requests.destroy.direct.json +++ b/acceptance/bundle/resources/permissions/apps/other_can_manage/out.requests.destroy.direct.json @@ -1,12 +0,0 @@ -{ - "method": "PATCH", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations/apps.foo.permissions", - "q": { - "update_mask": "state,error_message,resource_id,status" - }, - "body": { - "resource_id": "/apps/foo", - "status": "OPERATION_STATUS_SUCCEEDED", - "sequence_id": "0" - } -} diff --git a/acceptance/bundle/resources/permissions/clusters/target/out.requests.direct.json b/acceptance/bundle/resources/permissions/clusters/target/out.requests.direct.json index 407146fdcfc..9a0adb10f0f 100644 --- a/acceptance/bundle/resources/permissions/clusters/target/out.requests.direct.json +++ b/acceptance/bundle/resources/permissions/clusters/target/out.requests.direct.json @@ -12,19 +12,6 @@ "spark_version": "15.4.x-scala2.12" } } -{ - "method": "PATCH", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/clusters.cluster1", - "q": { - "update_mask": "state,error_message,resource_id,status" - }, - "body": { - "state": "{\"state\":{\"autotermination_minutes\":60,\"cluster_name\":\"[dev [USERNAME]] test-cluster\",\"custom_tags\":{\"dev\":\"[USERNAME]\"},\"node_type_id\":\"[NODE_TYPE_ID]\",\"num_workers\":1,\"spark_version\":\"15.4.x-scala2.12\"}}", - "resource_id": "[UUID]", - "status": "OPERATION_STATUS_SUCCEEDED", - "sequence_id": "0" - } -} { "method": "PUT", "path": "/api/2.0/permissions/clusters/[UUID]", @@ -49,16 +36,3 @@ ] } } -{ - "method": "PATCH", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/clusters.cluster1.permissions", - "q": { - "update_mask": "state,error_message,resource_id,status" - }, - "body": { - "state": "{\"state\":{\"object_id\":\"/clusters/[UUID]\",\"__embed__\":[{\"level\":\"CAN_ATTACH_TO\",\"service_principal_name\":\"[UUID]\"},{\"level\":\"CAN_RESTART\",\"service_principal_name\":\"[UUID]\"},{\"level\":\"CAN_MANAGE\",\"service_principal_name\":\"[UUID]\"},{\"level\":\"CAN_MANAGE\",\"user_name\":\"[USERNAME]\"}]},\"depends_on\":[{\"node\":\"resources.clusters.cluster1\",\"label\":\"${resources.clusters.cluster1.id}\"}]}", - "resource_id": "/clusters/[UUID]", - "status": "OPERATION_STATUS_SUCCEEDED", - "sequence_id": "0" - } -} diff --git a/acceptance/bundle/resources/permissions/database_instances/current_can_manage/out.requests.deploy.direct.json b/acceptance/bundle/resources/permissions/database_instances/current_can_manage/out.requests.deploy.direct.json index 5505e1d46d2..401bb4d72af 100644 --- a/acceptance/bundle/resources/permissions/database_instances/current_can_manage/out.requests.deploy.direct.json +++ b/acceptance/bundle/resources/permissions/database_instances/current_can_manage/out.requests.deploy.direct.json @@ -22,16 +22,3 @@ ] } } -{ - "method": "PATCH", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/database_instances.foo.permissions", - "q": { - "update_mask": "state,error_message,resource_id,status" - }, - "body": { - "state": "{\"state\":{\"object_id\":\"/database-instances/test-db-instance\",\"__embed__\":[{\"level\":\"CAN_USE\",\"user_name\":\"viewer@example.com\"},{\"level\":\"CAN_MANAGE\",\"group_name\":\"data-team\"},{\"level\":\"CAN_MANAGE\",\"service_principal_name\":\"[UUID]\"},{\"level\":\"CAN_MANAGE\",\"user_name\":\"[USERNAME]\"}]},\"depends_on\":[{\"node\":\"resources.database_instances.foo\",\"label\":\"${resources.database_instances.foo.id}\"}]}", - "resource_id": "/database-instances/test-db-instance", - "status": "OPERATION_STATUS_SUCCEEDED", - "sequence_id": "0" - } -} diff --git a/acceptance/bundle/resources/permissions/database_instances/current_can_manage/out.requests.destroy.direct.json b/acceptance/bundle/resources/permissions/database_instances/current_can_manage/out.requests.destroy.direct.json index 2dbd043b123..e69de29bb2d 100644 --- a/acceptance/bundle/resources/permissions/database_instances/current_can_manage/out.requests.destroy.direct.json +++ b/acceptance/bundle/resources/permissions/database_instances/current_can_manage/out.requests.destroy.direct.json @@ -1,12 +0,0 @@ -{ - "method": "PATCH", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations/database_instances.foo.permissions", - "q": { - "update_mask": "state,error_message,resource_id,status" - }, - "body": { - "resource_id": "/database-instances/test-db-instance", - "status": "OPERATION_STATUS_SUCCEEDED", - "sequence_id": "0" - } -} diff --git a/acceptance/bundle/resources/permissions/jobs/current_can_manage/out.requests.deploy.direct.json b/acceptance/bundle/resources/permissions/jobs/current_can_manage/out.requests.deploy.direct.json index dd5b3baefd4..5647c1a73b9 100644 --- a/acceptance/bundle/resources/permissions/jobs/current_can_manage/out.requests.deploy.direct.json +++ b/acceptance/bundle/resources/permissions/jobs/current_can_manage/out.requests.deploy.direct.json @@ -22,16 +22,3 @@ ] } } -{ - "method": "PATCH", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.foo.permissions", - "q": { - "update_mask": "state,error_message,resource_id,status" - }, - "body": { - "state": "{\"state\":{\"object_id\":\"/jobs/[NUMID]\",\"__embed__\":[{\"level\":\"CAN_VIEW\",\"user_name\":\"viewer@example.com\"},{\"level\":\"CAN_MANAGE\",\"group_name\":\"data-team\"},{\"level\":\"CAN_MANAGE\",\"service_principal_name\":\"[UUID]\"},{\"level\":\"IS_OWNER\",\"user_name\":\"[USERNAME]\"}]},\"depends_on\":[{\"node\":\"resources.jobs.foo\",\"label\":\"${resources.jobs.foo.id}\"}]}", - "resource_id": "/jobs/[NUMID]", - "status": "OPERATION_STATUS_SUCCEEDED", - "sequence_id": "0" - } -} diff --git a/acceptance/bundle/resources/permissions/jobs/current_can_manage/out.requests.destroy.direct.json b/acceptance/bundle/resources/permissions/jobs/current_can_manage/out.requests.destroy.direct.json index af78adb96dc..e69de29bb2d 100644 --- a/acceptance/bundle/resources/permissions/jobs/current_can_manage/out.requests.destroy.direct.json +++ b/acceptance/bundle/resources/permissions/jobs/current_can_manage/out.requests.destroy.direct.json @@ -1,12 +0,0 @@ -{ - "method": "PATCH", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations/jobs.foo.permissions", - "q": { - "update_mask": "state,error_message,resource_id,status" - }, - "body": { - "resource_id": "/jobs/[NUMID]", - "status": "OPERATION_STATUS_SUCCEEDED", - "sequence_id": "0" - } -} diff --git a/acceptance/bundle/resources/permissions/jobs/current_can_manage_run/out.destroy.requests.direct.json b/acceptance/bundle/resources/permissions/jobs/current_can_manage_run/out.destroy.requests.direct.json index 2ed516784ba..6564b6f8bde 100644 --- a/acceptance/bundle/resources/permissions/jobs/current_can_manage_run/out.destroy.requests.direct.json +++ b/acceptance/bundle/resources/permissions/jobs/current_can_manage_run/out.destroy.requests.direct.json @@ -1,15 +1,3 @@ -{ - "method": "PATCH", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations/jobs.interim_gold_layer_job.permissions", - "q": { - "update_mask": "state,error_message,resource_id,status" - }, - "body": { - "resource_id": "/jobs/[NUMID]", - "status": "OPERATION_STATUS_SUCCEEDED", - "sequence_id": "0" - } -} { "method": "POST", "path": "/api/2.2/jobs/delete", @@ -17,15 +5,3 @@ "job_id": [NUMID] } } -{ - "method": "PATCH", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations/jobs.interim_gold_layer_job", - "q": { - "update_mask": "state,error_message,resource_id,status" - }, - "body": { - "resource_id": "[NUMID]", - "status": "OPERATION_STATUS_SUCCEEDED", - "sequence_id": "0" - } -} diff --git a/acceptance/bundle/resources/permissions/jobs/current_is_owner/out.requests.deploy.direct.json b/acceptance/bundle/resources/permissions/jobs/current_is_owner/out.requests.deploy.direct.json index adaf900ff4e..f7aa2dbaa0d 100644 --- a/acceptance/bundle/resources/permissions/jobs/current_is_owner/out.requests.deploy.direct.json +++ b/acceptance/bundle/resources/permissions/jobs/current_is_owner/out.requests.deploy.direct.json @@ -10,16 +10,3 @@ ] } } -{ - "method": "PATCH", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.foo.permissions", - "q": { - "update_mask": "state,error_message,resource_id,status" - }, - "body": { - "state": "{\"state\":{\"object_id\":\"/jobs/[NUMID]\",\"__embed__\":[{\"level\":\"IS_OWNER\",\"user_name\":\"[USERNAME]\"}]},\"depends_on\":[{\"node\":\"resources.jobs.foo\",\"label\":\"${resources.jobs.foo.id}\"}]}", - "resource_id": "/jobs/[NUMID]", - "status": "OPERATION_STATUS_SUCCEEDED", - "sequence_id": "0" - } -} diff --git a/acceptance/bundle/resources/permissions/jobs/current_is_owner/out.requests.destroy.direct.json b/acceptance/bundle/resources/permissions/jobs/current_is_owner/out.requests.destroy.direct.json index af78adb96dc..e69de29bb2d 100644 --- a/acceptance/bundle/resources/permissions/jobs/current_is_owner/out.requests.destroy.direct.json +++ b/acceptance/bundle/resources/permissions/jobs/current_is_owner/out.requests.destroy.direct.json @@ -1,12 +0,0 @@ -{ - "method": "PATCH", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations/jobs.foo.permissions", - "q": { - "update_mask": "state,error_message,resource_id,status" - }, - "body": { - "resource_id": "/jobs/[NUMID]", - "status": "OPERATION_STATUS_SUCCEEDED", - "sequence_id": "0" - } -} diff --git a/acceptance/bundle/resources/permissions/jobs/other_can_manage/out.requests.deploy.direct.json b/acceptance/bundle/resources/permissions/jobs/other_can_manage/out.requests.deploy.direct.json index dd5b3baefd4..5647c1a73b9 100644 --- a/acceptance/bundle/resources/permissions/jobs/other_can_manage/out.requests.deploy.direct.json +++ b/acceptance/bundle/resources/permissions/jobs/other_can_manage/out.requests.deploy.direct.json @@ -22,16 +22,3 @@ ] } } -{ - "method": "PATCH", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.foo.permissions", - "q": { - "update_mask": "state,error_message,resource_id,status" - }, - "body": { - "state": "{\"state\":{\"object_id\":\"/jobs/[NUMID]\",\"__embed__\":[{\"level\":\"CAN_VIEW\",\"user_name\":\"viewer@example.com\"},{\"level\":\"CAN_MANAGE\",\"group_name\":\"data-team\"},{\"level\":\"CAN_MANAGE\",\"service_principal_name\":\"[UUID]\"},{\"level\":\"IS_OWNER\",\"user_name\":\"[USERNAME]\"}]},\"depends_on\":[{\"node\":\"resources.jobs.foo\",\"label\":\"${resources.jobs.foo.id}\"}]}", - "resource_id": "/jobs/[NUMID]", - "status": "OPERATION_STATUS_SUCCEEDED", - "sequence_id": "0" - } -} diff --git a/acceptance/bundle/resources/permissions/jobs/other_can_manage/out.requests.destroy.direct.json b/acceptance/bundle/resources/permissions/jobs/other_can_manage/out.requests.destroy.direct.json index af78adb96dc..e69de29bb2d 100644 --- a/acceptance/bundle/resources/permissions/jobs/other_can_manage/out.requests.destroy.direct.json +++ b/acceptance/bundle/resources/permissions/jobs/other_can_manage/out.requests.destroy.direct.json @@ -1,12 +0,0 @@ -{ - "method": "PATCH", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations/jobs.foo.permissions", - "q": { - "update_mask": "state,error_message,resource_id,status" - }, - "body": { - "resource_id": "/jobs/[NUMID]", - "status": "OPERATION_STATUS_SUCCEEDED", - "sequence_id": "0" - } -} diff --git a/acceptance/bundle/resources/permissions/jobs/other_can_manage_run/out.destroy.requests.direct.json b/acceptance/bundle/resources/permissions/jobs/other_can_manage_run/out.destroy.requests.direct.json index 58fdd00dd6f..6564b6f8bde 100644 --- a/acceptance/bundle/resources/permissions/jobs/other_can_manage_run/out.destroy.requests.direct.json +++ b/acceptance/bundle/resources/permissions/jobs/other_can_manage_run/out.destroy.requests.direct.json @@ -5,27 +5,3 @@ "job_id": [NUMID] } } -{ - "method": "PATCH", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations/jobs.interim_gold_layer_job.permissions", - "q": { - "update_mask": "state,error_message,resource_id,status" - }, - "body": { - "resource_id": "/jobs/[NUMID]", - "status": "OPERATION_STATUS_SUCCEEDED", - "sequence_id": "0" - } -} -{ - "method": "PATCH", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations/jobs.interim_gold_layer_job", - "q": { - "update_mask": "state,error_message,resource_id,status" - }, - "body": { - "resource_id": "[NUMID]", - "status": "OPERATION_STATUS_SUCCEEDED", - "sequence_id": "0" - } -} diff --git a/acceptance/bundle/resources/permissions/jobs/other_is_owner/out.requests.deploy.direct.json b/acceptance/bundle/resources/permissions/jobs/other_is_owner/out.requests.deploy.direct.json index f46c2203e72..b1eb1519e87 100644 --- a/acceptance/bundle/resources/permissions/jobs/other_is_owner/out.requests.deploy.direct.json +++ b/acceptance/bundle/resources/permissions/jobs/other_is_owner/out.requests.deploy.direct.json @@ -14,16 +14,3 @@ ] } } -{ - "method": "PATCH", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.foo.permissions", - "q": { - "update_mask": "state,error_message,resource_id,status" - }, - "body": { - "state": "{\"state\":{\"object_id\":\"/jobs/[NUMID]\",\"__embed__\":[{\"level\":\"IS_OWNER\",\"user_name\":\"other_user@databricks.com\"},{\"level\":\"CAN_MANAGE\",\"user_name\":\"[USERNAME]\"}]},\"depends_on\":[{\"node\":\"resources.jobs.foo\",\"label\":\"${resources.jobs.foo.id}\"}]}", - "resource_id": "/jobs/[NUMID]", - "status": "OPERATION_STATUS_SUCCEEDED", - "sequence_id": "0" - } -} diff --git a/acceptance/bundle/resources/permissions/jobs/other_is_owner/out.requests.destroy.direct.json b/acceptance/bundle/resources/permissions/jobs/other_is_owner/out.requests.destroy.direct.json index af78adb96dc..e69de29bb2d 100644 --- a/acceptance/bundle/resources/permissions/jobs/other_is_owner/out.requests.destroy.direct.json +++ b/acceptance/bundle/resources/permissions/jobs/other_is_owner/out.requests.destroy.direct.json @@ -1,12 +0,0 @@ -{ - "method": "PATCH", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations/jobs.foo.permissions", - "q": { - "update_mask": "state,error_message,resource_id,status" - }, - "body": { - "resource_id": "/jobs/[NUMID]", - "status": "OPERATION_STATUS_SUCCEEDED", - "sequence_id": "0" - } -} diff --git a/acceptance/bundle/resources/permissions/jobs/viewers/out.requests.deploy.direct.json b/acceptance/bundle/resources/permissions/jobs/viewers/out.requests.deploy.direct.json index aadff3ecc7d..45768bac0d5 100644 --- a/acceptance/bundle/resources/permissions/jobs/viewers/out.requests.deploy.direct.json +++ b/acceptance/bundle/resources/permissions/jobs/viewers/out.requests.deploy.direct.json @@ -22,16 +22,3 @@ ] } } -{ - "method": "PATCH", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.foo.permissions", - "q": { - "update_mask": "state,error_message,resource_id,status" - }, - "body": { - "state": "{\"state\":{\"object_id\":\"/jobs/[NUMID]\",\"__embed__\":[{\"level\":\"CAN_VIEW\",\"user_name\":\"viewer@example.com\"},{\"level\":\"CAN_VIEW\",\"group_name\":\"data-team\"},{\"level\":\"CAN_VIEW\",\"service_principal_name\":\"[UUID]\"},{\"level\":\"IS_OWNER\",\"user_name\":\"[USERNAME]\"}]},\"depends_on\":[{\"node\":\"resources.jobs.foo\",\"label\":\"${resources.jobs.foo.id}\"}]}", - "resource_id": "/jobs/[NUMID]", - "status": "OPERATION_STATUS_SUCCEEDED", - "sequence_id": "0" - } -} diff --git a/acceptance/bundle/resources/permissions/jobs/viewers/out.requests.destroy.direct.json b/acceptance/bundle/resources/permissions/jobs/viewers/out.requests.destroy.direct.json index af78adb96dc..e69de29bb2d 100644 --- a/acceptance/bundle/resources/permissions/jobs/viewers/out.requests.destroy.direct.json +++ b/acceptance/bundle/resources/permissions/jobs/viewers/out.requests.destroy.direct.json @@ -1,12 +0,0 @@ -{ - "method": "PATCH", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations/jobs.foo.permissions", - "q": { - "update_mask": "state,error_message,resource_id,status" - }, - "body": { - "resource_id": "/jobs/[NUMID]", - "status": "OPERATION_STATUS_SUCCEEDED", - "sequence_id": "0" - } -} diff --git a/acceptance/bundle/resources/permissions/output.txt b/acceptance/bundle/resources/permissions/output.txt index e5714e3f640..85a16ad6a38 100644 --- a/acceptance/bundle/resources/permissions/output.txt +++ b/acceptance/bundle/resources/permissions/output.txt @@ -17,54 +17,25 @@ DIFF apps/current_can_manage/out.requests.destroy.direct.json + "path": "/api/2.0/permissions/apps/foo" + } +] -DIFF apps/other_can_manage/out.requests.deploy.direct.json ---- apps/other_can_manage/out.requests.deploy.direct.json -+++ apps/other_can_manage/out.requests.deploy.terraform.json -@@ -22,18 +22,5 @@ - }, - "method": "PUT", - "path": "/api/2.0/permissions/apps/foo" -- }, -- { -- "body": { -- "resource_id": "/apps/foo", -- "sequence_id": "0", -- "state": "{/"state/":{/"object_id/":/"/apps/foo/",/"__embed__/":[{/"level/":/"CAN_USE/",/"user_name/":/"viewer@example.com/"},{/"level/":/"CAN_MANAGE/",/"group_name/":/"data-team/"},{/"level/":/"CAN_MANAGE/",/"service_principal_name/":/"[UUID]/"},{/"level/":/"CAN_MANAGE/",/"user_name/":/"[USERNAME]/"}]},/"depends_on/":[{/"node/":/"resources.apps.foo/",/"label/":/"${resources.apps.foo.id}/"}]}", -- "status": "OPERATION_STATUS_SUCCEEDED" -- }, -- "method": "PATCH", -- "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/apps.foo.permissions", -- "q": { -- "update_mask": "state,error_message,resource_id,status" -- } - } - ] +MATCH apps/other_can_manage/out.requests.deploy.direct.json DIFF apps/other_can_manage/out.requests.destroy.direct.json --- apps/other_can_manage/out.requests.destroy.direct.json +++ apps/other_can_manage/out.requests.destroy.terraform.json -@@ -1,14 +1,14 @@ - [ - { - "body": { -- "resource_id": "/apps/foo", -- "sequence_id": "0", -- "status": "OPERATION_STATUS_SUCCEEDED" +@@ -1 +1,14 @@ +-[]+[ ++ { ++ "body": { + "access_control_list": [ + { + "permission_level": "CAN_MANAGE", + "user_name": "[USERNAME]" + } + ] - }, -- "method": "PATCH", -- "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations/apps.foo.permissions", -- "q": { -- "update_mask": "state,error_message,resource_id,status" -- } ++ }, + "method": "PUT", + "path": "/api/2.0/permissions/apps/foo" - } - ] ++ } ++] MATCH clusters/current_can_manage/out.requests.deploy.direct.json DIFF clusters/current_can_manage/out.requests.destroy.direct.json --- clusters/current_can_manage/out.requests.destroy.direct.json @@ -84,48 +55,7 @@ DIFF clusters/current_can_manage/out.requests.destroy.direct.json + "path": "/api/2.0/permissions/clusters/[UUID]" + } +] -DIFF clusters/target/out.requests.direct.json ---- clusters/target/out.requests.direct.json -+++ clusters/target/out.requests.terraform.json -@@ -12,19 +12,6 @@ - }, - "method": "POST", - "path": "/api/2.1/clusters/create" -- }, -- { -- "body": { -- "resource_id": "[UUID]", -- "sequence_id": "0", -- "state": "{/"state/":{/"autotermination_minutes/":60,/"cluster_name/":/"[dev [USERNAME]] test-cluster/",/"custom_tags/":{/"dev/":/"[USERNAME]/"},/"node_type_id/":/"[NODE_TYPE_ID]/",/"num_workers/":1,/"spark_version/":/"15.4.x-scala2.12/"}}", -- "status": "OPERATION_STATUS_SUCCEEDED" -- }, -- "method": "PATCH", -- "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/clusters.cluster1", -- "q": { -- "update_mask": "state,error_message,resource_id,status" -- } - }, - { - "body": { -@@ -49,18 +36,5 @@ - }, - "method": "PUT", - "path": "/api/2.0/permissions/clusters/[UUID]" -- }, -- { -- "body": { -- "resource_id": "/clusters/[UUID]", -- "sequence_id": "0", -- "state": "{/"state/":{/"object_id/":/"/clusters/[UUID]/",/"__embed__/":[{/"level/":/"CAN_ATTACH_TO/",/"service_principal_name/":/"[UUID]/"},{/"level/":/"CAN_RESTART/",/"service_principal_name/":/"[UUID]/"},{/"level/":/"CAN_MANAGE/",/"service_principal_name/":/"[UUID]/"},{/"level/":/"CAN_MANAGE/",/"user_name/":/"[USERNAME]/"}]},/"depends_on/":[{/"node/":/"resources.clusters.cluster1/",/"label/":/"${resources.clusters.cluster1.id}/"}]}", -- "status": "OPERATION_STATUS_SUCCEEDED" -- }, -- "method": "PATCH", -- "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/clusters.cluster1.permissions", -- "q": { -- "update_mask": "state,error_message,resource_id,status" -- } - } - ] +MATCH clusters/target/out.requests.direct.json MATCH dashboards/create/out.requests.deploy.direct.json DIFF dashboards/create/out.requests.destroy.direct.json --- dashboards/create/out.requests.destroy.direct.json @@ -138,54 +68,25 @@ DIFF dashboards/create/out.requests.destroy.direct.json + "path": "/api/2.0/permissions/dashboards/[FOO_ID]" + } +] -DIFF database_instances/current_can_manage/out.requests.deploy.direct.json ---- database_instances/current_can_manage/out.requests.deploy.direct.json -+++ database_instances/current_can_manage/out.requests.deploy.terraform.json -@@ -22,18 +22,5 @@ - }, - "method": "PUT", - "path": "/api/2.0/permissions/database-instances/test-db-instance" -- }, -- { -- "body": { -- "resource_id": "/database-instances/test-db-instance", -- "sequence_id": "0", -- "state": "{/"state/":{/"object_id/":/"/database-instances/test-db-instance/",/"__embed__/":[{/"level/":/"CAN_USE/",/"user_name/":/"viewer@example.com/"},{/"level/":/"CAN_MANAGE/",/"group_name/":/"data-team/"},{/"level/":/"CAN_MANAGE/",/"service_principal_name/":/"[UUID]/"},{/"level/":/"CAN_MANAGE/",/"user_name/":/"[USERNAME]/"}]},/"depends_on/":[{/"node/":/"resources.database_instances.foo/",/"label/":/"${resources.database_instances.foo.id}/"}]}", -- "status": "OPERATION_STATUS_SUCCEEDED" -- }, -- "method": "PATCH", -- "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/database_instances.foo.permissions", -- "q": { -- "update_mask": "state,error_message,resource_id,status" -- } - } - ] +MATCH database_instances/current_can_manage/out.requests.deploy.direct.json DIFF database_instances/current_can_manage/out.requests.destroy.direct.json --- database_instances/current_can_manage/out.requests.destroy.direct.json +++ database_instances/current_can_manage/out.requests.destroy.terraform.json -@@ -1,14 +1,14 @@ - [ - { - "body": { -- "resource_id": "/database-instances/test-db-instance", -- "sequence_id": "0", -- "status": "OPERATION_STATUS_SUCCEEDED" +@@ -1 +1,14 @@ +-[]+[ ++ { ++ "body": { + "access_control_list": [ + { + "permission_level": "CAN_MANAGE", + "user_name": "[USERNAME]" + } + ] - }, -- "method": "PATCH", -- "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations/database_instances.foo.permissions", -- "q": { -- "update_mask": "state,error_message,resource_id,status" -- } ++ }, + "method": "PUT", + "path": "/api/2.0/permissions/database-instances/test-db-instance" - } - ] ++ } ++] MATCH experiments/current_can_manage/out.requests.deploy.direct.json DIFF experiments/current_can_manage/out.requests.destroy.direct.json --- experiments/current_can_manage/out.requests.destroy.direct.json @@ -200,147 +101,64 @@ DIFF experiments/current_can_manage/out.requests.destroy.direct.json +] DIRECT_ONLY genie_spaces/current_can_manage/out.requests.deploy.direct.json DIRECT_ONLY genie_spaces/current_can_manage/out.requests.destroy.direct.json -DIFF jobs/current_can_manage/out.requests.deploy.direct.json ---- jobs/current_can_manage/out.requests.deploy.direct.json -+++ jobs/current_can_manage/out.requests.deploy.terraform.json -@@ -22,18 +22,5 @@ - }, - "method": "PUT", - "path": "/api/2.0/permissions/jobs/[NUMID]" -- }, -- { -- "body": { -- "resource_id": "/jobs/[NUMID]", -- "sequence_id": "0", -- "state": "{/"state/":{/"object_id/":/"/jobs/[NUMID]/",/"__embed__/":[{/"level/":/"CAN_VIEW/",/"user_name/":/"viewer@example.com/"},{/"level/":/"CAN_MANAGE/",/"group_name/":/"data-team/"},{/"level/":/"CAN_MANAGE/",/"service_principal_name/":/"[UUID]/"},{/"level/":/"IS_OWNER/",/"user_name/":/"[USERNAME]/"}]},/"depends_on/":[{/"node/":/"resources.jobs.foo/",/"label/":/"${resources.jobs.foo.id}/"}]}", -- "status": "OPERATION_STATUS_SUCCEEDED" -- }, -- "method": "PATCH", -- "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.foo.permissions", -- "q": { -- "update_mask": "state,error_message,resource_id,status" -- } - } - ] +MATCH jobs/current_can_manage/out.requests.deploy.direct.json DIFF jobs/current_can_manage/out.requests.destroy.direct.json --- jobs/current_can_manage/out.requests.destroy.direct.json +++ jobs/current_can_manage/out.requests.destroy.terraform.json -@@ -1,14 +1,14 @@ - [ - { - "body": { -- "resource_id": "/jobs/[NUMID]", -- "sequence_id": "0", -- "status": "OPERATION_STATUS_SUCCEEDED" +@@ -1 +1,14 @@ +-[]+[ ++ { ++ "body": { + "access_control_list": [ + { + "permission_level": "IS_OWNER", + "user_name": "[USERNAME]" + } + ] - }, -- "method": "PATCH", -- "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations/jobs.foo.permissions", -- "q": { -- "update_mask": "state,error_message,resource_id,status" -- } ++ }, + "method": "PUT", + "path": "/api/2.0/permissions/jobs/[NUMID]" - } - ] ++ } ++] DIFF jobs/current_can_manage_run/out.destroy.requests.direct.json --- jobs/current_can_manage_run/out.destroy.requests.direct.json +++ jobs/current_can_manage_run/out.destroy.requests.terraform.json -@@ -1,15 +1,15 @@ +@@ -1,4 +1,16 @@ [ - { - "body": { -- "resource_id": "/jobs/[NUMID]", -- "sequence_id": "0", -- "status": "OPERATION_STATUS_SUCCEEDED" ++ { ++ "body": { + "access_control_list": [ + { + "permission_level": "IS_OWNER", + "user_name": "[USERNAME]" + } + ] - }, -- "method": "PATCH", -- "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations/jobs.interim_gold_layer_job.permissions", -- "q": { -- "update_mask": "state,error_message,resource_id,status" -- } ++ }, + "method": "PUT", + "path": "/api/2.0/permissions/jobs/[NUMID]" - }, ++ }, { "body": { -@@ -17,17 +17,5 @@ - }, - "method": "POST", - "path": "/api/2.2/jobs/delete" -- }, -- { -- "body": { -- "resource_id": "[NUMID]", -- "sequence_id": "0", -- "status": "OPERATION_STATUS_SUCCEEDED" -- }, -- "method": "PATCH", -- "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations/jobs.interim_gold_layer_job", -- "q": { -- "update_mask": "state,error_message,resource_id,status" -- } - } - ] -DIFF jobs/current_is_owner/out.requests.deploy.direct.json ---- jobs/current_is_owner/out.requests.deploy.direct.json -+++ jobs/current_is_owner/out.requests.deploy.terraform.json -@@ -10,18 +10,5 @@ - }, - "method": "PUT", - "path": "/api/2.0/permissions/jobs/[NUMID]" -- }, -- { -- "body": { -- "resource_id": "/jobs/[NUMID]", -- "sequence_id": "0", -- "state": "{/"state/":{/"object_id/":/"/jobs/[NUMID]/",/"__embed__/":[{/"level/":/"IS_OWNER/",/"user_name/":/"[USERNAME]/"}]},/"depends_on/":[{/"node/":/"resources.jobs.foo/",/"label/":/"${resources.jobs.foo.id}/"}]}", -- "status": "OPERATION_STATUS_SUCCEEDED" -- }, -- "method": "PATCH", -- "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.foo.permissions", -- "q": { -- "update_mask": "state,error_message,resource_id,status" -- } - } - ] + "job_id": "[NUMID]" +EXACT jobs/current_is_owner/out.requests.deploy.direct.json DIFF jobs/current_is_owner/out.requests.destroy.direct.json --- jobs/current_is_owner/out.requests.destroy.direct.json +++ jobs/current_is_owner/out.requests.destroy.terraform.json -@@ -1,14 +1,14 @@ - [ - { - "body": { -- "resource_id": "/jobs/[NUMID]", -- "sequence_id": "0", -- "status": "OPERATION_STATUS_SUCCEEDED" +@@ -1 +1,14 @@ +-[]+[ ++ { ++ "body": { + "access_control_list": [ + { + "permission_level": "IS_OWNER", + "user_name": "[USERNAME]" + } + ] - }, -- "method": "PATCH", -- "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations/jobs.foo.permissions", -- "q": { -- "update_mask": "state,error_message,resource_id,status" -- } ++ }, + "method": "PUT", + "path": "/api/2.0/permissions/jobs/[NUMID]" - } - ] ++ } ++] DIFF jobs/delete_one/out.requests_destroy.direct.json --- jobs/delete_one/out.requests_destroy.direct.json +++ jobs/delete_one/out.requests_destroy.terraform.json @@ -363,58 +181,29 @@ DIFF jobs/delete_one/out.requests_destroy.direct.json "job_id": "[JOB_WITH_PERMISSIONS_ID]" EXACT jobs/empty_list/out.requests.deploy.direct.json EXACT jobs/empty_list/out.requests.destroy.direct.json -DIFF jobs/other_can_manage/out.requests.deploy.direct.json ---- jobs/other_can_manage/out.requests.deploy.direct.json -+++ jobs/other_can_manage/out.requests.deploy.terraform.json -@@ -22,18 +22,5 @@ - }, - "method": "PUT", - "path": "/api/2.0/permissions/jobs/[NUMID]" -- }, -- { -- "body": { -- "resource_id": "/jobs/[NUMID]", -- "sequence_id": "0", -- "state": "{/"state/":{/"object_id/":/"/jobs/[NUMID]/",/"__embed__/":[{/"level/":/"CAN_VIEW/",/"user_name/":/"viewer@example.com/"},{/"level/":/"CAN_MANAGE/",/"group_name/":/"data-team/"},{/"level/":/"CAN_MANAGE/",/"service_principal_name/":/"[UUID]/"},{/"level/":/"IS_OWNER/",/"user_name/":/"[USERNAME]/"}]},/"depends_on/":[{/"node/":/"resources.jobs.foo/",/"label/":/"${resources.jobs.foo.id}/"}]}", -- "status": "OPERATION_STATUS_SUCCEEDED" -- }, -- "method": "PATCH", -- "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.foo.permissions", -- "q": { -- "update_mask": "state,error_message,resource_id,status" -- } - } - ] +MATCH jobs/other_can_manage/out.requests.deploy.direct.json DIFF jobs/other_can_manage/out.requests.destroy.direct.json --- jobs/other_can_manage/out.requests.destroy.direct.json +++ jobs/other_can_manage/out.requests.destroy.terraform.json -@@ -1,14 +1,14 @@ - [ - { - "body": { -- "resource_id": "/jobs/[NUMID]", -- "sequence_id": "0", -- "status": "OPERATION_STATUS_SUCCEEDED" +@@ -1 +1,14 @@ +-[]+[ ++ { ++ "body": { + "access_control_list": [ + { + "permission_level": "IS_OWNER", + "user_name": "[USERNAME]" + } + ] - }, -- "method": "PATCH", -- "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations/jobs.foo.permissions", -- "q": { -- "update_mask": "state,error_message,resource_id,status" -- } ++ }, + "method": "PUT", + "path": "/api/2.0/permissions/jobs/[NUMID]" - } - ] ++ } ++] DIFF jobs/other_can_manage_run/out.destroy.requests.direct.json --- jobs/other_can_manage_run/out.destroy.requests.direct.json +++ jobs/other_can_manage_run/out.destroy.requests.terraform.json -@@ -1,33 +1,21 @@ +@@ -1,4 +1,16 @@ [ + { + "body": { @@ -431,83 +220,25 @@ DIFF jobs/other_can_manage_run/out.destroy.requests.direct.json { "body": { "job_id": "[NUMID]" - }, - "method": "POST", - "path": "/api/2.2/jobs/delete" -- }, -- { -- "body": { -- "resource_id": "/jobs/[NUMID]", -- "sequence_id": "0", -- "status": "OPERATION_STATUS_SUCCEEDED" -- }, -- "method": "PATCH", -- "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations/jobs.interim_gold_layer_job.permissions", -- "q": { -- "update_mask": "state,error_message,resource_id,status" -- } -- }, -- { -- "body": { -- "resource_id": "[NUMID]", -- "sequence_id": "0", -- "status": "OPERATION_STATUS_SUCCEEDED" -- }, -- "method": "PATCH", -- "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations/jobs.interim_gold_layer_job", -- "q": { -- "update_mask": "state,error_message,resource_id,status" -- } - } - ] -DIFF jobs/other_is_owner/out.requests.deploy.direct.json ---- jobs/other_is_owner/out.requests.deploy.direct.json -+++ jobs/other_is_owner/out.requests.deploy.terraform.json -@@ -14,18 +14,5 @@ - }, - "method": "PUT", - "path": "/api/2.0/permissions/jobs/[NUMID]" -- }, -- { -- "body": { -- "resource_id": "/jobs/[NUMID]", -- "sequence_id": "0", -- "state": "{/"state/":{/"object_id/":/"/jobs/[NUMID]/",/"__embed__/":[{/"level/":/"IS_OWNER/",/"user_name/":/"other_user@databricks.com/"},{/"level/":/"CAN_MANAGE/",/"user_name/":/"[USERNAME]/"}]},/"depends_on/":[{/"node/":/"resources.jobs.foo/",/"label/":/"${resources.jobs.foo.id}/"}]}", -- "status": "OPERATION_STATUS_SUCCEEDED" -- }, -- "method": "PATCH", -- "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.foo.permissions", -- "q": { -- "update_mask": "state,error_message,resource_id,status" -- } - } - ] +EXACT jobs/other_is_owner/out.requests.deploy.direct.json DIFF jobs/other_is_owner/out.requests.destroy.direct.json --- jobs/other_is_owner/out.requests.destroy.direct.json +++ jobs/other_is_owner/out.requests.destroy.terraform.json -@@ -1,14 +1,14 @@ - [ - { - "body": { -- "resource_id": "/jobs/[NUMID]", -- "sequence_id": "0", -- "status": "OPERATION_STATUS_SUCCEEDED" +@@ -1 +1,14 @@ +-[]+[ ++ { ++ "body": { + "access_control_list": [ + { + "permission_level": "IS_OWNER", + "user_name": "[USERNAME]" + } + ] - }, -- "method": "PATCH", -- "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations/jobs.foo.permissions", -- "q": { -- "update_mask": "state,error_message,resource_id,status" -- } ++ }, + "method": "PUT", + "path": "/api/2.0/permissions/jobs/[NUMID]" - } - ] ++ } ++] DIFF jobs/update/out.requests_delete_all.direct.json --- jobs/update/out.requests_delete_all.direct.json +++ jobs/update/out.requests_delete_all.terraform.json @@ -545,54 +276,25 @@ DIFF jobs/update/out.requests_set_empty.direct.json + "path": "/api/2.0/permissions/jobs/[JOB_WITH_PERMISSIONS_ID]" + } +] -DIFF jobs/viewers/out.requests.deploy.direct.json ---- jobs/viewers/out.requests.deploy.direct.json -+++ jobs/viewers/out.requests.deploy.terraform.json -@@ -22,18 +22,5 @@ - }, - "method": "PUT", - "path": "/api/2.0/permissions/jobs/[NUMID]" -- }, -- { -- "body": { -- "resource_id": "/jobs/[NUMID]", -- "sequence_id": "0", -- "state": "{/"state/":{/"object_id/":/"/jobs/[NUMID]/",/"__embed__/":[{/"level/":/"CAN_VIEW/",/"user_name/":/"viewer@example.com/"},{/"level/":/"CAN_VIEW/",/"group_name/":/"data-team/"},{/"level/":/"CAN_VIEW/",/"service_principal_name/":/"[UUID]/"},{/"level/":/"IS_OWNER/",/"user_name/":/"[USERNAME]/"}]},/"depends_on/":[{/"node/":/"resources.jobs.foo/",/"label/":/"${resources.jobs.foo.id}/"}]}", -- "status": "OPERATION_STATUS_SUCCEEDED" -- }, -- "method": "PATCH", -- "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.foo.permissions", -- "q": { -- "update_mask": "state,error_message,resource_id,status" -- } - } - ] +MATCH jobs/viewers/out.requests.deploy.direct.json DIFF jobs/viewers/out.requests.destroy.direct.json --- jobs/viewers/out.requests.destroy.direct.json +++ jobs/viewers/out.requests.destroy.terraform.json -@@ -1,14 +1,14 @@ - [ - { - "body": { -- "resource_id": "/jobs/[NUMID]", -- "sequence_id": "0", -- "status": "OPERATION_STATUS_SUCCEEDED" +@@ -1 +1,14 @@ +-[]+[ ++ { ++ "body": { + "access_control_list": [ + { + "permission_level": "IS_OWNER", + "user_name": "[USERNAME]" + } + ] - }, -- "method": "PATCH", -- "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations/jobs.foo.permissions", -- "q": { -- "update_mask": "state,error_message,resource_id,status" -- } ++ }, + "method": "PUT", + "path": "/api/2.0/permissions/jobs/[NUMID]" - } - ] ++ } ++] MATCH models/current_can_manage/out.requests.deploy.direct.json DIFF models/current_can_manage/out.requests.destroy.direct.json --- models/current_can_manage/out.requests.destroy.direct.json @@ -612,210 +314,43 @@ DIFF models/current_can_manage/out.requests.destroy.direct.json + "path": "/api/2.0/permissions/registered-models/[FOO_MODEL_ID]" + } +] -DIFF pipelines/current_can_manage/out.requests.deploy.direct.json ---- pipelines/current_can_manage/out.requests.deploy.direct.json -+++ pipelines/current_can_manage/out.requests.deploy.terraform.json -@@ -22,18 +22,5 @@ - }, - "method": "PUT", - "path": "/api/2.0/permissions/pipelines/[UUID]" -- }, -- { -- "body": { -- "resource_id": "/pipelines/[UUID]", -- "sequence_id": "0", -- "state": "{/"state/":{/"object_id/":/"/pipelines/[UUID]/",/"__embed__/":[{/"level/":/"CAN_VIEW/",/"user_name/":/"viewer@example.com/"},{/"level/":/"CAN_MANAGE/",/"group_name/":/"data-team/"},{/"level/":/"CAN_MANAGE/",/"service_principal_name/":/"[UUID]/"},{/"level/":/"IS_OWNER/",/"user_name/":/"[USERNAME]/"}]},/"depends_on/":[{/"node/":/"resources.pipelines.foo/",/"label/":/"${resources.pipelines.foo.id}/"}]}", -- "status": "OPERATION_STATUS_SUCCEEDED" -- }, -- "method": "PATCH", -- "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/pipelines.foo.permissions", -- "q": { -- "update_mask": "state,error_message,resource_id,status" -- } - } - ] -DIFF pipelines/current_can_manage/out.requests.destroy.direct.json ---- pipelines/current_can_manage/out.requests.destroy.direct.json -+++ pipelines/current_can_manage/out.requests.destroy.terraform.json -@@ -1,14 +1 @@ --[ -- { -- "body": { -- "resource_id": "/pipelines/[UUID]", -- "sequence_id": "0", -- "status": "OPERATION_STATUS_SUCCEEDED" -- }, -- "method": "PATCH", -- "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations/pipelines.foo.permissions", -- "q": { -- "update_mask": "state,error_message,resource_id,status" -- } -- } --]+[] +MATCH pipelines/current_can_manage/out.requests.deploy.direct.json +EXACT pipelines/current_can_manage/out.requests.destroy.direct.json EXACT pipelines/current_is_owner/out.requests.deploy.direct.json EXACT pipelines/current_is_owner/out.requests.destroy.direct.json EXACT pipelines/empty_list/out.requests.deploy.direct.json EXACT pipelines/empty_list/out.requests.destroy.direct.json -DIFF pipelines/other_can_manage/out.requests.deploy.direct.json ---- pipelines/other_can_manage/out.requests.deploy.direct.json -+++ pipelines/other_can_manage/out.requests.deploy.terraform.json -@@ -22,18 +22,5 @@ - }, - "method": "PUT", - "path": "/api/2.0/permissions/pipelines/[UUID]" -- }, -- { -- "body": { -- "resource_id": "/pipelines/[UUID]", -- "sequence_id": "0", -- "state": "{/"state/":{/"object_id/":/"/pipelines/[UUID]/",/"__embed__/":[{/"level/":/"CAN_VIEW/",/"user_name/":/"viewer@example.com/"},{/"level/":/"CAN_MANAGE/",/"group_name/":/"data-team/"},{/"level/":/"CAN_MANAGE/",/"service_principal_name/":/"[UUID]/"},{/"level/":/"IS_OWNER/",/"user_name/":/"[USERNAME]/"}]},/"depends_on/":[{/"node/":/"resources.pipelines.foo/",/"label/":/"${resources.pipelines.foo.id}/"}]}", -- "status": "OPERATION_STATUS_SUCCEEDED" -- }, -- "method": "PATCH", -- "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/pipelines.foo.permissions", -- "q": { -- "update_mask": "state,error_message,resource_id,status" -- } - } - ] -DIFF pipelines/other_can_manage/out.requests.destroy.direct.json ---- pipelines/other_can_manage/out.requests.destroy.direct.json -+++ pipelines/other_can_manage/out.requests.destroy.terraform.json -@@ -1,14 +1 @@ --[ -- { -- "body": { -- "resource_id": "/pipelines/[UUID]", -- "sequence_id": "0", -- "status": "OPERATION_STATUS_SUCCEEDED" -- }, -- "method": "PATCH", -- "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations/pipelines.foo.permissions", -- "q": { -- "update_mask": "state,error_message,resource_id,status" -- } -- } --]+[] -DIFF pipelines/other_is_owner/out.requests.deploy.direct.json ---- pipelines/other_is_owner/out.requests.deploy.direct.json -+++ pipelines/other_is_owner/out.requests.deploy.terraform.json -@@ -14,18 +14,5 @@ - }, - "method": "PUT", - "path": "/api/2.0/permissions/pipelines/[UUID]" -- }, -- { -- "body": { -- "resource_id": "/pipelines/[UUID]", -- "sequence_id": "0", -- "state": "{/"state/":{/"object_id/":/"/pipelines/[UUID]/",/"__embed__/":[{/"level/":/"IS_OWNER/",/"user_name/":/"other_user@databricks.com/"},{/"level/":/"CAN_MANAGE/",/"user_name/":/"[USERNAME]/"}]},/"depends_on/":[{/"node/":/"resources.pipelines.foo/",/"label/":/"${resources.pipelines.foo.id}/"}]}", -- "status": "OPERATION_STATUS_SUCCEEDED" -- }, -- "method": "PATCH", -- "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/pipelines.foo.permissions", -- "q": { -- "update_mask": "state,error_message,resource_id,status" -- } - } - ] -DIFF pipelines/other_is_owner/out.requests.destroy.direct.json ---- pipelines/other_is_owner/out.requests.destroy.direct.json -+++ pipelines/other_is_owner/out.requests.destroy.terraform.json -@@ -1,14 +1 @@ --[ -- { -- "body": { -- "resource_id": "/pipelines/[UUID]", -- "sequence_id": "0", -- "status": "OPERATION_STATUS_SUCCEEDED" -- }, -- "method": "PATCH", -- "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations/pipelines.foo.permissions", -- "q": { -- "update_mask": "state,error_message,resource_id,status" -- } -- } --]+[] -DIFF postgres_projects/current_can_manage/out.requests.deploy.direct.json ---- postgres_projects/current_can_manage/out.requests.deploy.direct.json -+++ postgres_projects/current_can_manage/out.requests.deploy.terraform.json -@@ -22,18 +22,5 @@ - }, - "method": "PUT", - "path": "/api/2.0/permissions/database-projects/test-project" -- }, -- { -- "body": { -- "resource_id": "/database-projects/test-project", -- "sequence_id": "0", -- "state": "{/"state/":{/"object_id/":/"/database-projects/test-project/",/"__embed__/":[{/"level/":/"CAN_USE/",/"user_name/":/"viewer@example.com/"},{/"level/":/"CAN_MANAGE/",/"group_name/":/"data-team/"},{/"level/":/"CAN_MANAGE/",/"service_principal_name/":/"[UUID]/"},{/"level/":/"CAN_MANAGE/",/"user_name/":/"[USERNAME]/"}]},/"depends_on/":[{/"node/":/"resources.postgres_projects.foo/",/"label/":/"${resources.postgres_projects.foo.project_id}/"}]}", -- "status": "OPERATION_STATUS_SUCCEEDED" -- }, -- "method": "PATCH", -- "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/postgres_projects.foo.permissions", -- "q": { -- "update_mask": "state,error_message,resource_id,status" -- } - } - ] +MATCH pipelines/other_can_manage/out.requests.deploy.direct.json +EXACT pipelines/other_can_manage/out.requests.destroy.direct.json +EXACT pipelines/other_is_owner/out.requests.deploy.direct.json +EXACT pipelines/other_is_owner/out.requests.destroy.direct.json +MATCH postgres_projects/current_can_manage/out.requests.deploy.direct.json DIFF postgres_projects/current_can_manage/out.requests.destroy.direct.json --- postgres_projects/current_can_manage/out.requests.destroy.direct.json +++ postgres_projects/current_can_manage/out.requests.destroy.terraform.json -@@ -1,14 +1,14 @@ - [ - { - "body": { -- "resource_id": "/database-projects/test-project", -- "sequence_id": "0", -- "status": "OPERATION_STATUS_SUCCEEDED" +@@ -1 +1,14 @@ +-[]+[ ++ { ++ "body": { + "access_control_list": [ + { + "permission_level": "CAN_MANAGE", + "user_name": "[USERNAME]" + } + ] - }, -- "method": "PATCH", -- "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations/postgres_projects.foo.permissions", -- "q": { -- "update_mask": "state,error_message,resource_id,status" -- } ++ }, + "method": "PUT", + "path": "/api/2.0/permissions/database-projects/test-project" - } - ] -DIFF sql_warehouses/current_can_manage/out.requests.deploy.direct.json ---- sql_warehouses/current_can_manage/out.requests.deploy.direct.json -+++ sql_warehouses/current_can_manage/out.requests.deploy.terraform.json -@@ -22,18 +22,5 @@ - }, - "method": "PUT", - "path": "/api/2.0/permissions/sql/warehouses/[UUID]" -- }, -- { -- "body": { -- "resource_id": "/sql/warehouses/[UUID]", -- "sequence_id": "0", -- "state": "{/"state/":{/"object_id/":/"/sql/warehouses/[UUID]/",/"__embed__/":[{/"level/":/"CAN_VIEW/",/"user_name/":/"viewer@example.com/"},{/"level/":/"CAN_MANAGE/",/"group_name/":/"data-team/"},{/"level/":/"CAN_MANAGE/",/"service_principal_name/":/"[UUID]/"},{/"level/":/"IS_OWNER/",/"user_name/":/"[USERNAME]/"}]},/"depends_on/":[{/"node/":/"resources.sql_warehouses.foo/",/"label/":/"${resources.sql_warehouses.foo.id}/"}]}", -- "status": "OPERATION_STATUS_SUCCEEDED" -- }, -- "method": "PATCH", -- "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/sql_warehouses.foo.permissions", -- "q": { -- "update_mask": "state,error_message,resource_id,status" -- } - } - ] ++ } ++] +MATCH sql_warehouses/current_can_manage/out.requests.deploy.direct.json DIFF sql_warehouses/current_can_manage/out.requests.destroy.direct.json --- sql_warehouses/current_can_manage/out.requests.destroy.direct.json +++ sql_warehouses/current_can_manage/out.requests.destroy.terraform.json -@@ -1,14 +1,18 @@ - [ - { - "body": { -- "resource_id": "/sql/warehouses/[UUID]", -- "sequence_id": "0", -- "status": "OPERATION_STATUS_SUCCEEDED" +@@ -1 +1,18 @@ +-[]+[ ++ { ++ "body": { + "access_control_list": [ + { + "permission_level": "CAN_MANAGE", @@ -826,16 +361,11 @@ DIFF sql_warehouses/current_can_manage/out.requests.destroy.direct.json + "user_name": "[USERNAME]" + } + ] - }, -- "method": "PATCH", -- "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations/sql_warehouses.foo.permissions", -- "q": { -- "update_mask": "state,error_message,resource_id,status" -- } ++ }, + "method": "PUT", + "path": "/api/2.0/permissions/sql/warehouses/[UUID]" - } - ] ++ } ++] EXACT target_permissions/out.requests_create.direct.json DIFF target_permissions/out.requests_delete.direct.json --- target_permissions/out.requests_delete.direct.json diff --git a/acceptance/bundle/resources/permissions/pipelines/current_can_manage/out.requests.deploy.direct.json b/acceptance/bundle/resources/permissions/pipelines/current_can_manage/out.requests.deploy.direct.json index 208fe9a91d4..42daa83bd86 100644 --- a/acceptance/bundle/resources/permissions/pipelines/current_can_manage/out.requests.deploy.direct.json +++ b/acceptance/bundle/resources/permissions/pipelines/current_can_manage/out.requests.deploy.direct.json @@ -22,16 +22,3 @@ ] } } -{ - "method": "PATCH", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/pipelines.foo.permissions", - "q": { - "update_mask": "state,error_message,resource_id,status" - }, - "body": { - "state": "{\"state\":{\"object_id\":\"/pipelines/[UUID]\",\"__embed__\":[{\"level\":\"CAN_VIEW\",\"user_name\":\"viewer@example.com\"},{\"level\":\"CAN_MANAGE\",\"group_name\":\"data-team\"},{\"level\":\"CAN_MANAGE\",\"service_principal_name\":\"[UUID]\"},{\"level\":\"IS_OWNER\",\"user_name\":\"[USERNAME]\"}]},\"depends_on\":[{\"node\":\"resources.pipelines.foo\",\"label\":\"${resources.pipelines.foo.id}\"}]}", - "resource_id": "/pipelines/[UUID]", - "status": "OPERATION_STATUS_SUCCEEDED", - "sequence_id": "0" - } -} diff --git a/acceptance/bundle/resources/permissions/pipelines/current_can_manage/out.requests.destroy.direct.json b/acceptance/bundle/resources/permissions/pipelines/current_can_manage/out.requests.destroy.direct.json index beda36a358c..e69de29bb2d 100644 --- a/acceptance/bundle/resources/permissions/pipelines/current_can_manage/out.requests.destroy.direct.json +++ b/acceptance/bundle/resources/permissions/pipelines/current_can_manage/out.requests.destroy.direct.json @@ -1,12 +0,0 @@ -{ - "method": "PATCH", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations/pipelines.foo.permissions", - "q": { - "update_mask": "state,error_message,resource_id,status" - }, - "body": { - "resource_id": "/pipelines/[UUID]", - "status": "OPERATION_STATUS_SUCCEEDED", - "sequence_id": "0" - } -} diff --git a/acceptance/bundle/resources/permissions/pipelines/other_can_manage/out.requests.deploy.direct.json b/acceptance/bundle/resources/permissions/pipelines/other_can_manage/out.requests.deploy.direct.json index 208fe9a91d4..42daa83bd86 100644 --- a/acceptance/bundle/resources/permissions/pipelines/other_can_manage/out.requests.deploy.direct.json +++ b/acceptance/bundle/resources/permissions/pipelines/other_can_manage/out.requests.deploy.direct.json @@ -22,16 +22,3 @@ ] } } -{ - "method": "PATCH", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/pipelines.foo.permissions", - "q": { - "update_mask": "state,error_message,resource_id,status" - }, - "body": { - "state": "{\"state\":{\"object_id\":\"/pipelines/[UUID]\",\"__embed__\":[{\"level\":\"CAN_VIEW\",\"user_name\":\"viewer@example.com\"},{\"level\":\"CAN_MANAGE\",\"group_name\":\"data-team\"},{\"level\":\"CAN_MANAGE\",\"service_principal_name\":\"[UUID]\"},{\"level\":\"IS_OWNER\",\"user_name\":\"[USERNAME]\"}]},\"depends_on\":[{\"node\":\"resources.pipelines.foo\",\"label\":\"${resources.pipelines.foo.id}\"}]}", - "resource_id": "/pipelines/[UUID]", - "status": "OPERATION_STATUS_SUCCEEDED", - "sequence_id": "0" - } -} diff --git a/acceptance/bundle/resources/permissions/pipelines/other_can_manage/out.requests.destroy.direct.json b/acceptance/bundle/resources/permissions/pipelines/other_can_manage/out.requests.destroy.direct.json index beda36a358c..e69de29bb2d 100644 --- a/acceptance/bundle/resources/permissions/pipelines/other_can_manage/out.requests.destroy.direct.json +++ b/acceptance/bundle/resources/permissions/pipelines/other_can_manage/out.requests.destroy.direct.json @@ -1,12 +0,0 @@ -{ - "method": "PATCH", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations/pipelines.foo.permissions", - "q": { - "update_mask": "state,error_message,resource_id,status" - }, - "body": { - "resource_id": "/pipelines/[UUID]", - "status": "OPERATION_STATUS_SUCCEEDED", - "sequence_id": "0" - } -} diff --git a/acceptance/bundle/resources/permissions/pipelines/other_is_owner/out.requests.deploy.direct.json b/acceptance/bundle/resources/permissions/pipelines/other_is_owner/out.requests.deploy.direct.json index 3c80374a684..8925d4f66d4 100644 --- a/acceptance/bundle/resources/permissions/pipelines/other_is_owner/out.requests.deploy.direct.json +++ b/acceptance/bundle/resources/permissions/pipelines/other_is_owner/out.requests.deploy.direct.json @@ -14,16 +14,3 @@ ] } } -{ - "method": "PATCH", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/pipelines.foo.permissions", - "q": { - "update_mask": "state,error_message,resource_id,status" - }, - "body": { - "state": "{\"state\":{\"object_id\":\"/pipelines/[UUID]\",\"__embed__\":[{\"level\":\"IS_OWNER\",\"user_name\":\"other_user@databricks.com\"},{\"level\":\"CAN_MANAGE\",\"user_name\":\"[USERNAME]\"}]},\"depends_on\":[{\"node\":\"resources.pipelines.foo\",\"label\":\"${resources.pipelines.foo.id}\"}]}", - "resource_id": "/pipelines/[UUID]", - "status": "OPERATION_STATUS_SUCCEEDED", - "sequence_id": "0" - } -} diff --git a/acceptance/bundle/resources/permissions/pipelines/other_is_owner/out.requests.destroy.direct.json b/acceptance/bundle/resources/permissions/pipelines/other_is_owner/out.requests.destroy.direct.json index beda36a358c..e69de29bb2d 100644 --- a/acceptance/bundle/resources/permissions/pipelines/other_is_owner/out.requests.destroy.direct.json +++ b/acceptance/bundle/resources/permissions/pipelines/other_is_owner/out.requests.destroy.direct.json @@ -1,12 +0,0 @@ -{ - "method": "PATCH", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations/pipelines.foo.permissions", - "q": { - "update_mask": "state,error_message,resource_id,status" - }, - "body": { - "resource_id": "/pipelines/[UUID]", - "status": "OPERATION_STATUS_SUCCEEDED", - "sequence_id": "0" - } -} diff --git a/acceptance/bundle/resources/permissions/postgres_projects/current_can_manage/out.requests.deploy.direct.json b/acceptance/bundle/resources/permissions/postgres_projects/current_can_manage/out.requests.deploy.direct.json index 33472c67bf6..673b537f7f4 100644 --- a/acceptance/bundle/resources/permissions/postgres_projects/current_can_manage/out.requests.deploy.direct.json +++ b/acceptance/bundle/resources/permissions/postgres_projects/current_can_manage/out.requests.deploy.direct.json @@ -22,16 +22,3 @@ ] } } -{ - "method": "PATCH", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/postgres_projects.foo.permissions", - "q": { - "update_mask": "state,error_message,resource_id,status" - }, - "body": { - "state": "{\"state\":{\"object_id\":\"/database-projects/test-project\",\"__embed__\":[{\"level\":\"CAN_USE\",\"user_name\":\"viewer@example.com\"},{\"level\":\"CAN_MANAGE\",\"group_name\":\"data-team\"},{\"level\":\"CAN_MANAGE\",\"service_principal_name\":\"[UUID]\"},{\"level\":\"CAN_MANAGE\",\"user_name\":\"[USERNAME]\"}]},\"depends_on\":[{\"node\":\"resources.postgres_projects.foo\",\"label\":\"${resources.postgres_projects.foo.project_id}\"}]}", - "resource_id": "/database-projects/test-project", - "status": "OPERATION_STATUS_SUCCEEDED", - "sequence_id": "0" - } -} diff --git a/acceptance/bundle/resources/permissions/postgres_projects/current_can_manage/out.requests.destroy.direct.json b/acceptance/bundle/resources/permissions/postgres_projects/current_can_manage/out.requests.destroy.direct.json index 4a85a73dced..e69de29bb2d 100644 --- a/acceptance/bundle/resources/permissions/postgres_projects/current_can_manage/out.requests.destroy.direct.json +++ b/acceptance/bundle/resources/permissions/postgres_projects/current_can_manage/out.requests.destroy.direct.json @@ -1,12 +0,0 @@ -{ - "method": "PATCH", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations/postgres_projects.foo.permissions", - "q": { - "update_mask": "state,error_message,resource_id,status" - }, - "body": { - "resource_id": "/database-projects/test-project", - "status": "OPERATION_STATUS_SUCCEEDED", - "sequence_id": "0" - } -} diff --git a/acceptance/bundle/resources/permissions/sql_warehouses/current_can_manage/out.requests.deploy.direct.json b/acceptance/bundle/resources/permissions/sql_warehouses/current_can_manage/out.requests.deploy.direct.json index ab940b519df..2ef440a5941 100644 --- a/acceptance/bundle/resources/permissions/sql_warehouses/current_can_manage/out.requests.deploy.direct.json +++ b/acceptance/bundle/resources/permissions/sql_warehouses/current_can_manage/out.requests.deploy.direct.json @@ -22,16 +22,3 @@ ] } } -{ - "method": "PATCH", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/sql_warehouses.foo.permissions", - "q": { - "update_mask": "state,error_message,resource_id,status" - }, - "body": { - "state": "{\"state\":{\"object_id\":\"/sql/warehouses/[UUID]\",\"__embed__\":[{\"level\":\"CAN_VIEW\",\"user_name\":\"viewer@example.com\"},{\"level\":\"CAN_MANAGE\",\"group_name\":\"data-team\"},{\"level\":\"CAN_MANAGE\",\"service_principal_name\":\"[UUID]\"},{\"level\":\"IS_OWNER\",\"user_name\":\"[USERNAME]\"}]},\"depends_on\":[{\"node\":\"resources.sql_warehouses.foo\",\"label\":\"${resources.sql_warehouses.foo.id}\"}]}", - "resource_id": "/sql/warehouses/[UUID]", - "status": "OPERATION_STATUS_SUCCEEDED", - "sequence_id": "0" - } -} diff --git a/acceptance/bundle/resources/permissions/sql_warehouses/current_can_manage/out.requests.destroy.direct.json b/acceptance/bundle/resources/permissions/sql_warehouses/current_can_manage/out.requests.destroy.direct.json index f95b0b7057b..e69de29bb2d 100644 --- a/acceptance/bundle/resources/permissions/sql_warehouses/current_can_manage/out.requests.destroy.direct.json +++ b/acceptance/bundle/resources/permissions/sql_warehouses/current_can_manage/out.requests.destroy.direct.json @@ -1,12 +0,0 @@ -{ - "method": "PATCH", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations/sql_warehouses.foo.permissions", - "q": { - "update_mask": "state,error_message,resource_id,status" - }, - "body": { - "resource_id": "/sql/warehouses/[UUID]", - "status": "OPERATION_STATUS_SUCCEEDED", - "sequence_id": "0" - } -} diff --git a/acceptance/bundle/resources/permissions/vector_search_endpoints/current_can_manage/out.requests.deploy.direct.json b/acceptance/bundle/resources/permissions/vector_search_endpoints/current_can_manage/out.requests.deploy.direct.json index c2fbf21ae83..9118a4da780 100644 --- a/acceptance/bundle/resources/permissions/vector_search_endpoints/current_can_manage/out.requests.deploy.direct.json +++ b/acceptance/bundle/resources/permissions/vector_search_endpoints/current_can_manage/out.requests.deploy.direct.json @@ -22,16 +22,3 @@ ] } } -{ - "method": "PATCH", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/vector_search_endpoints.foo.permissions", - "q": { - "update_mask": "state,error_message,resource_id,status" - }, - "body": { - "state": "{\"state\":{\"object_id\":\"/vector-search-endpoints/[UUID]\",\"__embed__\":[{\"level\":\"CAN_USE\",\"user_name\":\"viewer@example.com\"},{\"level\":\"CAN_MANAGE\",\"group_name\":\"data-team\"},{\"level\":\"CAN_MANAGE\",\"service_principal_name\":\"[UUID]\"},{\"level\":\"CAN_MANAGE\",\"user_name\":\"[USERNAME]\"}]},\"depends_on\":[{\"node\":\"resources.vector_search_endpoints.foo\",\"label\":\"${resources.vector_search_endpoints.foo.endpoint_uuid}\"}]}", - "resource_id": "/vector-search-endpoints/[UUID]", - "status": "OPERATION_STATUS_SUCCEEDED", - "sequence_id": "0" - } -} diff --git a/acceptance/bundle/resources/permissions/vector_search_endpoints/current_can_manage/out.requests.destroy.direct.json b/acceptance/bundle/resources/permissions/vector_search_endpoints/current_can_manage/out.requests.destroy.direct.json index cd017d6b986..84c87416aa2 100644 --- a/acceptance/bundle/resources/permissions/vector_search_endpoints/current_can_manage/out.requests.destroy.direct.json +++ b/acceptance/bundle/resources/permissions/vector_search_endpoints/current_can_manage/out.requests.destroy.direct.json @@ -2,15 +2,3 @@ "method": "DELETE", "path": "/api/2.0/vector-search/endpoints/vs-permissions-endpoint" } -{ - "method": "PATCH", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations/vector_search_endpoints.foo.permissions", - "q": { - "update_mask": "state,error_message,resource_id,status" - }, - "body": { - "resource_id": "/vector-search-endpoints/[UUID]", - "status": "OPERATION_STATUS_SUCCEEDED", - "sequence_id": "0" - } -} diff --git a/acceptance/bundle/resources/pipelines/allow-duplicate-names/output.txt b/acceptance/bundle/resources/pipelines/allow-duplicate-names/output.txt index 6f57e516c04..4ddbabbe7ed 100644 --- a/acceptance/bundle/resources/pipelines/allow-duplicate-names/output.txt +++ b/acceptance/bundle/resources/pipelines/allow-duplicate-names/output.txt @@ -41,19 +41,6 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "name": "test-pipeline-same-name-[UNIQUE_NAME]" } } -{ - "method": "PATCH", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/pipelines.pipeline_one", - "q": { - "update_mask": "state,error_message,resource_id,status" - }, - "body": { - "state": "{\"state\":{\"allow_duplicate_names\":true,\"channel\":\"CURRENT\",\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/acc-bundle-deploy-pipeline-duplicate-names-[UNIQUE_NAME]/default/state/metadata.json\",\"version_id\":\"1\"},\"edition\":\"ADVANCED\",\"libraries\":[{\"file\":{\"path\":\"/Workspace/Users/[USERNAME]/.bundle/acc-bundle-deploy-pipeline-duplicate-names-[UNIQUE_NAME]/default/files/foo.py\"}}],\"name\":\"test-pipeline-same-name-[UNIQUE_NAME]\"}}", - "resource_id": "[UUID]", - "status": "OPERATION_STATUS_SUCCEEDED", - "sequence_id": "0" - } -} >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: diff --git a/acceptance/bundle/run_as/job_default/output.txt b/acceptance/bundle/run_as/job_default/output.txt index ec9405720d5..21613bc205a 100644 --- a/acceptance/bundle/run_as/job_default/output.txt +++ b/acceptance/bundle/run_as/job_default/output.txt @@ -40,19 +40,6 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged ] } } -{ - "method": "PATCH", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.job_with_run_as", - "q": { - "update_mask": "state,error_message,resource_id,status" - }, - "body": { - "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/run_as_job_default_[UNIQUE_NAME]/default/state/metadata.json\",\"version_id\":\"1\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"Untitled\",\"queue\":{\"enabled\":true},\"run_as\":{\"user_name\":\"deco-test-user@databricks.com\"},\"tasks\":[{\"new_cluster\":{\"node_type_id\":\"[NODE_TYPE_ID]\",\"num_workers\":1,\"spark_version\":\"13.3.x-snapshot-scala2.12\"},\"notebook_task\":{\"notebook_path\":\"/Workspace/Users/[USERNAME]/.bundle/run_as_job_default_[UNIQUE_NAME]/default/files/test\"},\"task_key\":\"task_one\"}]}}", - "resource_id": "[NUMID]", - "status": "OPERATION_STATUS_SUCCEEDED", - "sequence_id": "0" - } -} >>> [CLI] jobs get [NUMID] { @@ -105,19 +92,6 @@ Resources: 0 created, 1 changed, 0 deleted, 0 unchanged } } } -{ - "method": "PATCH", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations/jobs.job_with_run_as", - "q": { - "update_mask": "state,error_message,resource_id,status" - }, - "body": { - "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/run_as_job_default_[UNIQUE_NAME]/default/state/metadata.json\",\"version_id\":\"2\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"Untitled\",\"queue\":{\"enabled\":true},\"tasks\":[{\"new_cluster\":{\"node_type_id\":\"[NODE_TYPE_ID]\",\"num_workers\":1,\"spark_version\":\"13.3.x-snapshot-scala2.12\"},\"notebook_task\":{\"notebook_path\":\"/Workspace/Users/[USERNAME]/.bundle/run_as_job_default_[UNIQUE_NAME]/default/files/test\"},\"task_key\":\"task_one\"}]}}", - "resource_id": "[NUMID]", - "status": "OPERATION_STATUS_SUCCEEDED", - "sequence_id": "0" - } -} >>> [CLI] jobs get [NUMID] { diff --git a/acceptance/bundle/run_as/pipelines/_script b/acceptance/bundle/run_as/pipelines/_script index a4ae022ac22..293600ded7d 100644 --- a/acceptance/bundle/run_as/pipelines/_script +++ b/acceptance/bundle/run_as/pipelines/_script @@ -1,5 +1,5 @@ print_requests() { - jq --sort-keys 'select(.method != "GET" and (.path | contains("/pipelines")))' < out.requests.txt | nostamp + jq --sort-keys 'select(.method != "GET" and (.path | contains("/pipelines")) and (.path | contains("/api/2.0/bundle") | not))' < out.requests.txt | nostamp rm out.requests.txt } diff --git a/libs/testserver/bundle.go b/libs/testserver/bundle.go index ef60f0bd129..f9f39dee1bf 100644 --- a/libs/testserver/bundle.go +++ b/libs/testserver/bundle.go @@ -416,20 +416,29 @@ func (s *FakeWorkspace) UpdateOperation(req Request, deploymentID, versionID, re return Response{StatusCode: 500, Body: map[string]string{"message": err.Error()}} } - // Mirror onto the resource set the same way CreateOperation does, so the read - // path reflects the newest write. - if existing.State == "" { - delete(d.resources, resourceKey) - } else { - d.resources[resourceKey] = bundledeployments.Resource{ - Name: "deployments/" + deploymentID + "/resources/" + resourceKey, - ResourceKey: resourceKey, - ResourceId: existing.ResourceId, - ResourceType: existing.ResourceType, - LastActionType: existing.ActionType, - LastVersionId: versionID, - State: existing.State, + // Only an update that names state moves the resource: naming it with no value clears + // it and removes the resource, and an update that leaves it out - a failure reporting + // its outcome - must not disturb what the deployment already holds. Every version + // stages its operations without state, so re-deriving this from the operation + // regardless of the mask would drop a resource whose deploy failed before writing. + if update["state"] { + if existing.State == "" { + delete(d.resources, resourceKey) + } else { + d.resources[resourceKey] = bundledeployments.Resource{ + Name: "deployments/" + deploymentID + "/resources/" + resourceKey, + ResourceKey: resourceKey, + ResourceId: existing.ResourceId, + ResourceType: existing.ResourceType, + LastActionType: existing.ActionType, + LastVersionId: versionID, + State: existing.State, + } } + } else if resource, projected := d.resources[resourceKey]; projected && update["resource_id"] { + // resource_id is mirrored too, for a resource the deployment still holds. + resource.ResourceId = existing.ResourceId + d.resources[resourceKey] = resource } return Response{Body: body} From a788ef8230ac2c13f2945685286f9597c451ae00 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 18 Aug 2026 16:58:22 +0000 Subject: [PATCH 106/125] acceptance: format print_requests.py the way ruff wants Co-authored-by: Isaac --- acceptance/bin/print_requests.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/acceptance/bin/print_requests.py b/acceptance/bin/print_requests.py index aec82c56bc3..c151845fc40 100755 --- a/acceptance/bin/print_requests.py +++ b/acceptance/bin/print_requests.py @@ -136,7 +136,9 @@ def read_json_many(s): DMS_PATH = "/api/2.0/bundle" -def filter_requests(requests, path_filters, include_get, should_sort, unique=False, method_filter=None, include_dms=False): +def filter_requests( + requests, path_filters, include_get, should_sort, unique=False, method_filter=None, include_dms=False +): """Filter requests based on method and path filters.""" positive_filters = [] negative_filters = [] @@ -277,7 +279,9 @@ def main(): return requests = read_json_many(data) - filtered_requests = filter_requests(requests, args.path_filters, args.get, args.sort, args.unique, args.method, args.dms) + filtered_requests = filter_requests( + requests, args.path_filters, args.get, args.sort, args.unique, args.method, args.dms + ) for req in filtered_requests: body = req.get("body") From e52d5e94e64be34b0143dd984f555feb9a791fa3 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 18 Aug 2026 17:28:35 +0000 Subject: [PATCH 107/125] acceptance: strip the deployment stamp from tests that arrived with main whl_via_environment_key_extras prints a whole jobs/create body, which carries deployment_id and version_id in the recording run; its sibling whl_via_environment_key already pipes through nostamp. The two fetch-repository-info tests are new and need the recording variant in their out.test.toml. Co-authored-by: Isaac --- .../bundle/artifacts/whl_via_environment_key_extras/script | 2 +- .../debug/fetch-repository-info-repos-error/out.test.toml | 1 + acceptance/bundle/debug/fetch-repository-info/out.test.toml | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/acceptance/bundle/artifacts/whl_via_environment_key_extras/script b/acceptance/bundle/artifacts/whl_via_environment_key_extras/script index 426eb0b71c9..ccc0d9ed7f8 100644 --- a/acceptance/bundle/artifacts/whl_via_environment_key_extras/script +++ b/acceptance/bundle/artifacts/whl_via_environment_key_extras/script @@ -3,7 +3,7 @@ trace $CLI bundle deploy trace find.py --expect 1 whl title "Expecting the environments dependency to keep its [train] extras suffix" -trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body' out.requests.txt +trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body' out.requests.txt | nostamp title "Expecting 1 wheel to be uploaded under its bare filename (no extras)" trace jq .path < out.requests.txt | grep import | grep whl | sort diff --git a/acceptance/bundle/debug/fetch-repository-info-repos-error/out.test.toml b/acceptance/bundle/debug/fetch-repository-info-repos-error/out.test.toml index 0938e678987..2c6699da193 100644 --- a/acceptance/bundle/debug/fetch-repository-info-repos-error/out.test.toml +++ b/acceptance/bundle/debug/fetch-repository-info-repos-error/out.test.toml @@ -1,2 +1,3 @@ Cloud = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/debug/fetch-repository-info/out.test.toml b/acceptance/bundle/debug/fetch-repository-info/out.test.toml index c502b28221b..89d861bf4d0 100644 --- a/acceptance/bundle/debug/fetch-repository-info/out.test.toml +++ b/acceptance/bundle/debug/fetch-repository-info/out.test.toml @@ -1,2 +1,3 @@ Cloud = true EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] From c6982e34353d89815377724dbaf2237ba8292750 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 18 Aug 2026 20:31:06 +0000 Subject: [PATCH 108/125] bundle/direct: drop the recorded action type The version stages every operation with its action type, and an update never carries one, so a recorded operation had no reason to hold it and coalescing had no reason to pick one. The action is still validated where an operation is built: an unrecordable one has nothing staged to update. Co-authored-by: Isaac --- bundle/direct/oprecorder.go | 12 ++++-------- bundle/direct/opsink.go | 4 ---- bundle/direct/opsink_test.go | 18 ++---------------- 3 files changed, 6 insertions(+), 28 deletions(-) diff --git a/bundle/direct/oprecorder.go b/bundle/direct/oprecorder.go index 8222d0326f1..a6e46ae3a3a 100644 --- a/bundle/direct/oprecorder.go +++ b/bundle/direct/oprecorder.go @@ -37,7 +37,6 @@ const stagedSequenceID = "0" // the apply worker, not in the uploader, so a malformed state fails the resource that // produced it rather than the drain at the end of apply. type recordedOperation struct { - action bundledeployments.OperationActionType resourceID string status bundledeployments.OperationStatus @@ -67,8 +66,9 @@ var failedKeepingState = []string{"error_message", "status"} // RecordedState envelope the state DB just persisted, and nil for a delete, where // the resource is gone. It errors when the state exceeds maxOperationStateSize. func newStateOperation(info dstate.OperationInfo, resourceID string, state json.RawMessage) (recordedOperation, error) { - actionType, err := DeployActionToSDK(info.Action) - if err != nil { + // The action is not recorded - the version stages it - but an unrecordable one means + // there is no operation to update, so fail the resource rather than the whole drain. + if _, err := DeployActionToSDK(info.Action); err != nil { return recordedOperation{}, err } @@ -82,7 +82,6 @@ func newStateOperation(info dstate.OperationInfo, resourceID string, state json. } return recordedOperation{ - action: actionType, resourceID: resourceID, status: status, state: state, @@ -94,8 +93,7 @@ func newStateOperation(info dstate.OperationInfo, resourceID string, state json. // resource failed rather than leaving it pending. It carries no state: the version staged the // operation already, so a failure only ever narrows an existing record. func newFailedOperation(action deployplan.ActionType, resourceID string, cause error) (recordedOperation, error) { - actionType, err := DeployActionToSDK(action) - if err != nil { + if _, err := DeployActionToSDK(action); err != nil { return recordedOperation{}, err } @@ -116,7 +114,6 @@ func newFailedOperation(action deployplan.ActionType, resourceID string, cause e } return recordedOperation{ - action: actionType, resourceID: resourceID, status: bundledeployments.OperationStatusOperationStatusFailed, errorMessage: message, @@ -168,7 +165,6 @@ func (r *operationRecorder) upload(ctx context.Context, resourceKey string, op r dmsKey := strings.TrimPrefix(resourceKey, dstate.ResourceKeyPrefix) operation := bundledeployments.Operation{ - ActionType: op.action, ResourceId: op.resourceID, ResourceKey: dmsKey, Status: op.status, diff --git a/bundle/direct/opsink.go b/bundle/direct/opsink.go index 53ffe145632..48d2e95f687 100644 --- a/bundle/direct/opsink.go +++ b/bundle/direct/opsink.go @@ -120,11 +120,7 @@ func coalesce(older, newer recordedOperation) recordedOperation { merged.updateFields = unionFields(older.updateFields, newer.updateFields) if slices.Contains(newer.updateFields, "state") { - // Claiming state means this operation last acted on the resource, so its action_type is - // the one to record. One that claims none only reports an outcome, and the service - // would keep the earlier action anyway - action_type is fixed once the operation exists. merged.state = newer.state - merged.action = newer.action } if slices.Contains(newer.updateFields, "resource_id") { merged.resourceID = newer.resourceID diff --git a/bundle/direct/opsink_test.go b/bundle/direct/opsink_test.go index aaa2a6bbd8d..4124109cbb9 100644 --- a/bundle/direct/opsink_test.go +++ b/bundle/direct/opsink_test.go @@ -26,7 +26,6 @@ type fakeUploader struct { mu sync.Mutex uploads []string - actions map[string]bundledeployments.OperationActionType resourceIDs map[string]string statuses map[string]bundledeployments.OperationStatus errorMessages map[string]string @@ -42,13 +41,11 @@ func (f *fakeUploader) upload(ctx context.Context, resourceKey string, op record f.mu.Lock() f.uploads = append(f.uploads, resourceKey+"="+string(op.state)) - if f.actions == nil { - f.actions = map[string]bundledeployments.OperationActionType{} + if f.resourceIDs == nil { f.resourceIDs = map[string]string{} f.statuses = map[string]bundledeployments.OperationStatus{} f.errorMessages = map[string]string{} } - f.actions[resourceKey] = op.action f.resourceIDs[resourceKey] = op.resourceID f.statuses[resourceKey] = op.status f.errorMessages[resourceKey] = op.errorMessage @@ -63,12 +60,6 @@ func (f *fakeUploader) recorded() []string { return append([]string(nil), f.uploads...) } -func (f *fakeUploader) actionFor(resourceKey string) bundledeployments.OperationActionType { - f.mu.Lock() - defer f.mu.Unlock() - return f.actions[resourceKey] -} - func (f *fakeUploader) resourceIDFor(resourceKey string) string { f.mu.Lock() defer f.mu.Unlock() @@ -250,9 +241,6 @@ func TestOperationSinkCoalescedDeleteStillClearsState(t *testing.T) { `resources.jobs.busy={"state":{"name":"v1"}}`, `resources.jobs.foo=`, }, f.recorded()) - assert.Equal(t, - bundledeployments.OperationActionTypeOperationActionTypeDelete, - f.actionFor("resources.jobs.foo")) } func TestCoalesceLetsAWriteSupersedeAFailure(t *testing.T) { @@ -275,8 +263,7 @@ func TestCoalesceLetsAWriteSupersedeAFailure(t *testing.T) { func TestCoalesceKeepsTheWritesStateAndMask(t *testing.T) { // A failure claims only status and error_message, so the write's state, id and mask - // survive. The action is the write's too: an update that empties a resource records its - // write as a delete, and the service keeps whichever action created the operation. + // survive. write, err := newStateOperation(dstate.OperationInfo{Action: deployplan.Delete}, "id-new", nil) require.NoError(t, err) failed, err := newFailedOperation(deployplan.Update, "id-old", errors.New("boom")) @@ -289,7 +276,6 @@ func TestCoalesceKeepsTheWritesStateAndMask(t *testing.T) { assert.Equal(t, "id-new", got.resourceID) assert.Nil(t, got.state) assert.Equal(t, describesResource, got.updateFields) - assert.Equal(t, bundledeployments.OperationActionTypeOperationActionTypeDelete, got.action) } func TestOperationSinkRecordDuringUploadIsStillUploaded(t *testing.T) { From b15aaa6b7090e101b8625d3b3863fb9816c8c4aa Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 18 Aug 2026 21:20:00 +0000 Subject: [PATCH 109/125] libs/dms: own the whole DMS protocol Every call to the deployment metadata service now goes through one place, and what the CLI sends is described by types rather than strings: - a Client holds the generated calls plus the two the SDK cannot express, so bundle/direct no longer carries a second hand-rolled transport and the reason those two are hand-written lives in one file - ResourceKey is its own type, so the state key ("resources.jobs.foo") and the key DMS knows ("jobs.foo") cannot be swapped by accident - the update mask is a Fields bitset with one canonical rendering, so a typo is a compile error rather than INVALID_PARAMETER_VALUE, and coalescing is a field-wise merge on the payload itself - a Recording replaces the recorder: Prepare, Start, Finish, with a disabled implementation instead of a nil pointer, so the phases never nil-check it and the writer can only be obtained from a version that exists - the action type is gone from the write path entirely; the version stages it, and only the staging call maps a plan action to it - the bundle answers whether it records history, instead of three copies of the same expression, and the client is built once rather than twice doc.go states the contract the masks are built around, and a table test pins the projection rule against the fake service. Co-authored-by: Isaac --- bundle/bundle.go | 8 + .../mutator/initialize_deployment_history.go | 4 +- bundle/direct/apply.go | 18 +- bundle/direct/bind.go | 8 +- bundle/direct/bundle_apply.go | 9 +- bundle/direct/dstate/dms.go | 93 ++---- bundle/direct/dstate/dms_test.go | 25 +- bundle/direct/dstate/state.go | 30 +- bundle/direct/dstate/state_test.go | 43 ++- bundle/direct/opclient.go | 62 ---- bundle/direct/oprecorder.go | 230 ------------- bundle/direct/oprecorder_test.go | 313 ------------------ bundle/direct/opsink.go | 111 ++----- bundle/direct/opsink_test.go | 277 +++++++--------- bundle/direct/pkg.go | 9 +- bundle/migrate/build_state.go | 2 +- bundle/phases/deploy.go | 22 +- bundle/phases/destroy.go | 18 +- bundle/phases/dms.go | 88 +++-- bundle/phases/dms_test.go | 26 ++ cmd/bundle/utils/process.go | 10 +- libs/dms/client.go | 146 ++++++++ libs/dms/doc.go | 20 ++ libs/dms/fields.go | 50 +++ libs/dms/fields_test.go | 23 ++ libs/dms/key.go | 25 ++ libs/dms/key_test.go | 18 + libs/dms/operation.go | 111 +++++++ libs/dms/operation_test.go | 119 +++++++ libs/dms/{recorder.go => recording.go} | 296 +++++++---------- .../{recorder_test.go => recording_test.go} | 135 ++++---- libs/dms/resources.go | 40 +++ libs/dms/writer.go | 56 ++++ libs/dms/writer_test.go | 159 +++++++++ libs/testserver/bundle_test.go | 146 ++++++++ 35 files changed, 1477 insertions(+), 1273 deletions(-) delete mode 100644 bundle/direct/opclient.go delete mode 100644 bundle/direct/oprecorder.go delete mode 100644 bundle/direct/oprecorder_test.go create mode 100644 libs/dms/client.go create mode 100644 libs/dms/doc.go create mode 100644 libs/dms/fields.go create mode 100644 libs/dms/fields_test.go create mode 100644 libs/dms/key.go create mode 100644 libs/dms/key_test.go create mode 100644 libs/dms/operation.go create mode 100644 libs/dms/operation_test.go rename libs/dms/{recorder.go => recording.go} (51%) rename libs/dms/{recorder_test.go => recording_test.go} (65%) create mode 100644 libs/dms/resources.go create mode 100644 libs/dms/writer.go create mode 100644 libs/dms/writer_test.go create mode 100644 libs/testserver/bundle_test.go diff --git a/bundle/bundle.go b/bundle/bundle.go index bcceb752088..975a20fb09e 100644 --- a/bundle/bundle.go +++ b/bundle/bundle.go @@ -323,6 +323,14 @@ func (b *Bundle) WorkspaceClient(ctx context.Context) *databricks.WorkspaceClien return client } +// RecordsDeploymentHistory reports whether this bundle records deployment history with the +// deployment metadata service, from experimental.record_deployment_history or +// DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY. +func (b *Bundle) RecordsDeploymentHistory(ctx context.Context) bool { + configured := b.Config.Experimental != nil && b.Config.Experimental.RecordDeploymentHistory + return env.RecordsDeploymentHistory(ctx, configured) +} + // SetWorkpaceClient sets the workspace client for this bundle. // This is used to inject a mock client for testing. func (b *Bundle) SetWorkpaceClient(w *databricks.WorkspaceClient) { diff --git a/bundle/config/mutator/initialize_deployment_history.go b/bundle/config/mutator/initialize_deployment_history.go index 07d576c01f2..6e1e47d2f9e 100644 --- a/bundle/config/mutator/initialize_deployment_history.go +++ b/bundle/config/mutator/initialize_deployment_history.go @@ -5,7 +5,6 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" - "github.com/databricks/cli/bundle/env" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dms" "github.com/databricks/databricks-sdk-go/service/bundledeployments" @@ -24,8 +23,7 @@ func (m *initializeDeploymentHistory) Name() string { } func (m *initializeDeploymentHistory) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { - configured := b.Config.Experimental != nil && b.Config.Experimental.RecordDeploymentHistory - if !env.RecordsDeploymentHistory(ctx, configured) { + if !b.RecordsDeploymentHistory(ctx) { return nil } diff --git a/bundle/direct/apply.go b/bundle/direct/apply.go index 9c4c0292b35..f4ce99a08bf 100644 --- a/bundle/direct/apply.go +++ b/bundle/direct/apply.go @@ -78,7 +78,7 @@ func (d *DeploymentUnit) Create(ctx context.Context, db *dstate.DeploymentState, return err } - err = d.saveState(ctx, db, newID, newState, d.DependsOn, dstate.OperationInfo{Action: action}) + err = d.saveState(ctx, db, newID, newState, d.DependsOn) if err != nil { return fmt.Errorf("saving state after creating id=%s: %w", newID, err) } @@ -122,7 +122,7 @@ func (d *DeploymentUnit) Recreate(ctx context.Context, db *dstate.DeploymentStat // // Recorded as a recreate rather than a delete: if the create below fails, this is the // operation DMS is left with, and it says the resource is mid-recreate. - err = db.DeleteState(ctx, d.ResourceKey, dstate.OperationInfo{Action: deployplan.Recreate, InProgress: true}) + err = db.DeleteStateForRecreate(ctx, d.ResourceKey) if err != nil { return fmt.Errorf("deleting state: %w", err) } @@ -167,12 +167,12 @@ func (d *DeploymentUnit) Update(ctx context.Context, db *dstate.DeploymentState, // // Recorded as a delete, which is what it did to the state, not the update that // caused it. - err = db.DeleteState(ctx, d.ResourceKey, dstate.OperationInfo{Action: deployplan.Delete}) + err = db.DeleteState(ctx, d.ResourceKey) if err != nil { return fmt.Errorf("deleting state id=%s: %w", id, err) } } else { - err = d.saveState(ctx, db, id, newState, d.DependsOn, dstate.OperationInfo{Action: deployplan.Update}) + err = d.saveState(ctx, db, id, newState, d.DependsOn) if err != nil { return fmt.Errorf("saving state id=%s: %w", id, err) } @@ -217,7 +217,7 @@ func (d *DeploymentUnit) UpdateWithID(ctx context.Context, db *dstate.Deployment return err } - err = d.saveState(ctx, db, newID, newState, d.DependsOn, dstate.OperationInfo{Action: deployplan.UpdateWithID}) + err = d.saveState(ctx, db, newID, newState, d.DependsOn) if err != nil { return fmt.Errorf("saving state id=%s: %w", oldID, err) } @@ -259,7 +259,7 @@ func (d *DeploymentUnit) Delete(ctx context.Context, db *dstate.DeploymentState, } } - err = db.DeleteState(ctx, d.ResourceKey, dstate.OperationInfo{Action: deployplan.Delete}) + err = db.DeleteState(ctx, d.ResourceKey) if err != nil { return fmt.Errorf("deleting state id=%s: %w", oldID, err) } @@ -300,7 +300,7 @@ func (d *DeploymentUnit) Resize(ctx context.Context, db *dstate.DeploymentState, return fmt.Errorf("resizing id=%s: %w", id, err) } - err = d.saveState(ctx, db, id, newState, d.DependsOn, dstate.OperationInfo{Action: deployplan.Resize}) + err = d.saveState(ctx, db, id, newState, d.DependsOn) if err != nil { return fmt.Errorf("saving state id=%s: %w", id, err) } @@ -310,11 +310,11 @@ func (d *DeploymentUnit) Resize(ctx context.Context, db *dstate.DeploymentState, // saveState saves a state with sensitive fields replaced by a placeholder value so secrets are never written // to disk in plaintext. -func (d *DeploymentUnit) saveState(ctx context.Context, db *dstate.DeploymentState, newID string, state any, dependsOn []deployplan.DependsOnEntry, info dstate.OperationInfo) error { +func (d *DeploymentUnit) saveState(ctx context.Context, db *dstate.DeploymentState, newID string, state any, dependsOn []deployplan.DependsOnEntry) error { if err := zeroSensitiveFields(d.Adapter, state); err != nil { return fmt.Errorf("redacting state: %w", err) } - return db.SaveState(ctx, d.ResourceKey, newID, state, dependsOn, info) + return db.SaveState(ctx, d.ResourceKey, newID, state, dependsOn) } func parseState(destType reflect.Type, raw json.RawMessage) (any, error) { diff --git a/bundle/direct/bind.go b/bundle/direct/bind.go index 8e298e7a315..a7e47ca6ee4 100644 --- a/bundle/direct/bind.go +++ b/bundle/direct/bind.go @@ -93,7 +93,7 @@ func (b *DeploymentBundle) Bind(ctx context.Context, client *databricks.Workspac } // Save state with ID and empty state (like migrate does) - err = b.StateDB.SaveState(ctx, resourceKey, resourceID, struct{}{}, nil, dstate.OperationInfo{Action: deployplan.Create}) + err = b.StateDB.SaveState(ctx, resourceKey, resourceID, struct{}{}, nil) if err != nil { os.Remove(tmpStatePath) return nil, err @@ -151,7 +151,7 @@ func (b *DeploymentBundle) Bind(ctx context.Context, client *databricks.Workspac return nil, err } - err = b.StateDB.SaveState(ctx, resourceKey, resourceID, sv.Value, dependsOn, dstate.OperationInfo{Action: deployplan.Create}) + err = b.StateDB.SaveState(ctx, resourceKey, resourceID, sv.Value, dependsOn) if err != nil { os.Remove(tmpStatePath) return nil, err @@ -221,7 +221,7 @@ func (b *DeploymentBundle) Unbind(ctx context.Context, statePath, resourceKey st } // Delete the main resource - err = b.StateDB.DeleteState(ctx, resourceKey, dstate.OperationInfo{Action: deployplan.Delete}) + err = b.StateDB.DeleteState(ctx, resourceKey) if err != nil { return err } @@ -235,7 +235,7 @@ func (b *DeploymentBundle) Unbind(ctx context.Context, statePath, resourceKey st for key := range b.StateDB.Data.State { if key == permissionsKey || key == grantsKey || strings.HasPrefix(key, resourceKey+".") { - err = b.StateDB.DeleteState(ctx, key, dstate.OperationInfo{Action: deployplan.Delete}) + err = b.StateDB.DeleteState(ctx, key) if err != nil { return err } diff --git a/bundle/direct/bundle_apply.go b/bundle/direct/bundle_apply.go index 85567cc5908..7eb075657f1 100644 --- a/bundle/direct/bundle_apply.go +++ b/bundle/direct/bundle_apply.go @@ -8,7 +8,6 @@ import ( "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/deployplan" - "github.com/databricks/cli/bundle/direct/dstate" "github.com/databricks/cli/bundle/terraform_dabs_map" "github.com/databricks/cli/libs/logdiag" "github.com/databricks/cli/libs/structs/structaccess" @@ -35,7 +34,7 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa return } - // The state DB records every write through this sink, so DMS mirrors the WAL. Uploads run + // The state DB records every write through this sink, so DMS mirrors the WAL. Writes run // on one background goroutine, off the apply path, and are drained below once every // worker has finished recording. opSink := newOperationSink(ctx, b.OpRec) @@ -100,12 +99,12 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa if entry.Gone { // Planning confirmed the resource is already deleted remotely; only // remove it from the state, without calling the delete API. - err = b.StateDB.DeleteState(ctx, resourceKey, dstate.OperationInfo{Action: action}) + err = b.StateDB.DeleteState(ctx, resourceKey) } else { err = d.Destroy(ctx, &b.StateDB) } if err != nil { - opSink.recordFailure(ctx, resourceKey, action, deletedID, err) + opSink.recordFailure(resourceKey, deletedID, err) logdiag.LogError(ctx, fmt.Errorf("%s: %w", errorPrefix, err)) return false } @@ -140,7 +139,7 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa // Empty for a create that never got an ID, and for a recreate whose delete // step already dropped it. failedID := b.StateDB.GetResourceID(resourceKey) - opSink.recordFailure(ctx, resourceKey, action, failedID, err) + opSink.recordFailure(resourceKey, failedID, err) logdiag.LogError(ctx, fmt.Errorf("%s: %w", errorPrefix, err)) return false } diff --git a/bundle/direct/dstate/dms.go b/bundle/direct/dstate/dms.go index 04674cb9b8b..7df599122eb 100644 --- a/bundle/direct/dstate/dms.go +++ b/bundle/direct/dstate/dms.go @@ -6,15 +6,10 @@ import ( "fmt" "github.com/databricks/cli/bundle/deployplan" - "github.com/databricks/databricks-sdk-go/service/bundledeployments" + "github.com/databricks/cli/libs/dms" ) -// ResourceKeyPrefix is what a state key carries and a DMS resource key does not: state calls -// a job "resources.jobs.foo", DMS calls it "jobs.foo". Both sides must use this one constant -// or operations land under keys nothing reads. -const ResourceKeyPrefix = "resources." - -// RecordedState is what the CLI serializes into the DMS Operation.State field. It wraps the +// RecordedState is what the CLI serializes into the DMS operation's state field. It wraps the // config so depends_on survives the round trip: DMS has no field for dependency edges, and // nesting them in the config would collide with resource fields of the same name. type RecordedState struct { @@ -22,75 +17,51 @@ type RecordedState struct { DependsOn []deployplan.DependsOnEntry `json:"depends_on,omitempty"` } -// OperationInfo is what a state write reports to DMS. The caller describes the write; the -// sink does not infer it from the state being nil. -type OperationInfo struct { - // Action is the operation DMS records for this write. - Action deployplan.ActionType - - // InProgress marks a write that is half of a larger change, so an interrupted deploy does - // not leave the resource described as finished. Only a recreate's delete sets it; the - // create that follows updates the same operation to succeeded. - InProgress bool -} - -// OperationSink records one resource operation with DMS. Every state write calls it, so what -// DMS holds mirrors the WAL. It returns no error: the upload runs in the background, and the -// deploy learns of a failure when the queue is drained. +// OperationSink records one resource operation with DMS, so what DMS holds mirrors the WAL. +// inProgress marks a write that is half of a larger change. It returns no error: the write +// runs in the background, and the deploy learns of a failure when the queue is drained. type OperationSink interface { - RecordOperation(ctx context.Context, resourceKey string, info OperationInfo, resourceID string, state json.RawMessage) + RecordOperation(ctx context.Context, resourceKey string, inProgress bool, resourceID string, state json.RawMessage) } // readDMSState replaces the file-derived resource state with what DMS recorded. Recording is // only enabled for net-new deployments, so DMS owns the resource set outright: an empty set // means a successful deploy of nothing, not missing data. The caller holds db.mu. func (db *DeploymentState) readDMSState(ctx context.Context, src *DMSSource) error { - resources, err := fetchDeploymentResources(ctx, src.Client, src.DeploymentID) + recorded, err := src.Client.ListResources(ctx, src.DeploymentID) if err != nil { return err } - db.Data.State = resources - db.stateIDs = make(map[string]string, len(resources)) - for key, entry := range resources { - db.stateIDs[key] = entry.ID - } - return nil -} - -// fetchDeploymentResources lists every resource recorded for the deployment in -// DMS and maps them into state entries keyed by the fully-qualified resource key. -func fetchDeploymentResources(ctx context.Context, client bundledeployments.BundleDeploymentsInterface, deploymentID string) (map[string]ResourceEntry, error) { - it := client.ListResources(ctx, bundledeployments.ListResourcesRequest{ - Parent: "deployments/" + deploymentID, - }) - - out := make(map[string]ResourceEntry) - for it.HasNext(ctx) { - res, err := it.Next(ctx) + resources := make(map[string]ResourceEntry, len(recorded)) + db.stateIDs = make(map[string]string, len(recorded)) + for _, res := range recorded { + entry, err := stateEntry(res) if err != nil { - return nil, fmt.Errorf("listing resources from deployment metadata service: %w", err) + return err } + resources[res.Key.StateKey()] = entry + db.stateIDs[res.Key.StateKey()] = entry.ID + } - // DMS reports resource keys without the "resources." prefix (e.g. - // "jobs.foo"), but the state DB keys are fully qualified - // ("resources.jobs.foo"), so prepend it here. - key := ResourceKeyPrefix + res.ResourceKey - - var recorded RecordedState - if res.State != "" { - // The service stores state as an opaque string, so it arrives as the - // serialized envelope the write side sent. - if err := json.Unmarshal([]byte(res.State), &recorded); err != nil { - return nil, fmt.Errorf("interpreting state recorded for %s: %w", key, err) - } - } + db.Data.State = resources + return nil +} - out[key] = ResourceEntry{ - ID: res.ResourceId, - State: recorded.State, - DependsOn: recorded.DependsOn, +// stateEntry unwraps the envelope the write path recorded for a resource. +func stateEntry(res dms.Resource) (ResourceEntry, error) { + var recorded RecordedState + if res.State != "" { + // The service stores state as an opaque string, so it arrives as the serialized + // envelope the write side sent. + if err := json.Unmarshal([]byte(res.State), &recorded); err != nil { + return ResourceEntry{}, fmt.Errorf("interpreting state recorded for %s: %w", res.Key.StateKey(), err) } } - return out, nil + + return ResourceEntry{ + ID: res.ID, + State: recorded.State, + DependsOn: recorded.DependsOn, + }, nil } diff --git a/bundle/direct/dstate/dms_test.go b/bundle/direct/dstate/dms_test.go index 1120d09bb5d..a2cab988c73 100644 --- a/bundle/direct/dstate/dms_test.go +++ b/bundle/direct/dstate/dms_test.go @@ -6,6 +6,7 @@ import ( "testing" "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/cli/libs/dms" "github.com/databricks/databricks-sdk-go/listing" "github.com/databricks/databricks-sdk-go/service/bundledeployments" "github.com/stretchr/testify/assert" @@ -35,7 +36,12 @@ func (f *fakeResourceLister) ListResources(ctx context.Context, req bundledeploy ) } -func TestFetchDeploymentResourcesUnwrapsEnvelope(t *testing.T) { +// testClient returns a DMS client that lists the resources f holds. +func testClient(f *fakeResourceLister) *dms.Client { + return &dms.Client{Service: f} +} + +func TestReadDMSStateUnwrapsEnvelope(t *testing.T) { // The service stores state as an opaque string, so the envelope arrives verbatim. envelope := `{"state":{"name":"foo"},"depends_on":[{"node":"resources.pipelines.bar","label":"${resources.pipelines.bar.id}"}]}` f := &fakeResourceLister{resources: []bundledeployments.Resource{ @@ -43,8 +49,8 @@ func TestFetchDeploymentResourcesUnwrapsEnvelope(t *testing.T) { {ResourceKey: "pipelines.bar", ResourceId: "456"}, }} - got, err := fetchDeploymentResources(t.Context(), f, "dep-1") - require.NoError(t, err) + var db DeploymentState + require.NoError(t, db.readDMSState(t.Context(), &DMSSource{Client: testClient(f), DeploymentID: "dep-1"})) // depends_on comes back from the envelope, so a bundle whose local state was // wiped still has the edges needed for delete ordering. @@ -55,15 +61,16 @@ func TestFetchDeploymentResourcesUnwrapsEnvelope(t *testing.T) { DependsOn: []deployplan.DependsOnEntry{{Node: "resources.pipelines.bar", Label: "${resources.pipelines.bar.id}"}}, }, "resources.pipelines.bar": {ID: "456"}, - }, got) + }, db.Data.State) } -func TestFetchDeploymentResourcesRejectsMalformedState(t *testing.T) { +func TestReadDMSStateRejectsMalformedState(t *testing.T) { f := &fakeResourceLister{resources: []bundledeployments.Resource{ {ResourceKey: "jobs.foo", ResourceId: "123", State: "not json"}, }} - _, err := fetchDeploymentResources(t.Context(), f, "dep-1") + var db DeploymentState + err := db.readDMSState(t.Context(), &DMSSource{Client: testClient(f), DeploymentID: "dep-1"}) assert.ErrorContains(t, err, "interpreting state recorded for resources.jobs.foo") } @@ -71,9 +78,9 @@ func TestReadDMSStateReplacesLocalState(t *testing.T) { // readDMSState should replace the file-derived state with what DMS has, // even if the file has different resources. src := &DMSSource{ - Client: &fakeResourceLister{resources: []bundledeployments.Resource{ + Client: testClient(&fakeResourceLister{resources: []bundledeployments.Resource{ {ResourceKey: "jobs.foo", ResourceId: "dms-id", State: `{"state":{"name":"from-dms"}}`}, - }}, + }}), DeploymentID: "dep-1", } @@ -97,7 +104,7 @@ func TestReadDMSStateReplacesLocalState(t *testing.T) { func TestReadDMSStateAcceptsEmptyResourceList(t *testing.T) { // An empty DMS response is valid: it means a successful deploy of nothing. src := &DMSSource{ - Client: &fakeResourceLister{resources: []bundledeployments.Resource{}}, + Client: testClient(&fakeResourceLister{resources: []bundledeployments.Resource{}}), DeploymentID: "dep-1", } diff --git a/bundle/direct/dstate/state.go b/bundle/direct/dstate/state.go index 7d11cc1a6a0..20425bcb8af 100644 --- a/bundle/direct/dstate/state.go +++ b/bundle/direct/dstate/state.go @@ -16,8 +16,8 @@ import ( "github.com/databricks/cli/bundle/deployplan" "github.com/databricks/cli/bundle/statemgmt/resourcestate" "github.com/databricks/cli/internal/build" + "github.com/databricks/cli/libs/dms" "github.com/databricks/cli/libs/log" - "github.com/databricks/databricks-sdk-go/service/bundledeployments" "github.com/google/uuid" ) @@ -136,10 +136,8 @@ func NewDatabase(lineage string, serial int) Database { } } -// SaveState records the resource's state after an operation was applied to it. info -// is what the deployment metadata service reports for the write; it is ignored when -// the bundle does not record deployment history. -func (db *DeploymentState) SaveState(ctx context.Context, key, newID string, state any, dependsOn []deployplan.DependsOnEntry, info OperationInfo) error { +// SaveState records the resource's state after an operation was applied to it. +func (db *DeploymentState) SaveState(ctx context.Context, key, newID string, state any, dependsOn []deployplan.DependsOnEntry) error { db.AssertOpenedForWrite() sink, recorded, err := db.saveStateEntry(key, newID, state, dependsOn) @@ -151,7 +149,7 @@ func (db *DeploymentState) SaveState(ctx context.Context, key, newID string, sta // and outside the lock because recording waits when the service is behind - waiting under // db.mu would hold up every other resource's write. if sink != nil { - sink.RecordOperation(ctx, key, info, newID, recorded) + sink.RecordOperation(ctx, key, false, newID, recorded) } return nil @@ -197,9 +195,19 @@ func (db *DeploymentState) saveStateEntry(key, newID string, state any, dependsO return db.sink, recorded, nil } -// DeleteState drops the resource's state entry. info distinguishes a real delete -// from the intermediate drop a recreate performs, both of which are recorded. -func (db *DeploymentState) DeleteState(ctx context.Context, key string, info OperationInfo) error { +// DeleteState drops the resource's state entry: the resource is gone. +func (db *DeploymentState) DeleteState(ctx context.Context, key string) error { + return db.deleteState(ctx, key, false) +} + +// DeleteStateForRecreate drops the resource's state entry as the first half of a recreate. +// The operation is recorded as still in progress, so an interrupted deploy does not leave +// the resource described as finished. +func (db *DeploymentState) DeleteStateForRecreate(ctx context.Context, key string) error { + return db.deleteState(ctx, key, true) +} + +func (db *DeploymentState) deleteState(ctx context.Context, key string, inProgress bool) error { db.AssertOpenedForWrite() sink, deletedID, err := db.deleteStateEntry(key) @@ -210,7 +218,7 @@ func (db *DeploymentState) DeleteState(ctx context.Context, key string, info Ope // State is nil: the resource no longer exists. Recorded outside the lock for the // same reason as SaveState. if sink != nil { - sink.RecordOperation(ctx, key, info, deletedID, nil) + sink.RecordOperation(ctx, key, inProgress, deletedID, nil) } return nil @@ -294,7 +302,7 @@ type ( // service instead of the state file. Callers pass it only when the bundle set // experimental.record_deployment_history; a nil *DMSSource keeps Open file-only. type DMSSource struct { - Client bundledeployments.BundleDeploymentsInterface + Client *dms.Client // DeploymentID is resolved from the deployment's workspace node (see // dms.ResolveDeploymentID), and empty before the first recorded deploy. diff --git a/bundle/direct/dstate/state_test.go b/bundle/direct/dstate/state_test.go index 5ebced6829e..bed003d61df 100644 --- a/bundle/direct/dstate/state_test.go +++ b/bundle/direct/dstate/state_test.go @@ -8,7 +8,6 @@ import ( "path/filepath" "testing" - "github.com/databricks/cli/bundle/deployplan" "github.com/databricks/cli/internal/build" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -25,9 +24,9 @@ type fakeSink struct { ops []string } -func (f *fakeSink) RecordOperation(ctx context.Context, resourceKey string, info OperationInfo, resourceID string, state json.RawMessage) { - entry := fmt.Sprintf("%s %s id=%s state=%s", info.Action, resourceKey, resourceID, string(state)) - if info.InProgress { +func (f *fakeSink) RecordOperation(ctx context.Context, resourceKey string, inProgress bool, resourceID string, state json.RawMessage) { + entry := fmt.Sprintf("%s id=%s state=%s", resourceKey, resourceID, string(state)) + if inProgress { entry += " in_progress" } f.ops = append(f.ops, entry) @@ -45,25 +44,25 @@ func TestStateWritesRecordOperations(t *testing.T) { // the resource recorded as mid-recreate. name: "recreate reports both of its writes", write: func(t *testing.T, db *DeploymentState) { - require.NoError(t, db.SaveState(t.Context(), "jobs.my_job", "123", map[string]string{"key": "old"}, nil, OperationInfo{Action: deployplan.Create})) - require.NoError(t, db.DeleteState(t.Context(), "jobs.my_job", OperationInfo{Action: deployplan.Recreate, InProgress: true})) - require.NoError(t, db.SaveState(t.Context(), "jobs.my_job", "456", map[string]string{"key": "new"}, nil, OperationInfo{Action: deployplan.Recreate})) + require.NoError(t, db.SaveState(t.Context(), "jobs.my_job", "123", map[string]string{"key": "old"}, nil)) + require.NoError(t, db.DeleteStateForRecreate(t.Context(), "jobs.my_job")) + require.NoError(t, db.SaveState(t.Context(), "jobs.my_job", "456", map[string]string{"key": "new"}, nil)) }, want: []string{ - `create jobs.my_job id=123 state={"state":{"key":"old"}}`, - `recreate jobs.my_job id=123 state= in_progress`, - `recreate jobs.my_job id=456 state={"state":{"key":"new"}}`, + `jobs.my_job id=123 state={"state":{"key":"old"}}`, + `jobs.my_job id=123 state= in_progress`, + `jobs.my_job id=456 state={"state":{"key":"new"}}`, }, }, { name: "real delete reports the id it had and no state", write: func(t *testing.T, db *DeploymentState) { - require.NoError(t, db.SaveState(t.Context(), "jobs.my_job", "123", map[string]string{}, nil, OperationInfo{Action: deployplan.Create})) - require.NoError(t, db.DeleteState(t.Context(), "jobs.my_job", OperationInfo{Action: deployplan.Delete})) + require.NoError(t, db.SaveState(t.Context(), "jobs.my_job", "123", map[string]string{}, nil)) + require.NoError(t, db.DeleteState(t.Context(), "jobs.my_job")) }, want: []string{ - `create jobs.my_job id=123 state={"state":{}}`, - `delete jobs.my_job id=123 state=`, + `jobs.my_job id=123 state={"state":{}}`, + `jobs.my_job id=123 state=`, }, }, } @@ -91,8 +90,8 @@ func TestStateWritesRecordNothingWithoutSink(t *testing.T) { // No sink: recording is off, and the writes still succeed. var db DeploymentState require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil)) - require.NoError(t, db.SaveState(t.Context(), "jobs.my_job", "123", map[string]string{}, nil, OperationInfo{Action: deployplan.Create})) - require.NoError(t, db.DeleteState(t.Context(), "jobs.my_job", OperationInfo{Action: deployplan.Delete})) + require.NoError(t, db.SaveState(t.Context(), "jobs.my_job", "123", map[string]string{}, nil)) + require.NoError(t, db.DeleteState(t.Context(), "jobs.my_job")) mustFinalize(t, &db) } @@ -102,7 +101,7 @@ func TestOpenSaveFinalizeRoundTrip(t *testing.T) { var db DeploymentState require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil)) - require.NoError(t, db.SaveState(t.Context(), "jobs.my_job", "123", map[string]string{"key": "val"}, nil, OperationInfo{Action: deployplan.Create})) + require.NoError(t, db.SaveState(t.Context(), "jobs.my_job", "123", map[string]string{"key": "val"}, nil)) mustFinalize(t, &db) // Re-open and verify persisted data. @@ -195,7 +194,7 @@ func TestCLIVersionRecordsLastWriter(t *testing.T) { var db DeploymentState require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil)) - require.NoError(t, db.SaveState(t.Context(), "resources.jobs.my_job", "123", map[string]string{"k": "v"}, nil, OperationInfo{Action: deployplan.Create})) + require.NoError(t, db.SaveState(t.Context(), "resources.jobs.my_job", "123", map[string]string{"k": "v"}, nil)) mustFinalize(t, &db) var reopened DeploymentState @@ -234,7 +233,7 @@ func TestHeaderOnlyWALRecoveryDoesNotAdvanceSerial(t *testing.T) { // Commit serial 1 with one resource. var db DeploymentState require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil)) - require.NoError(t, db.SaveState(t.Context(), "jobs.my_job", "123", map[string]string{}, nil, OperationInfo{Action: deployplan.Create})) + require.NoError(t, db.SaveState(t.Context(), "jobs.my_job", "123", map[string]string{}, nil)) mustFinalize(t, &db) var committed DeploymentState @@ -298,12 +297,12 @@ func TestDeleteState(t *testing.T) { var db DeploymentState require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil)) - require.NoError(t, db.SaveState(t.Context(), "jobs.my_job", "123", map[string]string{}, nil, OperationInfo{Action: deployplan.Create})) + require.NoError(t, db.SaveState(t.Context(), "jobs.my_job", "123", map[string]string{}, nil)) mustFinalize(t, &db) var db2 DeploymentState require.NoError(t, db2.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil)) - require.NoError(t, db2.DeleteState(t.Context(), "jobs.my_job", OperationInfo{Action: deployplan.Delete})) + require.NoError(t, db2.DeleteState(t.Context(), "jobs.my_job")) mustFinalize(t, &db2) var db3 DeploymentState @@ -331,7 +330,7 @@ func TestGetOrInitLineageReadableBeforeWriteAndPersisted(t *testing.T) { // Upgrading to write reuses the same lineage (it goes into the WAL header), // and a write makes it durable. require.NoError(t, db.UpgradeToWrite()) - require.NoError(t, db.SaveState(t.Context(), "jobs.my_job", "123", map[string]string{}, nil, OperationInfo{Action: deployplan.Create})) + require.NoError(t, db.SaveState(t.Context(), "jobs.my_job", "123", map[string]string{}, nil)) mustFinalize(t, &db) // Re-open: the persisted lineage matches the one read before the write. diff --git a/bundle/direct/opclient.go b/bundle/direct/opclient.go deleted file mode 100644 index 97e3ced94eb..00000000000 --- a/bundle/direct/opclient.go +++ /dev/null @@ -1,62 +0,0 @@ -package direct - -import ( - "context" - "fmt" - "net/http" - "strings" - - "github.com/databricks/cli/libs/auth" - "github.com/databricks/databricks-sdk-go/client" - "github.com/databricks/databricks-sdk-go/service/bundledeployments" -) - -// These calls bypass the SDK because it cannot read the response: it types sequence_id as -// an int64 while the service sends a JSON string, so a CreateOperation response fails to -// unmarshal. TODO(DMS): drop this file once the OpenAPI spec types the field as a string. - -// operationResponse is the part of an operation response the CLI reads back. -type operationResponse struct { - // SequenceId is the concurrency token for the next update, typed as the service sends it. - SequenceId string `json:"sequence_id,omitempty"` -} - -// updateOperationRequest carries the fields a later write for the same resource changes. -// action_type and resource_key are left out: the service fixes them at creation. -type updateOperationRequest struct { - State string `json:"state,omitempty"` - ErrorMessage string `json:"error_message,omitempty"` - ResourceId string `json:"resource_id,omitempty"` - Status bundledeployments.OperationStatus `json:"status,omitempty"` - SequenceId string `json:"sequence_id,omitempty"` -} - -// operationClient fills in the operations a version staged. There is no create: the version -// records the whole set at CreateVersion, so every write here narrows an existing operation, -// and fields says which of them it changes. -type operationClient interface { - UpdateOperation(ctx context.Context, parent, resourceKey string, fields []string, body updateOperationRequest) (operationResponse, error) -} - -// apiOperationClient talks to the operations API through the workspace client. -type apiOperationClient struct { - client *client.DatabricksClient -} - -// newAPIOperationClient returns an operationClient that posts to the DMS API. -func newAPIOperationClient(c *client.DatabricksClient) operationClient { - return &apiOperationClient{client: c} -} - -func (a *apiOperationClient) UpdateOperation(ctx context.Context, parent, resourceKey string, fields []string, body updateOperationRequest) (operationResponse, error) { - var result operationResponse - path := fmt.Sprintf("/api/2.0/bundle/%s/operations/%s", parent, resourceKey) - err := a.client.Do(ctx, http.MethodPatch, path, - auth.WorkspaceIDHeaders(a.client.Config), - map[string]any{"update_mask": strings.Join(fields, ",")}, - body, &result) - if err != nil { - return operationResponse{}, err - } - return result, nil -} diff --git a/bundle/direct/oprecorder.go b/bundle/direct/oprecorder.go deleted file mode 100644 index a6e46ae3a3a..00000000000 --- a/bundle/direct/oprecorder.go +++ /dev/null @@ -1,230 +0,0 @@ -package direct - -import ( - "context" - "encoding/json" - "fmt" - "slices" - "strings" - "sync" - "unicode/utf8" - - "github.com/databricks/cli/bundle/deployplan" - "github.com/databricks/cli/bundle/direct/dstate" - "github.com/databricks/cli/libs/diag" - "github.com/databricks/databricks-sdk-go/client" - "github.com/databricks/databricks-sdk-go/service/bundledeployments" -) - -// maxOperationStateSize is the largest serialized state DMS accepts per operation. More is -// rejected server-side, so fail early with a message naming the resource. -const maxOperationStateSize = 64 * 1024 - -// maxOperationErrorMessageSize is the largest error message DMS accepts. A longer one is -// truncated rather than rejected, so recording cannot fail and hide the error it reports. -const maxOperationErrorMessageSize = 16 * 1024 - -// operationStatusInProgress marks an operation whose writes are not finished. Not taken from -// the SDK: the enum is generated from the OpenAPI spec, which trails the service proto -// (databricks-eng/universe#2394529). -const operationStatusInProgress bundledeployments.OperationStatus = "OPERATION_STATUS_IN_PROGRESS" - -// stagedSequenceID is what CreateVersion leaves on every operation it stages, and so the -// precondition for the first update of a resource. -const stagedSequenceID = "0" - -// recordedOperation is an applied resource operation waiting to be uploaded. It is built on -// the apply worker, not in the uploader, so a malformed state fails the resource that -// produced it rather than the drain at the end of apply. -type recordedOperation struct { - resourceID string - status bundledeployments.OperationStatus - - // errorMessage is set only when status is failed, which the service enforces. - errorMessage string - - // state is the serialized config after the operation: nil for a delete, and the - // pre-deploy state for a failure (see newFailedOperation). - state json.RawMessage - - // updateFields is the mask to send when updating an operation the service already has. - // It is taken literally: a field named here is written, one left out keeps its value. - updateFields []string -} - -// describesResource is the update mask for an operation that says how the resource looks: -// every field an update may change. Any other path is rejected with INVALID_PARAMETER_VALUE, -// so this doubles as the canonical field list and order. -var describesResource = []string{"state", "error_message", "resource_id", "status"} - -// failedKeepingState is the update mask for a failure: mark it failed and leave state alone. -// State means the resource is as it was written; no state means a delete went through and -// nothing replaced it, so the resource really is gone and the deployment should say so. -var failedKeepingState = []string{"error_message", "status"} - -// newStateOperation describes a state write for upload. state is the serialized -// RecordedState envelope the state DB just persisted, and nil for a delete, where -// the resource is gone. It errors when the state exceeds maxOperationStateSize. -func newStateOperation(info dstate.OperationInfo, resourceID string, state json.RawMessage) (recordedOperation, error) { - // The action is not recorded - the version stages it - but an unrecordable one means - // there is no operation to update, so fail the resource rather than the whole drain. - if _, err := DeployActionToSDK(info.Action); err != nil { - return recordedOperation{}, err - } - - if len(state) > maxOperationStateSize { - return recordedOperation{}, fmt.Errorf("serialized state is %d bytes, which exceeds the %d byte limit for recording deployment history", len(state), maxOperationStateSize) - } - - status := bundledeployments.OperationStatusOperationStatusSucceeded - if info.InProgress { - status = operationStatusInProgress - } - - return recordedOperation{ - resourceID: resourceID, - status: status, - state: state, - updateFields: describesResource, - }, nil -} - -// newFailedOperation records an operation that did not apply, so the history says why a -// resource failed rather than leaving it pending. It carries no state: the version staged the -// operation already, so a failure only ever narrows an existing record. -func newFailedOperation(action deployplan.ActionType, resourceID string, cause error) (recordedOperation, error) { - if _, err := DeployActionToSDK(action); err != nil { - return recordedOperation{}, err - } - - // Summarized, not cause.Error(): for an API failure that adds the status and error - // code, which is often the most actionable part of the history. - message := diag.FormatAPIErrorSummary(cause) - if len(message) > maxOperationErrorMessageSize { - message = message[:maxOperationErrorMessageSize] - // The cut can land inside a rune, and the service stores a string. Drop the partial - // one: at most UTFMax-1 bytes of it can be left, so a message that was already - // invalid loses those bytes rather than being stripped away entirely. - for range utf8.UTFMax - 1 { - if utf8.ValidString(message) { - break - } - message = message[:len(message)-1] - } - } - - return recordedOperation{ - resourceID: resourceID, - status: bundledeployments.OperationStatusOperationStatusFailed, - errorMessage: message, - updateFields: failedKeepingState, - }, nil -} - -// operationUploader records an applied resource operation with DMS. Uploads run on -// the operationSink goroutine, off the apply path. -type operationUploader interface { - upload(ctx context.Context, resourceKey string, op recordedOperation) error -} - -// operationRecorder uploads operations via the DMS operations API. -type operationRecorder struct { - ops operationClient - // parent is the version the operations are recorded under, formatted as - // "deployments/{deployment_id}/versions/{version_id}". - parent string - - // mu guards sequenceIDs. - mu sync.Mutex - - // sequenceIDs holds the sequence id the service last returned per resource key, echoed as - // the precondition on the next update. A key absent from the map has not been written yet, - // so its staged operation is still at stagedSequenceID. - sequenceIDs map[string]string -} - -// NewOperationRecorder returns an operationUploader backed by the DMS operations -// API. deploymentID and version identify the deployment version assigned by DMS -// that the operations are recorded under. -func NewOperationRecorder(apiClient *client.DatabricksClient, deploymentID string, version int64) operationUploader { - return newOperationRecorder(newAPIOperationClient(apiClient), deploymentID, version) -} - -// newOperationRecorder is the internal constructor, so tests can supply their own -// operationClient. -func newOperationRecorder(ops operationClient, deploymentID string, version int64) operationUploader { - return &operationRecorder{ - ops: ops, - parent: fmt.Sprintf("deployments/%s/versions/%d", deploymentID, version), - sequenceIDs: make(map[string]string), - } -} - -func (r *operationRecorder) upload(ctx context.Context, resourceKey string, op recordedOperation) error { - // The read path re-adds the prefix; see dstate.ResourceKeyPrefix. - dmsKey := strings.TrimPrefix(resourceKey, dstate.ResourceKeyPrefix) - - operation := bundledeployments.Operation{ - ResourceId: op.resourceID, - ResourceKey: dmsKey, - Status: op.status, - ErrorMessage: op.errorMessage, - // The service stores state as an opaque string, so the serialized envelope goes - // on the wire as-is. Empty means unset, which is what a delete records. - State: string(op.state), - } - - r.mu.Lock() - sequenceID, written := r.sequenceIDs[dmsKey] - r.mu.Unlock() - if !written { - sequenceID = stagedSequenceID - } - - update := updateOperationRequest{ - ErrorMessage: operation.ErrorMessage, - Status: operation.Status, - SequenceId: sequenceID, - } - // Send only what the mask names. The service would ignore the rest, and state is the - // largest field by far, so a failure that keeps the recorded state sends none. - if slices.Contains(op.updateFields, "state") { - update.State = operation.State - update.ResourceId = operation.ResourceId - } - - // action_type is fixed when the version stages the operation, so it is not sent. - result, err := r.ops.UpdateOperation(ctx, r.parent, dmsKey, op.updateFields, update) - if err != nil { - return err - } - - // The next write for this resource echoes the sequence id this one earned. - r.mu.Lock() - r.sequenceIDs[dmsKey] = result.SequenceId - r.mu.Unlock() - - return nil -} - -// DeployActionToSDK maps a deployplan action to its DMS operation action type. -// Only actions that mutate a resource are recordable; Skip and Undefined never -// reach a recorder and are rejected rather than silently coerced. -func DeployActionToSDK(a deployplan.ActionType) (bundledeployments.OperationActionType, error) { - switch a { - case deployplan.Create: - return bundledeployments.OperationActionTypeOperationActionTypeCreate, nil - case deployplan.Update: - return bundledeployments.OperationActionTypeOperationActionTypeUpdate, nil - case deployplan.UpdateWithID: - return bundledeployments.OperationActionTypeOperationActionTypeUpdateWithId, nil - case deployplan.Recreate: - return bundledeployments.OperationActionTypeOperationActionTypeRecreate, nil - case deployplan.Resize: - return bundledeployments.OperationActionTypeOperationActionTypeResize, nil - case deployplan.Delete: - return bundledeployments.OperationActionTypeOperationActionTypeDelete, nil - default: - return "", fmt.Errorf("cannot record operation: unsupported action %q", a) - } -} diff --git a/bundle/direct/oprecorder_test.go b/bundle/direct/oprecorder_test.go deleted file mode 100644 index ce62b39dd7d..00000000000 --- a/bundle/direct/oprecorder_test.go +++ /dev/null @@ -1,313 +0,0 @@ -package direct - -import ( - "context" - "encoding/json" - "errors" - "strings" - "sync" - "testing" - "unicode/utf8" - - "github.com/databricks/cli/bundle/deployplan" - "github.com/databricks/cli/bundle/direct/dstate" - "github.com/databricks/databricks-sdk-go/service/bundledeployments" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -// fakeOpCall is one recorded call to the operations API. -type fakeOpCall struct { - method string - parent string - resourceKey string - update updateOperationRequest - fields []string -} - -type fakeOpClient struct { - mu sync.Mutex - calls []fakeOpCall - // sequence is what the service reports back; a string, as the service sends it. - sequence string -} - -func (f *fakeOpClient) UpdateOperation(ctx context.Context, parent, resourceKey string, fields []string, body updateOperationRequest) (operationResponse, error) { - f.mu.Lock() - defer f.mu.Unlock() - f.calls = append(f.calls, fakeOpCall{method: "update", parent: parent, resourceKey: resourceKey, update: body, fields: fields}) - return operationResponse{SequenceId: f.sequence}, nil -} - -// uploadOne records a single operation through the given uploader, mirroring what -// an operationQueue worker does. -func uploadOne(t *testing.T, u operationUploader, resourceKey string, action deployplan.ActionType, resourceID string, state json.RawMessage) { - t.Helper() - op, err := newStateOperation(dstate.OperationInfo{Action: action}, resourceID, state) - require.NoError(t, err) - require.NoError(t, u.upload(t.Context(), resourceKey, op)) -} - -func TestOperationRecorderStripsResourcePrefix(t *testing.T) { - f := &fakeOpClient{sequence: "1"} - r := newOperationRecorder(f, "dep-1", 2) - - uploadOne(t, r, "resources.jobs.foo", deployplan.Create, "job-123", envelope(t, "foo")) - - require.Len(t, f.calls, 1) - c := f.calls[0] - // The version already staged this operation, so the first write updates it and echoes - // the sequence id staging left. The wire key drops the CLI-internal "resources." prefix. - assert.Equal(t, "update", c.method) - assert.Equal(t, "jobs.foo", c.resourceKey) - assert.Equal(t, "deployments/dep-1/versions/2", c.parent) - assert.Equal(t, stagedSequenceID, c.update.SequenceId) - assert.Equal(t, "job-123", c.update.ResourceId) - require.NotEmpty(t, c.update.State) -} - -func TestOperationRecorderUpdatesSecondWriteForSameResource(t *testing.T) { - // One operation per resource per version: the second write has to update the - // first, echoing the sequence_id the service returned as its precondition. - f := &fakeOpClient{sequence: "7"} - r := newOperationRecorder(f, "dep-1", 2) - - uploadOne(t, r, "resources.jobs.foo", deployplan.Recreate, "", nil) - uploadOne(t, r, "resources.jobs.foo", deployplan.Recreate, "job-456", envelope(t, "new")) - - require.Len(t, f.calls, 2) - assert.Equal(t, stagedSequenceID, f.calls[0].update.SequenceId) - - assert.Equal(t, "update", f.calls[1].method) - assert.Equal(t, "jobs.foo", f.calls[1].resourceKey) - assert.Equal(t, "7", f.calls[1].update.SequenceId) - assert.Equal(t, "job-456", f.calls[1].update.ResourceId) - require.NotEmpty(t, f.calls[1].update.State) -} - -func TestOperationRecorderFailureAfterAStateWriteKeepsTheState(t *testing.T) { - // The create wrote state for a resource that exists, then the wait failed. The update - // only marks it failed: sending empty state would drop the resource from the deployment. - f := &fakeOpClient{sequence: "3"} - r := newOperationRecorder(f, "dep-1", 2) - - uploadOne(t, r, "resources.job_runs.my_run", deployplan.Create, "run-1", envelope(t, "the run")) - - failed, err := newFailedOperation(deployplan.Create, "", errors.New("run did not succeed: FAILED")) - require.NoError(t, err) - require.NoError(t, r.upload(t.Context(), "resources.job_runs.my_run", failed)) - - require.Len(t, f.calls, 2) - assert.Equal(t, "update", f.calls[0].method) - - update := f.calls[1] - assert.Equal(t, "update", update.method) - assert.Equal(t, []string{"error_message", "status"}, update.fields) - assert.Equal(t, bundledeployments.OperationStatusOperationStatusFailed, update.update.Status) - assert.Equal(t, "run did not succeed: FAILED", update.update.ErrorMessage) - // Neither is in the mask, so what the create recorded stands. - assert.Empty(t, update.update.State) - assert.Empty(t, update.update.ResourceId) -} - -func TestOperationRecorderFailedRecreateKeepsTheResourceGone(t *testing.T) { - // The recreate's delete is recorded with no state, and then the create fails. The failure - // must not fill that gap with the pre-deploy state: the resource really is gone. - f := &fakeOpClient{sequence: "2"} - r := newOperationRecorder(f, "dep-1", 2) - - uploadOne(t, r, "resources.jobs.foo", deployplan.Recreate, "old-id", nil) - - failed, err := newFailedOperation(deployplan.Recreate, "old-id", errors.New("boom")) - require.NoError(t, err) - require.NoError(t, r.upload(t.Context(), "resources.jobs.foo", failed)) - - require.Len(t, f.calls, 2) - update := f.calls[1] - assert.Equal(t, "update", update.method) - assert.Equal(t, []string{"error_message", "status"}, update.fields) - assert.Empty(t, update.update.State) - assert.Equal(t, bundledeployments.OperationStatusOperationStatusFailed, update.update.Status) - assert.Equal(t, "boom", update.update.ErrorMessage) -} - -func TestOperationRecorderFailureCarryingALaterWriteSendsIt(t *testing.T) { - // Two writes, the first uploaded and the second still waiting when the resource failed, - // so the failure took it over. The update must name state or the first write's stands. - f := &fakeOpClient{sequence: "4"} - r := newOperationRecorder(f, "dep-1", 2) - - uploadOne(t, r, "resources.jobs.foo", deployplan.Update, "id-1", envelope(t, "first write")) - - second, err := newStateOperation(dstate.OperationInfo{Action: deployplan.Update}, "id-1", envelope(t, "second write")) - require.NoError(t, err) - failed, err := newFailedOperation(deployplan.Update, "id-old", errors.New("boom")) - require.NoError(t, err) - require.NoError(t, r.upload(t.Context(), "resources.jobs.foo", coalesce(second, failed))) - - require.Len(t, f.calls, 2) - update := f.calls[1] - assert.Equal(t, []string{"state", "error_message", "resource_id", "status"}, update.fields) - require.NotEmpty(t, update.update.State) - assert.Contains(t, update.update.State, "second write") - assert.Equal(t, "id-1", update.update.ResourceId) -} - -func TestOperationRecorderFailureBeforeAnyWriteNarrowsTheStagedOperation(t *testing.T) { - // Nothing was written for the resource, so the failure updates the operation the version - // staged, at the sequence id staging left. It sends no state: the resource was not - // touched, and the staged operation already holds whatever the deployment knows. - f := &fakeOpClient{sequence: "1"} - r := newOperationRecorder(f, "dep-1", 2) - - failed, err := newFailedOperation(deployplan.Update, "main.some_schema", errors.New("boom")) - require.NoError(t, err) - require.NoError(t, r.upload(t.Context(), "resources.schemas.foo", failed)) - - require.Len(t, f.calls, 1) - assert.Equal(t, "update", f.calls[0].method) - assert.Equal(t, stagedSequenceID, f.calls[0].update.SequenceId) - assert.Equal(t, []string{"error_message", "status"}, f.calls[0].fields) - assert.Empty(t, f.calls[0].update.State) -} - -func TestOperationRecorderTracksSequencePerResource(t *testing.T) { - // Each resource has its own staged operation, so each one's first write echoes the staged - // sequence id rather than a sequence another resource earned. - f := &fakeOpClient{sequence: "1"} - r := newOperationRecorder(f, "dep-1", 2) - - uploadOne(t, r, "resources.jobs.foo", deployplan.Create, "id-1", envelope(t, "foo")) - uploadOne(t, r, "resources.jobs.bar", deployplan.Create, "id-2", envelope(t, "bar")) - - require.Len(t, f.calls, 2) - assert.Equal(t, stagedSequenceID, f.calls[0].update.SequenceId) - assert.Equal(t, stagedSequenceID, f.calls[1].update.SequenceId) -} - -func TestNewStateOperationRecordsEnvelopeAsIs(t *testing.T) { - // The state DB serializes the envelope (see dstate.SaveState); the operation - // carries it through untouched, sensitive fields and all. - state := json.RawMessage(`{"state":{"name":"foo","token":"super-secret"}}`) - - op, err := newStateOperation(dstate.OperationInfo{Action: deployplan.Create}, "job-123", state) - require.NoError(t, err) - - assert.JSONEq(t, string(state), string(op.state)) - assert.Equal(t, bundledeployments.OperationStatusOperationStatusSucceeded, op.status) -} - -func TestNewStateOperationRejectsUnsupportedAction(t *testing.T) { - _, err := newStateOperation(dstate.OperationInfo{Action: deployplan.Skip}, "job-123", nil) - assert.Error(t, err) -} - -func TestNewStateOperationRejectsOversizedState(t *testing.T) { - big := json.RawMessage(strings.Repeat("x", maxOperationStateSize+1)) - - _, err := newStateOperation(dstate.OperationInfo{Action: deployplan.Create}, "job-123", big) - assert.ErrorContains(t, err, "exceeds the 65536 byte limit") -} - -func TestNewFailedOperationRecordsError(t *testing.T) { - op, err := newFailedOperation(deployplan.Create, "", errors.New("cluster spec is invalid")) - require.NoError(t, err) - - assert.Equal(t, bundledeployments.OperationStatusOperationStatusFailed, op.status) - assert.Equal(t, "cluster spec is invalid", op.errorMessage) - // The resource was never written, so there is no state to serve back for it. - assert.Nil(t, op.state) - // An update only marks the operation failed; see failedKeepingState. - assert.Equal(t, failedKeepingState, op.updateFields) -} - -func TestNewFailedOperationTruncatesLongError(t *testing.T) { - // Truncated rather than rejected: a message over the limit would make recording - // fail and hide the error it is reporting. - op, err := newFailedOperation(deployplan.Update, "job-123", errors.New(strings.Repeat("x", maxOperationErrorMessageSize+100))) - require.NoError(t, err) - - assert.Len(t, op.errorMessage, maxOperationErrorMessageSize) -} - -func TestNewFailedOperationPreservesUTF8OnTruncation(t *testing.T) { - // The cut lands one byte into the emoji, so a byte-wise truncation would leave a partial - // rune behind and the service stores state and messages as strings. - msg := strings.Repeat("a", maxOperationErrorMessageSize-1) + "❌" + "x" - - op, err := newFailedOperation(deployplan.Update, "job-123", errors.New(msg)) - require.NoError(t, err) - - assert.True(t, utf8.ValidString(op.errorMessage)) - // The whole emoji went, so the message is shorter than the limit rather than exactly it. - assert.Equal(t, strings.Repeat("a", maxOperationErrorMessageSize-1), op.errorMessage) -} - -func TestOperationRecorderReturnsAPIErrors(t *testing.T) { - // A failed upload returns its error and leaves the recorded sequence id alone, so a later - // write for the same resource still updates the operation with the precondition the - // service last gave us rather than trying to create a second one. - failingClient := &failingOpClient{sequence: "9", failOn: 1} - r := newOperationRecorder(failingClient, "dep-1", 2) - - uploadOne(t, r, "resources.jobs.foo", deployplan.Create, "job-1", envelope(t, "first")) - - second, err := newStateOperation(dstate.OperationInfo{Action: deployplan.Update}, "job-2", envelope(t, "second")) - require.NoError(t, err) - err = r.upload(t.Context(), "resources.jobs.foo", second) - require.Error(t, err) - assert.Equal(t, "injected error", err.Error()) - - // The third write is what proves the sequence id survived the failure. - uploadOne(t, r, "resources.jobs.foo", deployplan.Update, "job-3", envelope(t, "third")) - - require.Len(t, failingClient.calls, 3) - assert.Equal(t, "update", failingClient.calls[0].method) - assert.Equal(t, "update", failingClient.calls[1].method) - assert.Equal(t, "update", failingClient.calls[2].method) - assert.Equal(t, "9", failingClient.calls[2].update.SequenceId) -} - -// failingOpClient fails the call at index failOn and reports sequence on the rest. -type failingOpClient struct { - mu sync.Mutex - calls []fakeOpCall - sequence string - failOn int -} - -func (f *failingOpClient) UpdateOperation(ctx context.Context, parent, resourceKey string, fields []string, body updateOperationRequest) (operationResponse, error) { - f.mu.Lock() - defer f.mu.Unlock() - callNum := len(f.calls) - f.calls = append(f.calls, fakeOpCall{method: "update", parent: parent, resourceKey: resourceKey, update: body, fields: fields}) - if callNum == f.failOn { - return operationResponse{}, errors.New("injected error") - } - return operationResponse{SequenceId: f.sequence}, nil -} - -func TestDeployActionToSDK(t *testing.T) { - cases := []struct { - action deployplan.ActionType - want bundledeployments.OperationActionType - }{ - {deployplan.Create, bundledeployments.OperationActionTypeOperationActionTypeCreate}, - {deployplan.Update, bundledeployments.OperationActionTypeOperationActionTypeUpdate}, - {deployplan.UpdateWithID, bundledeployments.OperationActionTypeOperationActionTypeUpdateWithId}, - {deployplan.Recreate, bundledeployments.OperationActionTypeOperationActionTypeRecreate}, - {deployplan.Resize, bundledeployments.OperationActionTypeOperationActionTypeResize}, - {deployplan.Delete, bundledeployments.OperationActionTypeOperationActionTypeDelete}, - } - for _, c := range cases { - got, err := DeployActionToSDK(c.action) - require.NoError(t, err) - assert.Equal(t, c.want, got) - } - - // Skip and Undefined never reach a recorder and are rejected. - _, err := DeployActionToSDK(deployplan.Skip) - assert.Error(t, err) - _, err = DeployActionToSDK(deployplan.Undefined) - assert.Error(t, err) -} diff --git a/bundle/direct/opsink.go b/bundle/direct/opsink.go index 48d2e95f687..78995231a4a 100644 --- a/bundle/direct/opsink.go +++ b/bundle/direct/opsink.go @@ -4,11 +4,9 @@ import ( "context" "encoding/json" "fmt" - "slices" "sync" - "github.com/databricks/cli/bundle/deployplan" - "github.com/databricks/cli/bundle/direct/dstate" + "github.com/databricks/cli/libs/dms" ) // operationSinkQueueSize is how many resources may be waiting to be recorded before the @@ -16,18 +14,18 @@ import ( // and DMS is what the next plan reads. const operationSinkQueueSize = 10 -// operationSink uploads operations one at a time on a background goroutine, so a deploy -// never waits on a round trip. queue holds resource keys and pending holds the newest -// operation per key, so a second write for a resource simply replaces the first. +// operationSink writes operations one at a time on a background goroutine, so a deploy never +// waits on a round trip. queue holds bundle state keys, converted where they go on the wire, +// and pending the newest update per key, so a second write for a resource replaces the first. type operationSink struct { - uploader operationUploader + writer dms.OperationWriter // queue holds the keys that have something waiting. One slot per resource, so a full // queue means the deploy is that many resources ahead and the next write waits. Record // outside the state DB lock, or that wait blocks every other resource too. queue chan string - // done is closed once the uploader has drained the queue and returned. + // done is closed once the writer has drained the queue and returned. done chan struct{} // stopQueue closes the queue, wrapped so close can safely run twice. @@ -36,24 +34,24 @@ type operationSink struct { // mu guards the fields below. mu sync.Mutex - // pending holds the newest operation per resource key, absent once the uploader takes it. - pending map[string]recordedOperation + // pending holds the newest update per resource key, absent once the writer takes it. + pending map[string]dms.OperationUpdate err error } -// newOperationSink starts the uploader. It returns nil when recording is off, and every +// newOperationSink starts the writer. It returns nil when recording is off, and every // method is a no-op on a nil sink. ctx must outlive close. -func newOperationSink(ctx context.Context, uploader operationUploader) *operationSink { - if uploader == nil { +func newOperationSink(ctx context.Context, writer dms.OperationWriter) *operationSink { + if writer == nil { return nil } s := &operationSink{ - uploader: uploader, - queue: make(chan string, operationSinkQueueSize), - done: make(chan struct{}), - pending: make(map[string]recordedOperation), + writer: writer, + queue: make(chan string, operationSinkQueueSize), + done: make(chan struct{}), + pending: make(map[string]dms.OperationUpdate), } s.stopQueue = sync.OnceFunc(func() { close(s.queue) }) @@ -64,118 +62,77 @@ func newOperationSink(ctx context.Context, uploader operationUploader) *operatio // RecordOperation implements dstate.OperationSink, turning every state write into an // operation so DMS mirrors the local state. state is the serialized envelope, and nil for // a delete. An earlier failure does not stop it: keep recording, best effort. -func (s *operationSink) RecordOperation(ctx context.Context, resourceKey string, info dstate.OperationInfo, resourceID string, state json.RawMessage) { +func (s *operationSink) RecordOperation(ctx context.Context, resourceKey string, inProgress bool, resourceID string, state json.RawMessage) { if s == nil { return } - op, err := newStateOperation(info, resourceID, state) + update, err := dms.NewStateUpdate(resourceID, state, inProgress) if err != nil { s.setErr(fmt.Errorf("recording operation for %s: %w", resourceKey, err)) return } - s.record(resourceKey, op) + s.record(resourceKey, update) } // recordFailure records that a resource did not apply, so the history says why rather // than leaving the resource out. -func (s *operationSink) recordFailure(ctx context.Context, resourceKey string, action deployplan.ActionType, resourceID string, cause error) { +func (s *operationSink) recordFailure(resourceKey, resourceID string, cause error) { if s == nil { return } - op, err := newFailedOperation(action, resourceID, cause) - if err != nil { - s.setErr(fmt.Errorf("recording failure for %s: %w", resourceKey, err)) - return - } - - s.record(resourceKey, op) + s.record(resourceKey, dms.NewFailureUpdate(resourceID, cause)) } -// record makes op the one waiting for resourceKey, waiting itself while the queue is +// record makes update the one waiting for resourceKey, waiting itself while the queue is // full. Recording after close panics, so every caller must return before close. -func (s *operationSink) record(resourceKey string, op recordedOperation) { +func (s *operationSink) record(resourceKey string, update dms.OperationUpdate) { s.mu.Lock() waiting, queued := s.pending[resourceKey] if queued { - op = coalesce(waiting, op) + update = waiting.Merge(update) } - s.pending[resourceKey] = op + s.pending[resourceKey] = update s.mu.Unlock() - // Already queued: the uploader reads the map when it gets to the key, so it picks up + // Already queued: the writer reads the map when it gets to the key, so it picks up // what was just stored. No second slot, and no waiting. if !queued { s.queue <- resourceKey } } -// coalesce merges an operation with the one that superseded it while still waiting. Each field -// comes from whichever operation claimed it in its mask, newer winning when both did, and the -// mask is the union. What an operation claims is decided where it is built, not here. -func coalesce(older, newer recordedOperation) recordedOperation { - merged := older - merged.updateFields = unionFields(older.updateFields, newer.updateFields) - - if slices.Contains(newer.updateFields, "state") { - merged.state = newer.state - } - if slices.Contains(newer.updateFields, "resource_id") { - merged.resourceID = newer.resourceID - } - if slices.Contains(newer.updateFields, "error_message") { - merged.errorMessage = newer.errorMessage - } - if slices.Contains(newer.updateFields, "status") { - merged.status = newer.status - } - - return merged -} - -// unionFields returns every field either mask names, in describesResource's order so the -// merged mask is deterministic on the wire. -func unionFields(older, newer []string) []string { - merged := make([]string, 0, len(describesResource)) - for _, field := range describesResource { - if slices.Contains(older, field) || slices.Contains(newer, field) { - merged = append(merged, field) - } - } - return merged -} - -// take claims the operation waiting for resourceKey. -func (s *operationSink) take(resourceKey string) (recordedOperation, bool) { +// take claims the update waiting for resourceKey. +func (s *operationSink) take(resourceKey string) (dms.OperationUpdate, bool) { s.mu.Lock() defer s.mu.Unlock() - op, ok := s.pending[resourceKey] + update, ok := s.pending[resourceKey] delete(s.pending, resourceKey) - return op, ok + return update, ok } func (s *operationSink) run(ctx context.Context) { defer close(s.done) for resourceKey := range s.queue { - op, ok := s.take(resourceKey) + update, ok := s.take(resourceKey) if !ok { // Unreachable: a key is queued only when nothing was waiting for it. Guard so - // a stray key could never upload a zero-valued operation. + // a stray key could never write a zero-valued update. continue } - // Keep going after a failure, so one bad upload does not drop everything behind it. - if err := s.uploader.upload(ctx, resourceKey, op); err != nil { + // Keep going after a failure, so one bad write does not drop everything behind it. + if err := s.writer.Write(ctx, dms.KeyFromState(resourceKey), update); err != nil { s.setErr(fmt.Errorf("recording operation for %s with the deployment metadata service: %w", resourceKey, err)) } } } -// close drains what is waiting and returns the first upload error, which fails the deploy: +// close drains what is waiting and returns the first write error, which fails the deploy: // DMS is the source of truth, so a missing record would have the next deploy create a // resource that already exists. Safe to call twice. func (s *operationSink) close() error { diff --git a/bundle/direct/opsink_test.go b/bundle/direct/opsink_test.go index 4124109cbb9..dcabe8cbb8c 100644 --- a/bundle/direct/opsink_test.go +++ b/bundle/direct/opsink_test.go @@ -10,72 +10,74 @@ import ( "testing" "time" - "github.com/databricks/cli/bundle/deployplan" "github.com/databricks/cli/bundle/direct/dstate" + "github.com/databricks/cli/libs/dms" "github.com/databricks/databricks-sdk-go/service/bundledeployments" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) -// fakeUploader records the uploads it receives and optionally blocks until block is -// closed, so a test can hold the uploader and observe what coalesces behind it. -type fakeUploader struct { +// fakeWriter records the writes it receives and optionally blocks until block is +// closed, so a test can hold the writer and observe what coalesces behind it. +// +// Keys arrive in the DMS form, which is what the sink puts on the wire. +type fakeWriter struct { block chan struct{} - started chan string + started chan dms.ResourceKey err error mu sync.Mutex - uploads []string - resourceIDs map[string]string - statuses map[string]bundledeployments.OperationStatus - errorMessages map[string]string + writes []string + resourceIDs map[dms.ResourceKey]string + statuses map[dms.ResourceKey]bundledeployments.OperationStatus + errorMessages map[dms.ResourceKey]string } -func (f *fakeUploader) upload(ctx context.Context, resourceKey string, op recordedOperation) error { +func (f *fakeWriter) Write(ctx context.Context, key dms.ResourceKey, update dms.OperationUpdate) error { if f.started != nil { - f.started <- resourceKey + f.started <- key } if f.block != nil { <-f.block } f.mu.Lock() - f.uploads = append(f.uploads, resourceKey+"="+string(op.state)) + f.writes = append(f.writes, string(key)+"="+string(update.State)) if f.resourceIDs == nil { - f.resourceIDs = map[string]string{} - f.statuses = map[string]bundledeployments.OperationStatus{} - f.errorMessages = map[string]string{} + f.resourceIDs = map[dms.ResourceKey]string{} + f.statuses = map[dms.ResourceKey]bundledeployments.OperationStatus{} + f.errorMessages = map[dms.ResourceKey]string{} } - f.resourceIDs[resourceKey] = op.resourceID - f.statuses[resourceKey] = op.status - f.errorMessages[resourceKey] = op.errorMessage + f.resourceIDs[key] = update.ResourceID + f.statuses[key] = update.Status + f.errorMessages[key] = update.ErrorMessage f.mu.Unlock() return f.err } -func (f *fakeUploader) recorded() []string { +func (f *fakeWriter) recorded() []string { f.mu.Lock() defer f.mu.Unlock() - return append([]string(nil), f.uploads...) + return append([]string(nil), f.writes...) } -func (f *fakeUploader) resourceIDFor(resourceKey string) string { +func (f *fakeWriter) resourceIDFor(key dms.ResourceKey) string { f.mu.Lock() defer f.mu.Unlock() - return f.resourceIDs[resourceKey] + return f.resourceIDs[key] } -func (f *fakeUploader) statusFor(resourceKey string) bundledeployments.OperationStatus { +func (f *fakeWriter) statusFor(key dms.ResourceKey) bundledeployments.OperationStatus { f.mu.Lock() defer f.mu.Unlock() - return f.statuses[resourceKey] + return f.statuses[key] } -func (f *fakeUploader) errorMessageFor(resourceKey string) string { +func (f *fakeWriter) errorMessageFor(key dms.ResourceKey) string { f.mu.Lock() defer f.mu.Unlock() - return f.errorMessages[resourceKey] + return f.errorMessages[key] } // envelope builds the serialized RecordedState the state DB hands the sink. @@ -88,11 +90,11 @@ func envelope(t *testing.T, name string) json.RawMessage { func recordState(t *testing.T, s *operationSink, resourceKey, name string) { t.Helper() - s.RecordOperation(t.Context(), resourceKey, dstate.OperationInfo{Action: deployplan.Update}, "id-1", envelope(t, name)) + s.RecordOperation(t.Context(), resourceKey, false, "id-1", envelope(t, name)) } -func TestOperationSinkUploadsEachOperation(t *testing.T) { - f := &fakeUploader{} +func TestOperationSinkWritesEachOperation(t *testing.T) { + f := &fakeWriter{} s := newOperationSink(t.Context(), f) for i := range 20 { @@ -103,32 +105,32 @@ func TestOperationSinkUploadsEachOperation(t *testing.T) { assert.Len(t, f.recorded(), 20) } -func TestOperationSinkKeepsUploadingAfterGoingIdle(t *testing.T) { - // The uploader parks on an empty queue instead of returning. Apply spends most of a - // deploy inside resource CRUD, so the queue is empty far more often than not, and an - // uploader that exited while idle would silently drop everything recorded after it. - f := &fakeUploader{} +func TestOperationSinkKeepsWritingAfterGoingIdle(t *testing.T) { + // The writer parks on an empty queue instead of returning. Apply spends most of a + // deploy inside resource CRUD, so the queue is empty far more often than not, and a + // writer that exited while idle would silently drop everything recorded after it. + f := &fakeWriter{} s := newOperationSink(t.Context(), f) recordState(t, s, "resources.jobs.foo", "v1") require.Eventually(t, func() bool { return len(f.recorded()) == 1 }, 5*time.Second, time.Millisecond) - // The queue is drained and the uploader idle; what is recorded now still has to go. + // The queue is drained and the writer idle; what is recorded now still has to go. recordState(t, s, "resources.jobs.bar", "v1") require.NoError(t, s.close()) assert.Len(t, f.recorded(), 2) } -func TestOperationSinkCoalescesWritesBehindAnUpload(t *testing.T) { - // Hold the uploader on the first write so the two behind it pile up. They carry +func TestOperationSinkCoalescesWritesBehindAWrite(t *testing.T) { + // Hold the writer on the first write so the two behind it pile up. They carry // the resource's full state, so only the newest needs to go: the resource costs // two requests rather than three. - f := &fakeUploader{block: make(chan struct{}), started: make(chan string, 2)} + f := &fakeWriter{block: make(chan struct{}), started: make(chan dms.ResourceKey, 2)} s := newOperationSink(t.Context(), f) recordState(t, s, "resources.jobs.foo", "v1") - assert.Equal(t, "resources.jobs.foo", <-f.started) + assert.Equal(t, dms.ResourceKey("jobs.foo"), <-f.started) recordState(t, s, "resources.jobs.foo", "v2") recordState(t, s, "resources.jobs.foo", "v3") @@ -137,190 +139,150 @@ func TestOperationSinkCoalescesWritesBehindAnUpload(t *testing.T) { require.NoError(t, s.close()) assert.Equal(t, []string{ - `resources.jobs.foo={"state":{"name":"v1"}}`, - `resources.jobs.foo={"state":{"name":"v3"}}`, + `jobs.foo={"state":{"name":"v1"}}`, + `jobs.foo={"state":{"name":"v3"}}`, }, f.recorded()) } func TestOperationSinkCoalescedFailureKeepsTheStateItReplaces(t *testing.T) { - // The create writes state and then fails before the upload. The failure must not replace - // that state with its own emptiness, which would drop the resource from the deployment. - f := &fakeUploader{block: make(chan struct{}), started: make(chan string, 2)} + // The create writes state and then fails before the write goes out. The failure must not + // replace that state with its own emptiness, which would drop the resource from the + // deployment. + f := &fakeWriter{block: make(chan struct{}), started: make(chan dms.ResourceKey, 2)} s := newOperationSink(t.Context(), f) - // Occupy the uploader with an unrelated resource, so the two writes below both + // Occupy the writer with an unrelated resource, so the two writes below both // land in pending and coalesce. recordState(t, s, "resources.jobs.busy", "v1") - assert.Equal(t, "resources.jobs.busy", <-f.started) + assert.Equal(t, dms.ResourceKey("jobs.busy"), <-f.started) - s.RecordOperation(t.Context(), "resources.job_runs.my_run", dstate.OperationInfo{Action: deployplan.Create}, "run-1", envelope(t, "the run")) - // priorState and priorID are empty: the resource was created in this deploy, so - // there is no pre-deploy record to report. - s.recordFailure(t.Context(), "resources.job_runs.my_run", deployplan.Create, "", errors.New("run did not succeed: FAILED")) + s.RecordOperation(t.Context(), "resources.job_runs.my_run", false, "run-1", envelope(t, "the run")) + // The id is empty: the resource was created in this deploy, so there is no pre-deploy + // record to report. + s.recordFailure("resources.job_runs.my_run", "", errors.New("run did not succeed: FAILED")) close(f.block) require.NoError(t, s.close()) assert.Equal(t, []string{ - `resources.jobs.busy={"state":{"name":"v1"}}`, - `resources.job_runs.my_run={"state":{"name":"the run"}}`, + `jobs.busy={"state":{"name":"v1"}}`, + `job_runs.my_run={"state":{"name":"the run"}}`, }, f.recorded()) - assert.Equal(t, "run-1", f.resourceIDFor("resources.job_runs.my_run")) - assert.Equal(t, - bundledeployments.OperationStatusOperationStatusFailed, - f.statusFor("resources.job_runs.my_run")) - assert.Equal(t, "run did not succeed: FAILED", f.errorMessageFor("resources.job_runs.my_run")) + assert.Equal(t, "run-1", f.resourceIDFor("job_runs.my_run")) + assert.Equal(t, bundledeployments.OperationStatusOperationStatusFailed, f.statusFor("job_runs.my_run")) + assert.Equal(t, "run did not succeed: FAILED", f.errorMessageFor("job_runs.my_run")) } func TestOperationSinkCoalescedFailureAfterADeleteKeepsTheResourceGone(t *testing.T) { // The recreate's delete writes no state. When the create then fails, the failure takes // that absent state rather than the pre-deploy one, so the resource stays gone. - f := &fakeUploader{block: make(chan struct{}), started: make(chan string, 2)} + f := &fakeWriter{block: make(chan struct{}), started: make(chan dms.ResourceKey, 2)} s := newOperationSink(t.Context(), f) recordState(t, s, "resources.jobs.busy", "v1") - assert.Equal(t, "resources.jobs.busy", <-f.started) + assert.Equal(t, dms.ResourceKey("jobs.busy"), <-f.started) - s.RecordOperation(t.Context(), "resources.schemas.foo", dstate.OperationInfo{Action: deployplan.Recreate, InProgress: true}, "old-id", nil) - s.recordFailure(t.Context(), "resources.schemas.foo", deployplan.Recreate, "old-id", errors.New("Catalog 'other' does not exist")) + s.RecordOperation(t.Context(), "resources.schemas.foo", true, "old-id", nil) + s.recordFailure("resources.schemas.foo", "old-id", errors.New("Catalog 'other' does not exist")) close(f.block) require.NoError(t, s.close()) assert.Equal(t, []string{ - `resources.jobs.busy={"state":{"name":"v1"}}`, - `resources.schemas.foo=`, + `jobs.busy={"state":{"name":"v1"}}`, + `schemas.foo=`, }, f.recorded()) - assert.Equal(t, - bundledeployments.OperationStatusOperationStatusFailed, - f.statusFor("resources.schemas.foo")) + assert.Equal(t, bundledeployments.OperationStatusOperationStatusFailed, f.statusFor("schemas.foo")) } func TestOperationSinkCoalescedFailureDoesNotRevertToPriorState(t *testing.T) { - // An update that succeeded and then failed waiting carries the pre-deploy state, which the - // write it supersedes has moved past. Sending it would record the resource as it was before - // the deploy, and the next plan would read that back as current. - f := &fakeUploader{block: make(chan struct{}), started: make(chan string, 2)} + // An update that succeeded and then failed waiting carries the pre-deploy id, which the + // write it supersedes has moved past. Reporting it would record the resource as it was + // before the deploy, and the next plan would read that back as current. + f := &fakeWriter{block: make(chan struct{}), started: make(chan dms.ResourceKey, 2)} s := newOperationSink(t.Context(), f) recordState(t, s, "resources.jobs.busy", "v1") - assert.Equal(t, "resources.jobs.busy", <-f.started) + assert.Equal(t, dms.ResourceKey("jobs.busy"), <-f.started) - s.RecordOperation(t.Context(), "resources.jobs.foo", dstate.OperationInfo{Action: deployplan.Update}, "id-new", envelope(t, "after the update")) - s.recordFailure(t.Context(), "resources.jobs.foo", deployplan.Update, "id-old", errors.New("waiting after updating: timed out")) + s.RecordOperation(t.Context(), "resources.jobs.foo", false, "id-new", envelope(t, "after the update")) + s.recordFailure("resources.jobs.foo", "id-old", errors.New("waiting after updating: timed out")) close(f.block) require.NoError(t, s.close()) assert.Equal(t, []string{ - `resources.jobs.busy={"state":{"name":"v1"}}`, - `resources.jobs.foo={"state":{"name":"after the update"}}`, + `jobs.busy={"state":{"name":"v1"}}`, + `jobs.foo={"state":{"name":"after the update"}}`, }, f.recorded()) - assert.Equal(t, "id-new", f.resourceIDFor("resources.jobs.foo")) - assert.Equal(t, - bundledeployments.OperationStatusOperationStatusFailed, - f.statusFor("resources.jobs.foo")) + assert.Equal(t, "id-new", f.resourceIDFor("jobs.foo")) + assert.Equal(t, bundledeployments.OperationStatusOperationStatusFailed, f.statusFor("jobs.foo")) } func TestOperationSinkCoalescedDeleteStillClearsState(t *testing.T) { // A delete legitimately carries no state, and coalescing must let it through: the // resource is gone, and keeping the state it replaces would leave it listed. - f := &fakeUploader{block: make(chan struct{}), started: make(chan string, 2)} + f := &fakeWriter{block: make(chan struct{}), started: make(chan dms.ResourceKey, 2)} s := newOperationSink(t.Context(), f) recordState(t, s, "resources.jobs.busy", "v1") - assert.Equal(t, "resources.jobs.busy", <-f.started) + assert.Equal(t, dms.ResourceKey("jobs.busy"), <-f.started) - s.RecordOperation(t.Context(), "resources.jobs.foo", dstate.OperationInfo{Action: deployplan.Update}, "id-1", envelope(t, "before")) - s.RecordOperation(t.Context(), "resources.jobs.foo", dstate.OperationInfo{Action: deployplan.Delete}, "id-1", nil) + s.RecordOperation(t.Context(), "resources.jobs.foo", false, "id-1", envelope(t, "before")) + s.RecordOperation(t.Context(), "resources.jobs.foo", false, "id-1", nil) close(f.block) require.NoError(t, s.close()) assert.Equal(t, []string{ - `resources.jobs.busy={"state":{"name":"v1"}}`, - `resources.jobs.foo=`, + `jobs.busy={"state":{"name":"v1"}}`, + `jobs.foo=`, }, f.recorded()) } -func TestCoalesceLetsAWriteSupersedeAFailure(t *testing.T) { - // A failure is not the last word. A retry that writes state wins whole - state, id and - // mask - and the mask names error_message so the recorded failure is cleared. The service - // rejects a succeeded operation that still carries an error. - failed, err := newFailedOperation(deployplan.Update, "id-old", errors.New("boom")) - require.NoError(t, err) - retried, err := newStateOperation(dstate.OperationInfo{Action: deployplan.Update}, "id-new", envelope(t, "after the retry")) - require.NoError(t, err) - - got := coalesce(failed, retried) - - assert.Equal(t, bundledeployments.OperationStatusOperationStatusSucceeded, got.status) - assert.Empty(t, got.errorMessage) - assert.Equal(t, "id-new", got.resourceID) - assert.JSONEq(t, string(envelope(t, "after the retry")), string(got.state)) - assert.Equal(t, describesResource, got.updateFields) -} - -func TestCoalesceKeepsTheWritesStateAndMask(t *testing.T) { - // A failure claims only status and error_message, so the write's state, id and mask - // survive. - write, err := newStateOperation(dstate.OperationInfo{Action: deployplan.Delete}, "id-new", nil) - require.NoError(t, err) - failed, err := newFailedOperation(deployplan.Update, "id-old", errors.New("boom")) - require.NoError(t, err) - - got := coalesce(write, failed) - - assert.Equal(t, bundledeployments.OperationStatusOperationStatusFailed, got.status) - assert.Equal(t, "boom", got.errorMessage) - assert.Equal(t, "id-new", got.resourceID) - assert.Nil(t, got.state) - assert.Equal(t, describesResource, got.updateFields) -} - -func TestOperationSinkRecordDuringUploadIsStillUploaded(t *testing.T) { - f := &fakeUploader{block: make(chan struct{}), started: make(chan string, 2)} +func TestOperationSinkRecordDuringWriteIsStillWritten(t *testing.T) { + f := &fakeWriter{block: make(chan struct{}), started: make(chan dms.ResourceKey, 2)} s := newOperationSink(t.Context(), f) recordState(t, s, "resources.jobs.foo", "v1") - assert.Equal(t, "resources.jobs.foo", <-f.started) + assert.Equal(t, dms.ResourceKey("jobs.foo"), <-f.started) - // The uploader has taken this key off pending and is uploading it right now. + // The writer has taken this key off pending and is writing it right now. recordState(t, s, "resources.jobs.foo", "v2") close(f.block) require.NoError(t, s.close()) - // Two uploads, in order: an in-flight request cannot be recalled, so v2 goes up + // Two writes, in order: an in-flight request cannot be recalled, so v2 goes up // after v1 rather than replacing it. The service ends up with the newest state. assert.Equal(t, []string{ - `resources.jobs.foo={"state":{"name":"v1"}}`, - `resources.jobs.foo={"state":{"name":"v2"}}`, + `jobs.foo={"state":{"name":"v1"}}`, + `jobs.foo={"state":{"name":"v2"}}`, }, f.recorded()) assert.Empty(t, s.pending) } func TestOperationSinkRecordWaitsWhenTheQueueIsFull(t *testing.T) { // Recording holds the deploy back once every slot is taken. started is buffered for every - // upload: nothing reads it after the first, and an uploader blocked sending to it would + // write: nothing reads it after the first, and a writer blocked sending to it would // never drain the queue. - f := &fakeUploader{block: make(chan struct{}), started: make(chan string, operationSinkQueueSize+4)} + f := &fakeWriter{block: make(chan struct{}), started: make(chan dms.ResourceKey, operationSinkQueueSize+4)} s := newOperationSink(t.Context(), f) - // One key is taken off the queue and stuck in the uploader; the rest fill it. + // One key is taken off the queue and stuck in the writer; the rest fill it. recordState(t, s, "resources.jobs.busy", "v1") - assert.Equal(t, "resources.jobs.busy", <-f.started) + assert.Equal(t, dms.ResourceKey("jobs.busy"), <-f.started) for i := range operationSinkQueueSize { recordState(t, s, "resources.jobs.job"+strconv.Itoa(i), "v1") } - // The next distinct resource has nowhere to go until the uploader moves on. Called + // The next distinct resource has nowhere to go until the writer moves on. Called // directly rather than through recordState: its assertions may only run on the // test's own goroutine. late := envelope(t, "v1") blocked := make(chan struct{}) go func() { - s.RecordOperation(t.Context(), "resources.jobs.late", dstate.OperationInfo{Action: deployplan.Update}, "id-1", late) + s.RecordOperation(t.Context(), "resources.jobs.late", false, "id-1", late) close(blocked) }() @@ -337,23 +299,23 @@ func TestOperationSinkRecordWaitsWhenTheQueueIsFull(t *testing.T) { assert.Len(t, f.recorded(), operationSinkQueueSize+2) } -func TestOperationSinkReturnsUploadError(t *testing.T) { - uploadErr := errors.New("boom") - f := &fakeUploader{err: uploadErr} +func TestOperationSinkReturnsWriteError(t *testing.T) { + writeErr := errors.New("boom") + f := &fakeWriter{err: writeErr} s := newOperationSink(t.Context(), f) recordState(t, s, "resources.jobs.foo", "v1") err := s.close() require.Error(t, err) - assert.ErrorIs(t, err, uploadErr) + assert.ErrorIs(t, err, writeErr) assert.ErrorContains(t, err, "resources.jobs.foo") } -func TestOperationSinkKeepsRecordingAfterUploadError(t *testing.T) { - // One failed upload must not drop the records for everything behind it, so DMS +func TestOperationSinkKeepsRecordingAfterWriteError(t *testing.T) { + // One failed write must not drop the records for everything behind it, so DMS // ends up as close to reality as it can get. - f := &fakeUploader{err: errors.New("boom")} + f := &fakeWriter{err: errors.New("boom")} s := newOperationSink(t.Context(), f) recordState(t, s, "resources.jobs.foo", "v1") @@ -364,7 +326,7 @@ func TestOperationSinkKeepsRecordingAfterUploadError(t *testing.T) { } func TestOperationSinkFirstErrIsWhatStopsTheDeploy(t *testing.T) { - f := &fakeUploader{err: errors.New("boom")} + f := &fakeWriter{err: errors.New("boom")} s := newOperationSink(t.Context(), f) assert.NoError(t, s.firstErr()) @@ -377,29 +339,14 @@ func TestOperationSinkFirstErrIsWhatStopsTheDeploy(t *testing.T) { assert.Error(t, s.firstErr()) } -func TestOperationSinkFailsOnUnsupportedAction(t *testing.T) { - f := &fakeUploader{} - s := newOperationSink(t.Context(), f) - - // Skip never reaches a sink, so this is a programming error rather than anything a - // user did - but it still has to fail the deploy rather than pass silently, because - // the resource would be left out of the deployment. - s.RecordOperation(t.Context(), "resources.jobs.foo", dstate.OperationInfo{Action: deployplan.Skip}, "id-1", nil) - - err := s.close() - require.Error(t, err) - assert.ErrorContains(t, err, "resources.jobs.foo") - assert.Empty(t, f.recorded()) -} - func TestOperationSinkFailsOnOversizedState(t *testing.T) { - // The service will not take a state this large, so the resource cannot be recorded. - // Failing here says so, where reporting nothing would leave DMS without the resource - // and the next plan would create it again. - f := &fakeUploader{} + // The service will not take a state this large (the limit lives in libs/dms), so the + // resource cannot be recorded. Failing here says so, where reporting nothing would leave + // DMS without the resource and the next plan would create it again. + f := &fakeWriter{} s := newOperationSink(t.Context(), f) - s.RecordOperation(t.Context(), "resources.jobs.foo", dstate.OperationInfo{Action: deployplan.Create}, "id-1", envelope(t, strings.Repeat("x", maxOperationStateSize))) + s.RecordOperation(t.Context(), "resources.jobs.foo", false, "id-1", envelope(t, strings.Repeat("x", 64*1024))) err := s.close() require.Error(t, err) @@ -408,7 +355,7 @@ func TestOperationSinkFailsOnOversizedState(t *testing.T) { } func TestOperationSinkCloseIsIdempotent(t *testing.T) { - f := &fakeUploader{} + f := &fakeWriter{} s := newOperationSink(t.Context(), f) recordState(t, s, "resources.jobs.foo", "v1") @@ -420,13 +367,13 @@ func TestOperationSinkCloseIsIdempotent(t *testing.T) { func TestNilOperationSinkIsNoOp(t *testing.T) { var s *operationSink - s.RecordOperation(t.Context(), "resources.jobs.foo", dstate.OperationInfo{Action: deployplan.Create}, "id-1", nil) - s.recordFailure(t.Context(), "resources.jobs.foo", deployplan.Create, "id-1", errors.New("boom")) + s.RecordOperation(t.Context(), "resources.jobs.foo", false, "id-1", nil) + s.recordFailure("resources.jobs.foo", "id-1", errors.New("boom")) assert.NoError(t, s.firstErr()) assert.NoError(t, s.close()) } -func TestNewOperationSinkNilUploaderIsNil(t *testing.T) { +func TestNewOperationSinkNilWriterIsNil(t *testing.T) { // Recording off: the sink is nil so the state DB's nil check leaves it unset. assert.Nil(t, newOperationSink(t.Context(), nil)) } diff --git a/bundle/direct/pkg.go b/bundle/direct/pkg.go index b7a7a1a4693..b58b8766429 100644 --- a/bundle/direct/pkg.go +++ b/bundle/direct/pkg.go @@ -10,6 +10,7 @@ import ( "github.com/databricks/cli/bundle/direct/dresources" "github.com/databricks/cli/bundle/direct/dstate" "github.com/databricks/cli/bundle/statemgmt/resourcestate" + "github.com/databricks/cli/libs/dms" "github.com/databricks/cli/libs/structs/structvar" ) @@ -45,10 +46,10 @@ type DeploymentBundle struct { RemoteStateCache sync.Map StateCache structvar.Cache - // OpRec uploads applied operations to DMS. Nil unless the bundle records deployment - // history, in which case the deploy phase sets it once CreateVersion has claimed a - // version. Apply drains it before returning. - OpRec operationUploader + // OpRec records applied operations with DMS. Nil unless the bundle records deployment + // history, in which case the deploy phase sets it once the version exists. Apply drains + // it before returning. + OpRec dms.OperationWriter } // SetRemoteState updates the remote state with type validation and marks as fresh. diff --git a/bundle/migrate/build_state.go b/bundle/migrate/build_state.go index 639dcb4cfc5..70c48de6371 100644 --- a/bundle/migrate/build_state.go +++ b/bundle/migrate/build_state.go @@ -207,7 +207,7 @@ func BuildStateFromTF( // Migration rebuilds local state from terraform's; nothing is deployed, and // the DMS sink is never set on this state, so the action is not reported. - if err := stateDB.SaveState(ctx, node, id, sv.Value, dependsOn, dstate.OperationInfo{Action: deployplan.Create}); err != nil { + if err := stateDB.SaveState(ctx, node, id, sv.Value, dependsOn); err != nil { return warningsSeen, fmt.Errorf("%s: SaveState: %w", node, err) } } diff --git a/bundle/phases/deploy.go b/bundle/phases/deploy.go index e2ffa45783d..475ae045e16 100644 --- a/bundle/phases/deploy.go +++ b/bundle/phases/deploy.go @@ -208,13 +208,13 @@ func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHand // // The version is created only after approval; CompleteVersion is deferred before // lock.Release and no-ops until then. - recorder, err := newDeploymentRecorder(ctx, b, stateEngine, dms.VersionTypeDeploy) + recording, err := newRecording(ctx, b, stateEngine, dms.VersionTypeDeploy) if err != nil { logdiag.LogError(ctx, err) return } defer func() { - if err := recorder.CompleteVersion(ctx, !logdiag.HasError(ctx)); err != nil { + if err := recording.Finish(ctx, !logdiag.HasError(ctx)); err != nil { logdiag.LogError(ctx, err) } bundle.ApplyContext(ctx, b, lock.Release(lock.GoalDeploy)) @@ -281,16 +281,16 @@ func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHand // Settle deployment and version before planning. Plan snapshots the config, so // both must be stamped before it is computed. Version itself is created after approval. - if err := recorder.PrepareDeployment(ctx); err != nil { + if err := recording.Prepare(ctx); err != nil { logdiag.LogError(ctx, err) return } - if recorder != nil { + if recording.Enabled() { // The deployment ID is stamped earlier, when the state is opened; only the // version is new here. A first deploy has no ID until now, so stamp both. bundle.ApplySeqContext(ctx, b, - metadata.AnnotateDeployment(recorder.DeploymentID()), - metadata.AnnotateDeploymentVersion(recorder.Version()), + metadata.AnnotateDeployment(recording.DeploymentID()), + metadata.AnnotateDeploymentVersion(recording.Version()), ) if logdiag.HasError(ctx) { return @@ -355,17 +355,15 @@ func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHand logdiag.LogError(ctx, err) return } - if err := recorder.CreateVersion(ctx, staged); err != nil { + writer, err := recording.Start(ctx, staged) + if err != nil { logdiag.LogError(ctx, err) return } - logDeploymentVersion(ctx, b, recorder) + logDeploymentVersion(ctx, b, recording) // Record operations under that version, so DMS holds the deployed resource state. - setOperationRecorder(ctx, b, recorder) - if logdiag.HasError(ctx) { - return - } + setOperationWriter(b, recording, writer) deployCore(ctx, b, plan, stateEngine, requestedEngine) if logdiag.HasError(ctx) { diff --git a/bundle/phases/destroy.go b/bundle/phases/destroy.go index 2ad57e12282..0016d4adf8f 100644 --- a/bundle/phases/destroy.go +++ b/bundle/phases/destroy.go @@ -135,7 +135,7 @@ func approvalForDestroy(ctx context.Context, b *bundle.Bundle, plan *deployplan. return cmdio.AskYesOrNo(ctx, "Would you like to proceed?") } -func destroyCore(ctx context.Context, b *bundle.Bundle, plan *deployplan.Plan, engine engine.EngineType, recorder *dms.Recorder) { +func destroyCore(ctx context.Context, b *bundle.Bundle, plan *deployplan.Plan, engine engine.EngineType, recording dms.Recording) { if engine.IsDirect() { b.DeploymentBundle.Apply(ctx, b.WorkspaceClient(ctx), plan) } else { @@ -160,7 +160,7 @@ func destroyCore(ctx context.Context, b *bundle.Bundle, plan *deployplan.Plan, e } // Complete version before deleting remote files; the deployment node is under statePath. - if err := recorder.CompleteVersion(ctx, true); err != nil { + if err := recording.Finish(ctx, true); err != nil { logdiag.LogError(ctx, err) return } @@ -205,13 +205,13 @@ func Destroy(ctx context.Context, b *bundle.Bundle, engine engine.EngineType) { // Set up DMS recording of this destroy. Version is created after approval; cancelled // destroy records nothing. Deferred before lock.Release to hold the lock. - recorder, err := newDeploymentRecorder(ctx, b, engine, dms.VersionTypeDestroy) + recording, err := newRecording(ctx, b, engine, dms.VersionTypeDestroy) if err != nil { logdiag.LogError(ctx, err) return } defer func() { - if err := recorder.CompleteVersion(ctx, !logdiag.HasError(ctx)); err != nil { + if err := recording.Finish(ctx, !logdiag.HasError(ctx)); err != nil { logdiag.LogError(ctx, err) } bundle.ApplyContext(ctx, b, lock.Release(lock.GoalDestroy)) @@ -277,15 +277,13 @@ func Destroy(ctx context.Context, b *bundle.Bundle, engine engine.EngineType) { logdiag.LogError(ctx, err) return } - if err := recorder.CreateVersion(ctx, staged); err != nil { + writer, err := recording.Start(ctx, staged) + if err != nil { logdiag.LogError(ctx, err) return } - setOperationRecorder(ctx, b, recorder) - if logdiag.HasError(ctx) { - return - } - destroyCore(ctx, b, plan, engine, recorder) + setOperationWriter(b, recording, writer) + destroyCore(ctx, b, plan, engine, recording) } else { cmdio.LogString(ctx, "Destroy cancelled!") } diff --git a/bundle/phases/dms.go b/bundle/phases/dms.go index 6bf5b23c30e..b997f10fcec 100644 --- a/bundle/phases/dms.go +++ b/bundle/phases/dms.go @@ -10,41 +10,33 @@ import ( "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/engine" "github.com/databricks/cli/bundle/deployplan" - "github.com/databricks/cli/bundle/direct" - "github.com/databricks/cli/bundle/direct/dstate" - "github.com/databricks/cli/bundle/env" "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/dms" "github.com/databricks/cli/libs/log" - "github.com/databricks/cli/libs/logdiag" "github.com/databricks/cli/libs/workspaceurls" - "github.com/databricks/databricks-sdk-go/client" "github.com/databricks/databricks-sdk-go/service/bundledeployments" ) -// newDeploymentRecorder returns a recorder for the deployment, or nil if recording -// does not apply. Enabled only for direct engine and when the bundle opts in. -// The deployment ID is resolved from the workspace node, empty on first deploy. -func newDeploymentRecorder(ctx context.Context, b *bundle.Bundle, eng engine.EngineType, versionType dms.VersionType) (*dms.Recorder, error) { - if !recordsDeploymentHistory(ctx, b) { - return nil, nil - } - if !eng.IsDirect() { - return nil, nil +// newRecording returns what this run records with DMS, or a disabled recording when +// nothing is: recording needs the direct engine and the bundle's opt-in. The deployment ID +// is resolved from the workspace node, and is empty on a first deploy. +func newRecording(ctx context.Context, b *bundle.Bundle, eng engine.EngineType, versionType dms.VersionType) (dms.Recording, error) { + if !b.RecordsDeploymentHistory(ctx) || !eng.IsDirect() { + return dms.Disabled(), nil } + w := b.WorkspaceClient(ctx) statePath := b.Config.Workspace.StatePath - deploymentID, err := dms.ResolveDeploymentID(ctx, b.WorkspaceClient(ctx), statePath) + deploymentID, err := dms.ResolveDeploymentID(ctx, w, statePath) if err != nil { return nil, err } - apiClient, err := client.New(b.WorkspaceClient(ctx).Config) + client, err := dms.NewClient(w) if err != nil { return nil, err } - return dms.NewRecorder(dms.RecorderOptions{ - Service: b.WorkspaceClient(ctx).BundleDeployments, - Versions: dms.NewAPIVersionCreator(apiClient), + return dms.NewRecording(dms.RecordingOptions{ + Client: client, DeploymentID: deploymentID, StatePath: statePath, VersionType: versionType, @@ -62,48 +54,54 @@ func stagedOperations(plan *deployplan.Plan) ([]dms.StagedOperation, error) { if action.ActionType == deployplan.Skip || action.ActionType == deployplan.Undefined { continue } - actionType, err := direct.DeployActionToSDK(action.ActionType) + actionType, err := actionToSDK(action.ActionType) if err != nil { return nil, fmt.Errorf("%s: %w", action.ResourceKey, err) } staged = append(staged, dms.StagedOperation{ - // The service wants the key without the CLI's "resources." prefix. - ResourceKey: strings.TrimPrefix(action.ResourceKey, dstate.ResourceKeyPrefix), + ResourceKey: dms.KeyFromState(action.ResourceKey), ActionType: actionType, }) } return staged, nil } -// recordsDeploymentHistory reports whether this bundle records deployment history, -// from experimental.record_deployment_history or -// DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY. -func recordsDeploymentHistory(ctx context.Context, b *bundle.Bundle) bool { - configured := b.Config.Experimental != nil && b.Config.Experimental.RecordDeploymentHistory - return env.RecordsDeploymentHistory(ctx, configured) -} - -// setOperationRecorder points the deployment at the version the recorder claimed, so -// the state writes during apply are recorded under it. A nil recorder means recording -// is off and leaves the deployment's uploader unset. -func setOperationRecorder(ctx context.Context, b *bundle.Bundle, recorder *dms.Recorder) { - if recorder == nil { - return +// actionToSDK maps a deployplan action to the DMS action type a staged operation records. +// Only actions that mutate a resource are recordable; Skip and Undefined are rejected +// rather than silently coerced. +func actionToSDK(a deployplan.ActionType) (bundledeployments.OperationActionType, error) { + switch a { + case deployplan.Create: + return bundledeployments.OperationActionTypeOperationActionTypeCreate, nil + case deployplan.Update: + return bundledeployments.OperationActionTypeOperationActionTypeUpdate, nil + case deployplan.UpdateWithID: + return bundledeployments.OperationActionTypeOperationActionTypeUpdateWithId, nil + case deployplan.Recreate: + return bundledeployments.OperationActionTypeOperationActionTypeRecreate, nil + case deployplan.Resize: + return bundledeployments.OperationActionTypeOperationActionTypeResize, nil + case deployplan.Delete: + return bundledeployments.OperationActionTypeOperationActionTypeDelete, nil + default: + return "", fmt.Errorf("cannot record operation: unsupported action %q", a) } +} - apiClient, err := client.New(b.WorkspaceClient(ctx).Config) - if err != nil { - logdiag.LogError(ctx, err) +// setOperationWriter has the state writes during apply recorded under the started version. +// A disabled recording leaves the writer unset, which is also what keeps the state DB from +// serializing an envelope for every write. +func setOperationWriter(b *bundle.Bundle, recording dms.Recording, writer dms.OperationWriter) { + if !recording.Enabled() { return } - - b.DeploymentBundle.OpRec = direct.NewOperationRecorder(apiClient, recorder.DeploymentID(), recorder.Version()) + b.DeploymentBundle.OpRec = writer } // logDeploymentVersion logs the deployment version URL. Workspace ID is omitted // so the page stays clickable in a terminal and redirects correctly without it. -func logDeploymentVersion(ctx context.Context, b *bundle.Bundle, recorder *dms.Recorder) { - if recorder == nil || recorder.Version() == 0 { +func logDeploymentVersion(ctx context.Context, b *bundle.Bundle, recording dms.Recording) { + if recording.Version() == 0 { return } @@ -112,11 +110,11 @@ func logDeploymentVersion(ctx context.Context, b *bundle.Bundle, recorder *dms.R // Only the link is lost, so report the version without it rather than failing // a deploy over it. log.Debugf(ctx, "Not linking to the recorded deployment: %s", err) - cmdio.LogString(ctx, fmt.Sprintf("Current Deployment Version: %s version %d", recorder.DeploymentID(), recorder.Version())) + cmdio.LogString(ctx, fmt.Sprintf("Current Deployment Version: %s version %d", recording.DeploymentID(), recording.Version())) return } - cmdio.LogString(ctx, "Current Deployment Version: "+workspaceurls.DeploymentURL(*baseURL, recorder.DeploymentID(), recorder.Version())) + cmdio.LogString(ctx, "Current Deployment Version: "+workspaceurls.DeploymentURL(*baseURL, recording.DeploymentID(), recording.Version())) } // deploymentMetadata describes the bundle this deploy came from and where it diff --git a/bundle/phases/dms_test.go b/bundle/phases/dms_test.go index 72567d3a757..272b89d3dea 100644 --- a/bundle/phases/dms_test.go +++ b/bundle/phases/dms_test.go @@ -53,3 +53,29 @@ func TestStagedOperationsEmptyPlan(t *testing.T) { require.NoError(t, err) assert.Empty(t, staged) } + +func TestActionToSDK(t *testing.T) { + cases := []struct { + action deployplan.ActionType + want bundledeployments.OperationActionType + }{ + {deployplan.Create, bundledeployments.OperationActionTypeOperationActionTypeCreate}, + {deployplan.Update, bundledeployments.OperationActionTypeOperationActionTypeUpdate}, + {deployplan.UpdateWithID, bundledeployments.OperationActionTypeOperationActionTypeUpdateWithId}, + {deployplan.Recreate, bundledeployments.OperationActionTypeOperationActionTypeRecreate}, + {deployplan.Resize, bundledeployments.OperationActionTypeOperationActionTypeResize}, + {deployplan.Delete, bundledeployments.OperationActionTypeOperationActionTypeDelete}, + } + for _, c := range cases { + got, err := actionToSDK(c.action) + require.NoError(t, err) + assert.Equal(t, c.want, got) + } + + // Nothing is applied for these, so stagedOperations leaves them out rather than + // mapping them; the guard is here in case a new action type arrives without a mapping. + _, err := actionToSDK(deployplan.Skip) + assert.Error(t, err) + _, err = actionToSDK(deployplan.Undefined) + assert.Error(t, err) +} diff --git a/cmd/bundle/utils/process.go b/cmd/bundle/utils/process.go index c602ea1ec6f..73285862160 100644 --- a/cmd/bundle/utils/process.go +++ b/cmd/bundle/utils/process.go @@ -16,7 +16,6 @@ import ( "github.com/databricks/cli/bundle/deployplan" "github.com/databricks/cli/bundle/direct" "github.com/databricks/cli/bundle/direct/dstate" - "github.com/databricks/cli/bundle/env" "github.com/databricks/cli/bundle/phases" "github.com/databricks/cli/bundle/statemgmt" "github.com/databricks/cli/cmd/root" @@ -233,15 +232,20 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle // Recording makes the service the source of truth for state. var dmsSource *dstate.DMSSource - if env.RecordsDeploymentHistory(ctx, b.Config.Experimental != nil && b.Config.Experimental.RecordDeploymentHistory) { + if b.RecordsDeploymentHistory(ctx) { w := b.WorkspaceClient(ctx) deploymentID, err := dms.ResolveDeploymentID(ctx, w, b.Config.Workspace.StatePath) if err != nil { logdiag.LogError(ctx, err) return b, stateDesc, root.ErrAlreadyPrinted } + dmsClient, err := dms.NewClient(w) + if err != nil { + logdiag.LogError(ctx, err) + return b, stateDesc, root.ErrAlreadyPrinted + } dmsSource = &dstate.DMSSource{ - Client: w.BundleDeployments, + Client: dmsClient, DeploymentID: deploymentID, } diff --git a/libs/dms/client.go b/libs/dms/client.go new file mode 100644 index 00000000000..922bc1e281f --- /dev/null +++ b/libs/dms/client.go @@ -0,0 +1,146 @@ +package dms + +import ( + "context" + "fmt" + "net/http" + + "github.com/databricks/cli/libs/auth" + "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/client" + "github.com/databricks/databricks-sdk-go/service/bundledeployments" +) + +// Client is every call the CLI makes to DMS. Most go through the generated client; the two +// the SDK cannot express are written by hand below, each with its own interface so a test +// can capture what the CLI sends. +type Client struct { + // Service is the generated client, used for every call it can express. + Service bundledeployments.BundleDeploymentsInterface + + // Versions creates versions; see VersionCreator. + Versions VersionCreator + + // Operations fills in staged operations; see OperationUpdater. + Operations OperationUpdater +} + +// NewClient returns a Client for the workspace w. +func NewClient(w *databricks.WorkspaceClient) (*Client, error) { + api, err := client.New(w.Config) + if err != nil { + return nil, err + } + raw := &rawClient{client: api} + return &Client{Service: w.BundleDeployments, Versions: raw, Operations: raw}, nil +} + +// VersionCreator creates a version under a deployment. Hand-written because the generated +// struct has no previous_version_id, which the service needs as its concurrency check - +// without it every deploy after the first is rejected. +type VersionCreator interface { + CreateVersion(ctx context.Context, deploymentID, versionID string, body CreateVersionRequest) (*bundledeployments.Version, error) +} + +// OperationUpdater fills in an operation the version staged, and returns the sequence id the +// next update for that resource must send. Hand-written because the SDK types sequence_id as +// an int64 while the service sends a JSON string. TODO(DMS): drop once the spec agrees. +type OperationUpdater interface { + // sequenceID is the token the previous update for this resource returned, or 0 for the + // first, which is what staging leaves. + UpdateOperation(ctx context.Context, deploymentID string, version int64, key ResourceKey, sequenceID string, update OperationUpdate) (next string, err error) +} + +// CreateVersionRequest is the CreateVersion request body. +type CreateVersionRequest struct { + CliVersion string `json:"cli_version"` + VersionType VersionType `json:"version_type"` + TargetName string `json:"target_name,omitempty"` + // DisplayName names the deployment in the UI. The service keeps it on the + // deployment's node, so a version that omits it leaves the deployment unnamed. + DisplayName string `json:"display_name,omitempty"` + // PreviousVersionId is the deployment's most recent version, unset for a + // deployment's first version. + PreviousVersionId string `json:"previous_version_id,omitempty"` + // DeploymentMode is the bundle target's mode, unset when the target sets none. + DeploymentMode bundledeployments.DeploymentMode `json:"deployment_mode,omitempty"` + // GitInfo and WorkspaceInfo record where the deployed source came from and + // where it landed. The service denormalizes both onto the deployment. + GitInfo *bundledeployments.GitInfo `json:"git_info,omitempty"` + WorkspaceInfo *bundledeployments.WorkspaceInfo `json:"workspace_info,omitempty"` + // Operations is every resource this version will touch; see StagedOperation. + Operations []StagedOperation `json:"operations,omitempty"` +} + +// StagedOperation is one resource the version will record an operation for. The service +// creates it in OPERATION_STATUS_PENDING at sequence id 0, and the CLI fills in the outcome +// with UpdateOperation as the resource is applied. +type StagedOperation struct { + ResourceKey ResourceKey `json:"resource_key"` + ActionType bundledeployments.OperationActionType `json:"action_type"` +} + +// updateOperationRequest carries the values an update writes. action_type and resource_key +// are left out: the service fixes them when the version stages the operation. +type updateOperationRequest struct { + State string `json:"state,omitempty"` + ErrorMessage string `json:"error_message,omitempty"` + ResourceId string `json:"resource_id,omitempty"` + Status bundledeployments.OperationStatus `json:"status,omitempty"` + SequenceId string `json:"sequence_id,omitempty"` +} + +// operationResponse is the part of an operation response the CLI reads back. +type operationResponse struct { + // SequenceId is the concurrency token for the next update, typed as the service sends it. + SequenceId string `json:"sequence_id,omitempty"` +} + +// rawClient sends the requests the generated client cannot express. +type rawClient struct { + client *client.DatabricksClient +} + +func (r *rawClient) CreateVersion(ctx context.Context, deploymentID, versionID string, body CreateVersionRequest) (*bundledeployments.Version, error) { + var version bundledeployments.Version + path := fmt.Sprintf("/api/2.0/bundle/deployments/%s/versions", deploymentID) + err := r.client.Do(ctx, http.MethodPost, path, + auth.WorkspaceIDHeaders(r.client.Config), + map[string]any{"version_id": versionID}, + body, &version) + if err != nil { + return nil, err + } + return &version, nil +} + +// newUpdateRequest builds the request body for update. Only what the mask names is sent: +// the service would ignore the rest, and state is the largest field by far, so a failure +// that keeps the recorded state sends none. +func newUpdateRequest(update OperationUpdate, sequenceID string) updateOperationRequest { + body := updateOperationRequest{ + ErrorMessage: update.ErrorMessage, + Status: update.Status, + SequenceId: sequenceID, + } + if update.Fields.Has(FieldState) { + body.State = string(update.State) + body.ResourceId = update.ResourceID + } + return body +} + +func (r *rawClient) UpdateOperation(ctx context.Context, deploymentID string, version int64, key ResourceKey, sequenceID string, update OperationUpdate) (string, error) { + body := newUpdateRequest(update, sequenceID) + + var result operationResponse + path := fmt.Sprintf("/api/2.0/bundle/deployments/%s/versions/%d/operations/%s", deploymentID, version, key) + err := r.client.Do(ctx, http.MethodPatch, path, + auth.WorkspaceIDHeaders(r.client.Config), + map[string]any{"update_mask": update.Fields.Mask()}, + body, &result) + if err != nil { + return "", err + } + return result.SequenceId, nil +} diff --git a/libs/dms/doc.go b/libs/dms/doc.go new file mode 100644 index 00000000000..1ae54848d0d --- /dev/null +++ b/libs/dms/doc.go @@ -0,0 +1,20 @@ +// Package dms records bundle deployment history with the deployment metadata service. +// +// One deploy or destroy is one version under one deployment. CreateVersion stages an +// operation for every resource the plan touches, in OPERATION_STATUS_PENDING at sequence +// id 0, and the CLI fills each one in with UpdateOperation as the resource is applied. +// The set is fixed at CreateVersion: the service has no call to add one later, and it caps +// how many a version may stage. +// +// An update is taken literally. The field mask decides what changes, and a field left out +// keeps the value it had. The service mirrors state and resource_id onto the +// deployment-level resource, which is what the next plan reads, so a mask that names state +// with no value is what removes a resource - and a failure that must not disturb the state +// an earlier write left names only error_message and status. +// +// Every update carries the sequence id the previous one returned, starting from the 0 that +// staging leaves, so a write from a stale deploy is rejected rather than applied. +// +// Two calls are written by hand rather than taken from the SDK, which is generated from an +// OpenAPI spec that trails the service; see client.go. +package dms diff --git a/libs/dms/fields.go b/libs/dms/fields.go new file mode 100644 index 00000000000..e3bd271c3b1 --- /dev/null +++ b/libs/dms/fields.go @@ -0,0 +1,50 @@ +package dms + +import "strings" + +// Fields is the set of operation fields an update writes, sent as its update mask. The +// service rejects any other path, so this is the whole vocabulary. +type Fields uint8 + +const ( + FieldState Fields = 1 << iota + FieldErrorMessage + FieldResourceID + FieldStatus +) + +// DescribesResource is what a write that says how the resource looks claims: every field +// an update may change. +const DescribesResource = FieldState | FieldErrorMessage | FieldResourceID | FieldStatus + +// KeepsState is what a failure claims: mark it failed and leave state alone. State means +// the resource is as it was written; no state means a delete went through and nothing +// replaced it, so the resource really is gone and the deployment should say so. +const KeepsState = FieldErrorMessage | FieldStatus + +// wireNames pairs each field with its name on the wire, in the order a mask lists them. +var wireNames = []struct { + field Fields + name string +}{ + {FieldState, "state"}, + {FieldErrorMessage, "error_message"}, + {FieldResourceID, "resource_id"}, + {FieldStatus, "status"}, +} + +// Has reports whether f contains every field in other. +func (f Fields) Has(other Fields) bool { + return f&other == other +} + +// Mask renders f as the update_mask the service expects, always in the same order. +func (f Fields) Mask() string { + names := make([]string, 0, len(wireNames)) + for _, w := range wireNames { + if f.Has(w.field) { + names = append(names, w.name) + } + } + return strings.Join(names, ",") +} diff --git a/libs/dms/fields_test.go b/libs/dms/fields_test.go new file mode 100644 index 00000000000..198a55efca8 --- /dev/null +++ b/libs/dms/fields_test.go @@ -0,0 +1,23 @@ +package dms + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestFieldsMask(t *testing.T) { + // The order is fixed, so the same set always sends the same mask. + assert.Equal(t, "state,error_message,resource_id,status", DescribesResource.Mask()) + assert.Equal(t, "error_message,status", KeepsState.Mask()) + assert.Equal(t, "state", FieldState.Mask()) + assert.Empty(t, Fields(0).Mask()) +} + +func TestFieldsHas(t *testing.T) { + assert.True(t, DescribesResource.Has(FieldState)) + assert.False(t, KeepsState.Has(FieldState)) + // Has asks for every field, not any of them. + assert.True(t, DescribesResource.Has(FieldState|FieldStatus)) + assert.False(t, KeepsState.Has(FieldState|FieldStatus)) +} diff --git a/libs/dms/key.go b/libs/dms/key.go new file mode 100644 index 00000000000..9627a5b64c2 --- /dev/null +++ b/libs/dms/key.go @@ -0,0 +1,25 @@ +package dms + +import "strings" + +// statePrefix is what a bundle state key carries and a DMS resource key does not: state +// calls a job "resources.jobs.foo", DMS calls it "jobs.foo". +const statePrefix = "resources." + +// ResourceKey is how DMS names one resource. Its own type is the point: a state key sent +// as a DMS key records the operation under a name nothing reads back. +type ResourceKey string + +// KeyFromState converts a bundle state key to the key DMS knows the resource by. +func KeyFromState(stateKey string) ResourceKey { + return ResourceKey(strings.TrimPrefix(stateKey, statePrefix)) +} + +// StateKey converts back to the bundle state key. +func (k ResourceKey) StateKey() string { + return statePrefix + string(k) +} + +func (k ResourceKey) String() string { + return string(k) +} diff --git a/libs/dms/key_test.go b/libs/dms/key_test.go new file mode 100644 index 00000000000..34998ed3fcd --- /dev/null +++ b/libs/dms/key_test.go @@ -0,0 +1,18 @@ +package dms + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestResourceKeyDropsAndRestoresTheStatePrefix(t *testing.T) { + key := KeyFromState("resources.jobs.foo") + assert.Equal(t, ResourceKey("jobs.foo"), key) + assert.Equal(t, "resources.jobs.foo", key.StateKey()) +} + +func TestResourceKeyFromAKeyWithoutThePrefixIsUnchanged(t *testing.T) { + // The service never sends the prefix, so a key read back converts as-is. + assert.Equal(t, ResourceKey("jobs.foo"), KeyFromState("jobs.foo")) +} diff --git a/libs/dms/operation.go b/libs/dms/operation.go new file mode 100644 index 00000000000..6a08251303f --- /dev/null +++ b/libs/dms/operation.go @@ -0,0 +1,111 @@ +package dms + +import ( + "encoding/json" + "fmt" + "unicode/utf8" + + "github.com/databricks/cli/libs/diag" + "github.com/databricks/databricks-sdk-go/service/bundledeployments" +) + +// maxStateSize is the largest serialized state DMS accepts per operation. More than this +// and the resource cannot be recorded at all, so the deploy fails rather than leaving the +// service holding a resource with no state. +const maxStateSize = 64 * 1024 + +// maxErrorMessageSize is how much of a failure's message the service stores. +const maxErrorMessageSize = 16 * 1024 + +// StatusInProgress marks an operation whose writes are not finished. Not taken from the +// SDK: the enum is generated from the OpenAPI spec, which trails the service proto +// (databricks-eng/universe#2394529). +const StatusInProgress bundledeployments.OperationStatus = "OPERATION_STATUS_IN_PROGRESS" + +// OperationUpdate is one write to an operation the version staged: the fields it claims +// and their values. It is built where the outcome is known, so a malformed one fails the +// resource that produced it rather than the upload at the end of apply. +type OperationUpdate struct { + // Fields is the mask to send. It is taken literally: a field named here is written, + // one left out keeps its value. See the package doc. + Fields Fields + + // State is the serialized state after the operation, and nil for a delete. + State json.RawMessage + + ResourceID string + Status bundledeployments.OperationStatus + ErrorMessage string +} + +// NewStateUpdate describes how the resource looks now: state is the serialized envelope the +// state DB just persisted, and nil for a delete. inProgress marks a write that is half of a +// larger change - a recreate's delete - so an interrupted deploy does not report it finished. +func NewStateUpdate(resourceID string, state json.RawMessage, inProgress bool) (OperationUpdate, error) { + if len(state) > maxStateSize { + return OperationUpdate{}, fmt.Errorf("serialized state is %d bytes, which exceeds the %d byte limit for recording deployment history", len(state), maxStateSize) + } + + status := bundledeployments.OperationStatusOperationStatusSucceeded + if inProgress { + status = StatusInProgress + } + + return OperationUpdate{ + Fields: DescribesResource, + State: state, + ResourceID: resourceID, + Status: status, + }, nil +} + +// NewFailureUpdate records that an operation did not apply, so the history says why a +// resource failed rather than leaving it pending. It claims no state, so whatever an +// earlier write recorded stands. +func NewFailureUpdate(resourceID string, cause error) OperationUpdate { + // Summarized, not cause.Error(): for an API failure that adds the status and error + // code, which is often the most actionable part of the history. + message := diag.FormatAPIErrorSummary(cause) + if len(message) > maxErrorMessageSize { + message = message[:maxErrorMessageSize] + // The cut can land inside a rune, and the service stores a string. Drop the partial + // one: at most UTFMax-1 bytes of it can be left, so a message that was already + // invalid loses those bytes rather than being stripped away entirely. + for range utf8.UTFMax - 1 { + if utf8.ValidString(message) { + break + } + message = message[:len(message)-1] + } + } + + return OperationUpdate{ + Fields: KeepsState, + ResourceID: resourceID, + Status: bundledeployments.OperationStatusOperationStatusFailed, + ErrorMessage: message, + } +} + +// Merge folds a later update into u, for a resource written twice before either upload ran. +// Each field comes from whichever update claimed it, newer winning when both did, and the +// mask is the union. What an update claims is decided where it is built, not here. +func (u OperationUpdate) Merge(newer OperationUpdate) OperationUpdate { + merged := u + merged.Fields = u.Fields | newer.Fields + + if newer.Fields.Has(FieldState) { + merged.State = newer.State + } + if newer.Fields.Has(FieldResourceID) { + merged.ResourceID = newer.ResourceID + } + if newer.Fields.Has(FieldErrorMessage) { + merged.ErrorMessage = newer.ErrorMessage + } + if newer.Fields.Has(FieldStatus) { + merged.Status = newer.Status + } + + return merged +} diff --git a/libs/dms/operation_test.go b/libs/dms/operation_test.go new file mode 100644 index 00000000000..d48c63a334e --- /dev/null +++ b/libs/dms/operation_test.go @@ -0,0 +1,119 @@ +package dms + +import ( + "encoding/json" + "errors" + "strings" + "testing" + "unicode/utf8" + + "github.com/databricks/databricks-sdk-go/service/bundledeployments" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestNewStateUpdateRecordsEnvelopeAsIs(t *testing.T) { + // The state DB serializes the envelope (see dstate.SaveState); the update carries it + // through untouched, sensitive fields and all. + state := json.RawMessage(`{"state":{"name":"foo","token":"super-secret"}}`) + + update, err := NewStateUpdate("job-123", state, false) + require.NoError(t, err) + + assert.JSONEq(t, string(state), string(update.State)) + assert.Equal(t, bundledeployments.OperationStatusOperationStatusSucceeded, update.Status) + assert.Equal(t, DescribesResource, update.Fields) +} + +func TestNewStateUpdateInProgressIsNotFinished(t *testing.T) { + // A recreate's delete is half of a larger change, so an interrupted deploy must not + // leave the resource described as finished. + update, err := NewStateUpdate("", nil, true) + require.NoError(t, err) + + assert.Equal(t, StatusInProgress, update.Status) +} + +func TestNewStateUpdateRejectsOversizedState(t *testing.T) { + big := json.RawMessage(strings.Repeat("x", maxStateSize+1)) + + _, err := NewStateUpdate("job-123", big, false) + assert.ErrorContains(t, err, "exceeds the 65536 byte limit") +} + +func TestNewFailureUpdateRecordsError(t *testing.T) { + update := NewFailureUpdate("", errors.New("cluster spec is invalid")) + + assert.Equal(t, bundledeployments.OperationStatusOperationStatusFailed, update.Status) + assert.Equal(t, "cluster spec is invalid", update.ErrorMessage) + // The resource was never written, so there is no state to serve back for it. + assert.Nil(t, update.State) + // The update only marks the operation failed; see KeepsState. + assert.Equal(t, KeepsState, update.Fields) +} + +func TestNewFailureUpdateTruncatesLongError(t *testing.T) { + // Truncated rather than rejected: a message over the limit would make recording + // fail and hide the error it is reporting. + update := NewFailureUpdate("job-123", errors.New(strings.Repeat("x", maxErrorMessageSize+100))) + + assert.Len(t, update.ErrorMessage, maxErrorMessageSize) +} + +func TestNewFailureUpdatePreservesUTF8OnTruncation(t *testing.T) { + // The cut lands one byte into the emoji, so a byte-wise truncation would leave a partial + // rune behind and the service stores state and messages as strings. + msg := strings.Repeat("a", maxErrorMessageSize-1) + "❌" + "x" + + update := NewFailureUpdate("job-123", errors.New(msg)) + + assert.True(t, utf8.ValidString(update.ErrorMessage)) + // The whole emoji went, so the message is shorter than the limit rather than exactly it. + assert.Equal(t, strings.Repeat("a", maxErrorMessageSize-1), update.ErrorMessage) +} + +func TestMergeLetsAWriteSupersedeAFailure(t *testing.T) { + // A failure is not the last word. A retry that writes state wins whole - state, id and + // mask - and the mask names error_message so the recorded failure is cleared. The service + // rejects a succeeded operation that still carries an error. + failed := NewFailureUpdate("id-old", errors.New("boom")) + retried, err := NewStateUpdate("id-new", json.RawMessage(`{"state":{"name":"after"}}`), false) + require.NoError(t, err) + + merged := failed.Merge(retried) + + assert.Equal(t, bundledeployments.OperationStatusOperationStatusSucceeded, merged.Status) + assert.Empty(t, merged.ErrorMessage) + assert.Equal(t, "id-new", merged.ResourceID) + assert.JSONEq(t, `{"state":{"name":"after"}}`, string(merged.State)) + assert.Equal(t, DescribesResource, merged.Fields) +} + +func TestMergeKeepsTheWritesStateAndMask(t *testing.T) { + // A failure claims only status and error_message, so the write's state, id and mask + // survive. + write, err := NewStateUpdate("id-new", nil, false) + require.NoError(t, err) + failed := NewFailureUpdate("id-old", errors.New("boom")) + + merged := write.Merge(failed) + + assert.Equal(t, bundledeployments.OperationStatusOperationStatusFailed, merged.Status) + assert.Equal(t, "boom", merged.ErrorMessage) + assert.Equal(t, "id-new", merged.ResourceID) + assert.Nil(t, merged.State) + assert.Equal(t, DescribesResource, merged.Fields) +} + +func TestMergeLetsADeleteClearTheState(t *testing.T) { + // A delete legitimately carries no state, and merging must let it through: the resource + // is gone, and keeping the state it replaces would leave it listed. + write, err := NewStateUpdate("id-1", json.RawMessage(`{"state":{"name":"before"}}`), false) + require.NoError(t, err) + deleted, err := NewStateUpdate("id-1", nil, false) + require.NoError(t, err) + + merged := write.Merge(deleted) + + assert.Nil(t, merged.State) +} diff --git a/libs/dms/recorder.go b/libs/dms/recording.go similarity index 51% rename from libs/dms/recorder.go rename to libs/dms/recording.go index 7b7ea499710..7e94bff6e8b 100644 --- a/libs/dms/recorder.go +++ b/libs/dms/recording.go @@ -10,10 +10,8 @@ import ( "time" "github.com/databricks/cli/internal/build" - "github.com/databricks/cli/libs/auth" "github.com/databricks/cli/libs/log" "github.com/databricks/databricks-sdk-go/apierr" - "github.com/databricks/databricks-sdk-go/client" "github.com/databricks/databricks-sdk-go/service/bundledeployments" ) @@ -29,110 +27,49 @@ const ( VersionTypeDestroy VersionType = bundledeployments.VersionTypeVersionTypeDestroy ) -// StagedOperation is one resource the version will record an operation for. The service -// creates it in OPERATION_STATUS_PENDING at sequence_id 0, and the CLI fills in the outcome -// with UpdateOperation as the resource is applied. -// -// Hand-written for the same reason as createVersionRequest: the SDK is generated from the -// OpenAPI spec, which does not carry this message yet. -type StagedOperation struct { - // ResourceKey is the DMS form, without the CLI's "resources." prefix (e.g. "jobs.foo"). - // The service requires a known resource-type prefix and rejects duplicates. - ResourceKey string `json:"resource_key"` - ActionType bundledeployments.OperationActionType `json:"action_type"` +// Recording is what one deploy or destroy records with DMS: a version, the operations it +// stages, and their outcomes. A disabled recording is a no-op throughout, so callers do not +// branch on whether recording is on. +type Recording interface { + // Enabled reports whether anything is recorded. Only a caller that would otherwise do + // pointless work - serializing state on every write - needs to ask. + Enabled() bool + + // Prepare settles the deployment and the version number this run will create, without + // creating it. Both are needed before the plan, which the version number is stamped onto. + Prepare(ctx context.Context) error + + // DeploymentID is the deployment being recorded under. Empty until Prepare, which + // creates the deployment on a first deploy. + DeploymentID() string + + // Version is the version number Prepare claimed, and zero before it runs. + Version() int64 + + // Start creates the version, staging an operation for each resource, and returns the + // writer that fills them in. The staged set is fixed here: the service has no call to + // add one later, so a resource left out can never be recorded. + Start(ctx context.Context, staged []StagedOperation) (OperationWriter, error) + + // Finish completes the version. It is a no-op before Start, which is what lets a caller + // defer it and still not complete a version a cancelled deploy never created, and it is + // safe to call twice. + Finish(ctx context.Context, success bool) error } -// createVersionRequest is the CreateVersion request body. Hand-written because the -// generated struct has no previous_version_id, which the service needs as its -// concurrency check - without it every deploy after the first is rejected. -type createVersionRequest struct { - CliVersion string `json:"cli_version"` - VersionType VersionType `json:"version_type"` - TargetName string `json:"target_name,omitempty"` - // DisplayName names the deployment in the UI. The service keeps it on the - // deployment's node, so a version that omits it leaves the deployment unnamed. - DisplayName string `json:"display_name,omitempty"` - // PreviousVersionId is the deployment's most recent version, unset for a - // deployment's first version. - PreviousVersionId string `json:"previous_version_id,omitempty"` - // DeploymentMode is the bundle target's mode, unset when the target sets none. - DeploymentMode bundledeployments.DeploymentMode `json:"deployment_mode,omitempty"` - // GitInfo and WorkspaceInfo record where the deployed source came from and - // where it landed. The service denormalizes both onto the deployment. - GitInfo *bundledeployments.GitInfo `json:"git_info,omitempty"` - WorkspaceInfo *bundledeployments.WorkspaceInfo `json:"workspace_info,omitempty"` - // Operations is every resource this version will touch. The set is fixed here: the - // service has no API to add one later, so a resource left out cannot be recorded. - Operations []StagedOperation `json:"operations,omitempty"` -} - -// versionCreator creates a version under a deployment. It exists because the -// generated client cannot express the request body (see createVersionRequest). -type versionCreator interface { - CreateVersion(ctx context.Context, deploymentID, versionID string, body createVersionRequest) (*bundledeployments.Version, error) -} - -// apiVersionCreator creates versions through the workspace API client. -type apiVersionCreator struct { - client *client.DatabricksClient -} - -// NewAPIVersionCreator returns a versionCreator that posts to the DMS API. -func NewAPIVersionCreator(c *client.DatabricksClient) versionCreator { - return &apiVersionCreator{client: c} -} - -func (a *apiVersionCreator) CreateVersion(ctx context.Context, deploymentID, versionID string, body createVersionRequest) (*bundledeployments.Version, error) { - var version bundledeployments.Version - path := fmt.Sprintf("/api/2.0/bundle/deployments/%s/versions", deploymentID) - err := a.client.Do(ctx, http.MethodPost, path, - auth.WorkspaceIDHeaders(a.client.Config), - map[string]any{"version_id": versionID}, - body, &version) - if err != nil { - return nil, err - } - return &version, nil -} +// RecordingOptions are the dependencies and deployment identity a Recording needs. +type RecordingOptions struct { + Client *Client -// Recorder records a deploy/destroy as a version with DMS. Server assigns ID on first deploy. -type Recorder struct { - svc bundledeployments.BundleDeploymentsInterface - versions versionCreator - deploymentID string - statePath string - versionType VersionType - metadata Metadata - - // populated by PrepareDeployment: the version number this deploy intends to - // create. It is known before the version exists so it can be stamped onto the - // resources the plan is computed from. - versionNum int64 - previousVersionID string - - // populated by CreateVersion, once the version actually exists. A deploy the user - // declines never gets here, so there is nothing to complete or heartbeat. - versionCreated bool - stopHeartbeat context.CancelFunc - - // completed makes CompleteVersion idempotent, so a caller that completes the - // version early can still defer it unconditionally. - completed bool -} - -// RecorderOptions are the dependencies and deployment identity a Recorder needs. -type RecorderOptions struct { - // Service handles every DMS call except CreateVersion. - Service bundledeployments.BundleDeploymentsInterface - // Versions handles CreateVersion; see versionCreator. - Versions versionCreator // DeploymentID is resolved from the deployment's workspace node, empty until the - // first recorded deploy (CreateVersion assigns one then). + // first recorded deploy (Prepare creates the deployment then). DeploymentID string + // StatePath is the bundle's remote state directory, under which DMS registers // the deployment node. StatePath string VersionType VersionType + // Metadata is what the version records about the deploy; see Metadata. Metadata Metadata } @@ -151,11 +88,10 @@ type Metadata struct { Workspace *bundledeployments.WorkspaceInfo } -// NewRecorder returns a Recorder for the deployment described by opts. -func NewRecorder(opts RecorderOptions) *Recorder { - return &Recorder{ - svc: opts.Service, - versions: opts.Versions, +// NewRecording returns a Recording for the deployment described by opts. +func NewRecording(opts RecordingOptions) Recording { + return &recording{ + client: opts.Client, deploymentID: opts.DeploymentID, statePath: opts.StatePath, versionType: opts.VersionType, @@ -163,37 +99,78 @@ func NewRecorder(opts RecorderOptions) *Recorder { } } -// DeploymentID returns the DMS deployment ID this recorder is bound to. It is -// empty until CreateVersion has created the deployment record (on a first -// deploy) and non-empty afterwards, so callers can parent operations under it. -func (r *Recorder) DeploymentID() string { - if r == nil { - return "" - } - return r.deploymentID +// Disabled returns a Recording that records nothing, for a bundle that does not record +// deployment history. +func Disabled() Recording { + return disabled{} } -// Version returns the version number claimed by CreateVersion. It is zero until -// CreateVersion has run; callers use it to parent operations under the version. -func (r *Recorder) Version() int64 { - if r == nil { - return 0 - } - return r.versionNum +// disabled records nothing. Its Prepare leaves no deployment and no version, so DeploymentID +// and Version stay empty for a caller that stamps them onto resources. +type disabled struct{} + +func (disabled) Enabled() bool { return false } +func (disabled) Prepare(context.Context) error { return nil } +func (disabled) DeploymentID() string { return "" } +func (disabled) Version() int64 { return 0 } +func (disabled) Finish(context.Context, bool) error { return nil } + +func (disabled) Start(context.Context, []StagedOperation) (OperationWriter, error) { + return noopWriter{}, nil } -// CreateVersion registers a new version with DMS, claiming it for the deployment, and stages -// an operation for every resource in operations. The set cannot be added to later, so a -// resource left out here can never be recorded. Nil Recorder is a no-op. -func (r *Recorder) CreateVersion(ctx context.Context, operations []StagedOperation) error { - if r == nil { - return nil +// recording records with the service. +type recording struct { + client *Client + deploymentID string + statePath string + versionType VersionType + metadata Metadata + + // populated by Prepare: the version number this deploy intends to create. It is known + // before the version exists so it can be stamped onto the resources the plan is + // computed from. + versionNum int64 + previousVersionID string + + // populated by Start, once the version actually exists. A deploy the user declines + // never gets here, so there is nothing to complete or heartbeat. + versionCreated bool + stopHeartbeat context.CancelFunc + + // completed makes Finish idempotent, so a caller that completes the version early can + // still defer it unconditionally. + completed bool +} + +func (r *recording) Enabled() bool { return true } + +func (r *recording) DeploymentID() string { return r.deploymentID } + +func (r *recording) Version() int64 { return r.versionNum } + +// Prepare implements Recording. +func (r *recording) Prepare(ctx context.Context) error { + versionID, err := r.resolveNextVersion(ctx) + if err != nil { + return err } - // A deploy calls PrepareDeployment first, because it needs the version number to - // stamp onto the plan. A destroy has no such need, so settle it here instead. + + versionNum, err := strconv.ParseInt(versionID, 10, 64) + if err != nil { + return fmt.Errorf("failed to parse version ID %q: %w", versionID, err) + } + r.versionNum = versionNum + return nil +} + +// Start implements Recording. +func (r *recording) Start(ctx context.Context, staged []StagedOperation) (OperationWriter, error) { + // A deploy calls Prepare first, because it needs the version number to stamp onto the + // plan. A destroy has no such need, so settle it here instead. if r.versionNum == 0 { - if err := r.PrepareDeployment(ctx); err != nil { - return err + if err := r.Prepare(ctx); err != nil { + return nil, err } } @@ -202,14 +179,14 @@ func (r *Recorder) CreateVersion(ctx context.Context, operations []StagedOperati // The server rejects this unless versionID exceeds last_version_id and // previous_version_id matches it, which is what makes claiming the number up front // safe: a deploy that took it in the meantime is reported, not overwritten. - version, err := r.versions.CreateVersion(ctx, r.deploymentID, versionID, createVersionRequest{ + version, err := r.client.Versions.CreateVersion(ctx, r.deploymentID, versionID, CreateVersionRequest{ CliVersion: build.GetInfo().Version, VersionType: r.versionType, TargetName: r.metadata.TargetName, DisplayName: r.metadata.DisplayName, PreviousVersionId: r.previousVersionID, DeploymentMode: r.metadata.Mode, - Operations: operations, + Operations: staged, GitInfo: r.metadata.Git, WorkspaceInfo: r.metadata.Workspace, }) @@ -217,35 +194,36 @@ func (r *Recorder) CreateVersion(ctx context.Context, operations []StagedOperati // The service caps how many operations one version may stage, so a bundle past the // cap cannot be recorded at all. Say so rather than passing the raw API error on. if isResourceExhaustedErr(err) { - return fmt.Errorf("this bundle deploys %d resources, more than the deployment metadata service records in one version: %w", len(operations), err) + return nil, fmt.Errorf("this bundle deploys %d resources, more than the deployment metadata service records in one version: %w", len(staged), err) } // A 409 ABORTED means another deploy claimed this version number in between - // PrepareDeployment and here. + // Prepare and here. if isAbortedErr(err) { - return fmt.Errorf("another deploy already claimed version %s of this deployment, try again: %w", versionID, err) + return nil, fmt.Errorf("another deploy already claimed version %s of this deployment, try again: %w", versionID, err) } - return fmt.Errorf("failed to create deployment version: %w", err) + return nil, fmt.Errorf("failed to create deployment version: %w", err) } r.versionCreated = true - r.stopHeartbeat = startHeartbeat(ctx, r.svc, r.deploymentID, versionID) + r.stopHeartbeat = startHeartbeat(ctx, r.client.Service, r.deploymentID, versionID) log.Infof(ctx, "Created deployment version: deployment=%s version=%s", r.deploymentID, version.VersionId) - return nil + + return &operationWriter{ + ops: r.client.Operations, + deploymentID: r.deploymentID, + version: r.versionNum, + sequenceIDs: make(map[ResourceKey]string), + }, nil } -// CompleteVersion finalizes the version created by CreateVersion. It is a no-op -// when CreateVersion never ran, which is what lets callers defer it and still not -// complete a version a cancelled deploy never created. -func (r *Recorder) CompleteVersion(ctx context.Context, success bool) error { +// Finish implements Recording. +func (r *recording) Finish(ctx context.Context, success bool) error { reason := bundledeployments.VersionCompleteVersionCompleteSuccess if !success { reason = bundledeployments.VersionCompleteVersionCompleteFailure } - return r.completeVersion(ctx, reason) -} -func (r *Recorder) completeVersion(ctx context.Context, reason bundledeployments.VersionComplete) error { - if r == nil || !r.versionCreated || r.completed { + if !r.versionCreated || r.completed { return nil } r.completed = true @@ -255,7 +233,7 @@ func (r *Recorder) completeVersion(ctx context.Context, reason bundledeployments versionIDStr := strconv.FormatInt(r.versionNum, 10) versionName := fmt.Sprintf("deployments/%s/versions/%s", r.deploymentID, versionIDStr) - _, err := r.svc.CompleteVersion(ctx, bundledeployments.CompleteVersionRequest{ + _, err := r.client.Service.CompleteVersion(ctx, bundledeployments.CompleteVersionRequest{ Name: versionName, CompletionReason: reason, }) @@ -267,7 +245,7 @@ func (r *Recorder) completeVersion(ctx context.Context, reason bundledeployments // For destroy operations, delete the deployment record after the version // completes successfully. if reason == bundledeployments.VersionCompleteVersionCompleteSuccess && r.versionType == VersionTypeDestroy { - err = r.svc.DeleteDeployment(ctx, bundledeployments.DeleteDeploymentRequest{ + err = r.client.Service.DeleteDeployment(ctx, bundledeployments.DeleteDeploymentRequest{ Name: "deployments/" + r.deploymentID, }) if err != nil { @@ -278,34 +256,14 @@ func (r *Recorder) completeVersion(ctx context.Context, reason bundledeployments return nil } -// PrepareDeployment ensures the deployment exists and determines the version number to create, -// without creating it. Both are needed before planning; version itself is created by CreateVersion. -func (r *Recorder) PrepareDeployment(ctx context.Context) error { - if r == nil { - return nil - } - - versionID, err := r.resolveNextVersion(ctx) - if err != nil { - return err - } - - versionNum, err := strconv.ParseInt(versionID, 10, 64) - if err != nil { - return fmt.Errorf("failed to parse version ID %q: %w", versionID, err) - } - r.versionNum = versionNum - return nil -} - // resolveNextVersion creates the deployment if this is the first deploy, and returns // the version ID to create under it. -func (r *Recorder) resolveNextVersion(ctx context.Context) (versionID string, err error) { +func (r *recording) resolveNextVersion(ctx context.Context) (versionID string, err error) { if r.deploymentID != "" { // The ID came from a BUNDLE_DEPLOYMENT node that get-status returned, and by // design the service has a deployment for every such node, so a not-found // here means that invariant is broken rather than anything the user did. - dep, getErr := r.svc.GetDeployment(ctx, bundledeployments.GetDeploymentRequest{ + dep, getErr := r.client.Service.GetDeployment(ctx, bundledeployments.GetDeploymentRequest{ Name: "deployments/" + r.deploymentID, }) switch { @@ -329,7 +287,7 @@ func (r *Recorder) resolveNextVersion(ctx context.Context) (versionID string, er // First deploy: create the deployment so the server assigns an ID. // initial_parent_path is required - the node the service creates under it is // what ResolveDeploymentID reads back later. - dep, createErr := r.svc.CreateDeployment(ctx, bundledeployments.CreateDeploymentRequest{ + dep, createErr := r.client.Service.CreateDeployment(ctx, bundledeployments.CreateDeploymentRequest{ Deployment: bundledeployments.Deployment{ InitialParentPath: r.statePath, TargetName: r.metadata.TargetName, @@ -393,13 +351,13 @@ func startHeartbeat(ctx context.Context, svc bundledeployments.BundleDeployments return cancel } -// isAbortedErr reports whether err is an HTTP 409 ABORTED from the DMS API. // isResourceExhaustedErr reports whether the service refused the call for exceeding a quota. func isResourceExhaustedErr(err error) bool { apiErr, ok := errors.AsType[*apierr.APIError](err) return ok && apiErr.ErrorCode == "RESOURCE_EXHAUSTED" } +// isAbortedErr reports whether err is an HTTP 409 ABORTED from the DMS API. func isAbortedErr(err error) bool { apiErr, ok := errors.AsType[*apierr.APIError](err) return ok && apiErr.StatusCode == http.StatusConflict && apiErr.ErrorCode == "ABORTED" diff --git a/libs/dms/recorder_test.go b/libs/dms/recording_test.go similarity index 65% rename from libs/dms/recorder_test.go rename to libs/dms/recording_test.go index 72ccc0bf2ef..0b4c0689cdf 100644 --- a/libs/dms/recorder_test.go +++ b/libs/dms/recording_test.go @@ -43,18 +43,18 @@ type fakeDMS struct { type fakeVersionRequest struct { deploymentID string versionID string - body createVersionRequest + body CreateVersionRequest } // fakeVersions captures CreateVersion calls. It is separate from fakeDMS because // the CLI does not create versions through the generated client (see -// createVersionRequest), so the two use different signatures. +// CreateVersionRequest), so the two use different signatures. type fakeVersions struct { requests *[]fakeVersionRequest err error } -func (f fakeVersions) CreateVersion(ctx context.Context, deploymentID, versionID string, body createVersionRequest) (*bundledeployments.Version, error) { +func (f fakeVersions) CreateVersion(ctx context.Context, deploymentID, versionID string, body CreateVersionRequest) (*bundledeployments.Version, error) { *f.requests = append(*f.requests, fakeVersionRequest{deploymentID: deploymentID, versionID: versionID, body: body}) if f.err != nil { return nil, f.err @@ -88,12 +88,25 @@ func (f *fakeDMS) Heartbeat(ctx context.Context, req bundledeployments.Heartbeat return &bundledeployments.HeartbeatResponse{}, nil } -func TestRecorderFirstDeployCreatesDeploymentWithServerAssignedID(t *testing.T) { +// testClient wires a Recording to f, failing CreateVersion with versionErr when set. +func testClient(f *fakeDMS, versionErr error) *Client { + return &Client{Service: f, Versions: fakeVersions{requests: &f.versions, err: versionErr}} +} + +// startVersion creates the version and discards the writer, for a test that only asserts +// what reached the service. +func startVersion(t *testing.T, r Recording, staged []StagedOperation) { + t.Helper() + _, err := r.Start(t.Context(), staged) + require.NoError(t, err) +} + +func TestRecordingFirstDeployCreatesDeploymentWithServerAssignedID(t *testing.T) { f := &fakeDMS{assignedID: "server-generated-id"} // A first deploy resolves no deployment ID from the workspace. - r := NewRecorder(RecorderOptions{Service: f, Versions: fakeVersions{requests: &f.versions}, StatePath: testStatePath, Metadata: Metadata{TargetName: "dev", DisplayName: testDisplayName}, VersionType: VersionTypeDeploy}) + r := NewRecording(RecordingOptions{Client: testClient(f, nil), StatePath: testStatePath, Metadata: Metadata{TargetName: "dev", DisplayName: testDisplayName}, VersionType: VersionTypeDeploy}) - require.NoError(t, r.CreateVersion(t.Context(), nil)) + startVersion(t, r, nil) // The server assigned the ID, and the recorder exposes it for the rest of the // deploy (it parents the operations recorded under this version). @@ -116,22 +129,22 @@ func TestRecorderFirstDeployCreatesDeploymentWithServerAssignedID(t *testing.T) // A first version supersedes nothing, so previous_version_id is unset. assert.Empty(t, f.versions[0].body.PreviousVersionId) - require.NoError(t, r.CompleteVersion(t.Context(), true)) + require.NoError(t, r.Finish(t.Context(), true)) require.Len(t, f.completed, 1) assert.Equal(t, bundledeployments.VersionCompleteVersionCompleteSuccess, f.completed[0].CompletionReason) assert.Empty(t, f.deleted) } -func TestRecorderSubsequentDeployReusesDeploymentAndIncrementsVersion(t *testing.T) { +func TestRecordingSubsequentDeployReusesDeploymentAndIncrementsVersion(t *testing.T) { f := &fakeDMS{ getDeployment: func(id string) (*bundledeployments.Deployment, error) { return &bundledeployments.Deployment{Name: "deployments/" + id, LastVersionId: "4"}, nil }, } // A subsequent deploy passes the stored deployment ID. - r := NewRecorder(RecorderOptions{Service: f, Versions: fakeVersions{requests: &f.versions}, DeploymentID: "stored-id", StatePath: testStatePath, Metadata: Metadata{TargetName: "dev", DisplayName: testDisplayName}, VersionType: VersionTypeDeploy}) + r := NewRecording(RecordingOptions{Client: testClient(f, nil), DeploymentID: "stored-id", StatePath: testStatePath, Metadata: Metadata{TargetName: "dev", DisplayName: testDisplayName}, VersionType: VersionTypeDeploy}) - require.NoError(t, r.CreateVersion(t.Context(), nil)) + startVersion(t, r, nil) // No new deployment is created; the version increments to last_version_id + 1. assert.Empty(t, f.created) @@ -143,20 +156,20 @@ func TestRecorderSubsequentDeployReusesDeploymentAndIncrementsVersion(t *testing assert.Equal(t, "4", f.versions[0].body.PreviousVersionId) } -func TestRecorderGetDeploymentErrorFailsDeploy(t *testing.T) { +func TestRecordingGetDeploymentErrorFailsDeploy(t *testing.T) { f := &fakeDMS{ getDeployment: func(id string) (*bundledeployments.Deployment, error) { return nil, errors.New("boom") }, } - r := NewRecorder(RecorderOptions{Service: f, Versions: fakeVersions{requests: &f.versions}, DeploymentID: "stored-id", StatePath: testStatePath, Metadata: Metadata{TargetName: "dev", DisplayName: testDisplayName}, VersionType: VersionTypeDeploy}) + r := NewRecording(RecordingOptions{Client: testClient(f, nil), DeploymentID: "stored-id", StatePath: testStatePath, Metadata: Metadata{TargetName: "dev", DisplayName: testDisplayName}, VersionType: VersionTypeDeploy}) - err := r.CreateVersion(t.Context(), nil) + _, err := r.Start(t.Context(), nil) assert.ErrorContains(t, err, "failed to get deployment") assert.Empty(t, f.created) } -func TestRecorderMissingDeploymentIsInternalError(t *testing.T) { +func TestRecordingMissingDeploymentIsInternalError(t *testing.T) { // The service has a deployment for every BUNDLE_DEPLOYMENT node, so a not-found // for a node get-status just returned is a broken invariant, not a state the // deploy can recover from. @@ -165,83 +178,90 @@ func TestRecorderMissingDeploymentIsInternalError(t *testing.T) { return nil, fmt.Errorf("deployment: %w", apierr.ErrNotFound) }, } - r := NewRecorder(RecorderOptions{Service: f, Versions: fakeVersions{requests: &f.versions}, DeploymentID: "stored-id", StatePath: testStatePath, Metadata: Metadata{TargetName: "dev", DisplayName: testDisplayName}, VersionType: VersionTypeDeploy}) + r := NewRecording(RecordingOptions{Client: testClient(f, nil), DeploymentID: "stored-id", StatePath: testStatePath, Metadata: Metadata{TargetName: "dev", DisplayName: testDisplayName}, VersionType: VersionTypeDeploy}) - err := r.CreateVersion(t.Context(), nil) + _, err := r.Start(t.Context(), nil) assert.ErrorContains(t, err, "internal error: no deployment found for the file with object id stored-id") assert.Empty(t, f.created) assert.Empty(t, f.versions) } -func TestRecorderDestroyDeletesDeploymentOnSuccess(t *testing.T) { +func TestRecordingDestroyDeletesDeploymentOnSuccess(t *testing.T) { f := &fakeDMS{ getDeployment: func(id string) (*bundledeployments.Deployment, error) { return &bundledeployments.Deployment{Name: "deployments/" + id, LastVersionId: "2"}, nil }, } - r := NewRecorder(RecorderOptions{Service: f, Versions: fakeVersions{requests: &f.versions}, DeploymentID: "stored-id", StatePath: testStatePath, Metadata: Metadata{TargetName: "dev", DisplayName: testDisplayName}, VersionType: VersionTypeDestroy}) + r := NewRecording(RecordingOptions{Client: testClient(f, nil), DeploymentID: "stored-id", StatePath: testStatePath, Metadata: Metadata{TargetName: "dev", DisplayName: testDisplayName}, VersionType: VersionTypeDestroy}) - require.NoError(t, r.CreateVersion(t.Context(), nil)) + startVersion(t, r, nil) assert.Equal(t, bundledeployments.VersionTypeVersionTypeDestroy, f.versions[0].body.VersionType) - require.NoError(t, r.CompleteVersion(t.Context(), true)) + require.NoError(t, r.Finish(t.Context(), true)) // A successful destroy deletes the deployment record. require.Equal(t, []string{"deployments/stored-id"}, f.deleted) } -func TestRecorderFailedDestroyKeepsDeployment(t *testing.T) { +func TestRecordingFailedDestroyKeepsDeployment(t *testing.T) { f := &fakeDMS{ getDeployment: func(id string) (*bundledeployments.Deployment, error) { return &bundledeployments.Deployment{Name: "deployments/" + id, LastVersionId: "2"}, nil }, } - r := NewRecorder(RecorderOptions{Service: f, Versions: fakeVersions{requests: &f.versions}, DeploymentID: "stored-id", StatePath: testStatePath, Metadata: Metadata{TargetName: "dev", DisplayName: testDisplayName}, VersionType: VersionTypeDestroy}) + r := NewRecording(RecordingOptions{Client: testClient(f, nil), DeploymentID: "stored-id", StatePath: testStatePath, Metadata: Metadata{TargetName: "dev", DisplayName: testDisplayName}, VersionType: VersionTypeDestroy}) - require.NoError(t, r.CreateVersion(t.Context(), nil)) - require.NoError(t, r.CompleteVersion(t.Context(), false)) + startVersion(t, r, nil) + require.NoError(t, r.Finish(t.Context(), false)) assert.Equal(t, bundledeployments.VersionCompleteVersionCompleteFailure, f.completed[0].CompletionReason) // A failed destroy leaves the deployment in place. assert.Empty(t, f.deleted) } -func TestRecorderCompleteVersionIsIdempotent(t *testing.T) { +func TestRecordingFinishIsIdempotent(t *testing.T) { // Destroy completes the version before deleting the remote files, because that - // deletes the deployment's node, and still defers CompleteVersion. The second + // deletes the deployment's node, and still defers Finish. The second // call must not reach the server, which would fail with 404. f := &fakeDMS{ getDeployment: func(id string) (*bundledeployments.Deployment, error) { return &bundledeployments.Deployment{Name: "deployments/" + id, LastVersionId: "2"}, nil }, } - r := NewRecorder(RecorderOptions{Service: f, Versions: fakeVersions{requests: &f.versions}, DeploymentID: "stored-id", StatePath: testStatePath, Metadata: Metadata{TargetName: "dev", DisplayName: testDisplayName}, VersionType: VersionTypeDestroy}) + r := NewRecording(RecordingOptions{Client: testClient(f, nil), DeploymentID: "stored-id", StatePath: testStatePath, Metadata: Metadata{TargetName: "dev", DisplayName: testDisplayName}, VersionType: VersionTypeDestroy}) - require.NoError(t, r.CreateVersion(t.Context(), nil)) - require.NoError(t, r.CompleteVersion(t.Context(), true)) - require.NoError(t, r.CompleteVersion(t.Context(), true)) + startVersion(t, r, nil) + require.NoError(t, r.Finish(t.Context(), true)) + require.NoError(t, r.Finish(t.Context(), true)) assert.Len(t, f.completed, 1) // The destroy deletes the deployment record once, not once per call. assert.Equal(t, []string{"deployments/stored-id"}, f.deleted) } -func TestNilRecorderIsNoOp(t *testing.T) { - var r *Recorder - assert.NoError(t, r.CreateVersion(t.Context(), nil)) - assert.NoError(t, r.CompleteVersion(t.Context(), true)) +func TestDisabledRecordingIsNoOp(t *testing.T) { + r := Disabled() + + require.NoError(t, r.Prepare(t.Context())) + writer, err := r.Start(t.Context(), []StagedOperation{{ResourceKey: "jobs.foo"}}) + require.NoError(t, err) + require.NoError(t, r.Finish(t.Context(), true)) + + assert.False(t, r.Enabled()) assert.Empty(t, r.DeploymentID()) assert.Zero(t, r.Version()) + // The writer it hands out records nothing, so a caller needs no nil check. + assert.NoError(t, writer.Write(t.Context(), "jobs.foo", OperationUpdate{})) } -func TestRecorderCompleteVersionNoOpWithoutCreateVersion(t *testing.T) { +func TestRecordingFinishIsNoOpWithoutStart(t *testing.T) { f := &fakeDMS{} - r := NewRecorder(RecorderOptions{Service: f, Versions: fakeVersions{requests: &f.versions}, DeploymentID: "stored-id", StatePath: testStatePath, Metadata: Metadata{TargetName: "dev", DisplayName: testDisplayName}, VersionType: VersionTypeDeploy}) - // CompleteVersion before CreateVersion is a no-op (nothing was claimed). - require.NoError(t, r.CompleteVersion(t.Context(), true)) + r := NewRecording(RecordingOptions{Client: testClient(f, nil), DeploymentID: "stored-id", StatePath: testStatePath, Metadata: Metadata{TargetName: "dev", DisplayName: testDisplayName}, VersionType: VersionTypeDeploy}) + // Finish before Start is a no-op (nothing was claimed). + require.NoError(t, r.Finish(t.Context(), true)) assert.Empty(t, f.completed) } -func TestRecorderPrepareDeploymentClaimsNoVersion(t *testing.T) { +func TestRecordingPrepareClaimsNoVersion(t *testing.T) { // A deploy the user declines prepares but never creates: the version number is // known, so the plan can be stamped with it, but no version exists to complete and // the number is left for the next deploy to take. @@ -250,27 +270,27 @@ func TestRecorderPrepareDeploymentClaimsNoVersion(t *testing.T) { return &bundledeployments.Deployment{Name: "deployments/" + id, LastVersionId: "4"}, nil }, } - r := NewRecorder(RecorderOptions{Service: f, Versions: fakeVersions{requests: &f.versions}, DeploymentID: "stored-id", StatePath: testStatePath, Metadata: Metadata{TargetName: "dev", DisplayName: testDisplayName}, VersionType: VersionTypeDeploy}) + r := NewRecording(RecordingOptions{Client: testClient(f, nil), DeploymentID: "stored-id", StatePath: testStatePath, Metadata: Metadata{TargetName: "dev", DisplayName: testDisplayName}, VersionType: VersionTypeDeploy}) - require.NoError(t, r.PrepareDeployment(t.Context())) + require.NoError(t, r.Prepare(t.Context())) assert.Equal(t, int64(5), r.Version()) assert.Empty(t, f.versions, "no version created") - require.NoError(t, r.CompleteVersion(t.Context(), true)) + require.NoError(t, r.Finish(t.Context(), true)) assert.Empty(t, f.completed, "nothing to complete") } -func TestRecorderCreateVersionUsesThePreparedNumber(t *testing.T) { +func TestRecordingStartUsesThePreparedNumber(t *testing.T) { f := &fakeDMS{ getDeployment: func(id string) (*bundledeployments.Deployment, error) { return &bundledeployments.Deployment{Name: "deployments/" + id, LastVersionId: "4"}, nil }, } - r := NewRecorder(RecorderOptions{Service: f, Versions: fakeVersions{requests: &f.versions}, DeploymentID: "stored-id", StatePath: testStatePath, Metadata: Metadata{TargetName: "dev", DisplayName: testDisplayName}, VersionType: VersionTypeDeploy}) + r := NewRecording(RecordingOptions{Client: testClient(f, nil), DeploymentID: "stored-id", StatePath: testStatePath, Metadata: Metadata{TargetName: "dev", DisplayName: testDisplayName}, VersionType: VersionTypeDeploy}) - require.NoError(t, r.PrepareDeployment(t.Context())) - require.NoError(t, r.CreateVersion(t.Context(), nil)) + require.NoError(t, r.Prepare(t.Context())) + startVersion(t, r, nil) // The version created is the one the plan was stamped with, and it reports the // version it supersedes so the service rejects a racing deploy. @@ -280,7 +300,7 @@ func TestRecorderCreateVersionUsesThePreparedNumber(t *testing.T) { assert.Equal(t, int64(5), r.Version()) } -func TestRecorderCreateVersionDetectsAbortedConflict(t *testing.T) { +func TestRecordingStartDetectsAbortedConflict(t *testing.T) { f := &fakeDMS{ getDeployment: func(id string) (*bundledeployments.Deployment, error) { return &bundledeployments.Deployment{Name: "deployments/" + id, LastVersionId: "4"}, nil @@ -291,17 +311,16 @@ func TestRecorderCreateVersionDetectsAbortedConflict(t *testing.T) { StatusCode: 409, ErrorCode: "ABORTED", } - r := NewRecorder(RecorderOptions{ - Service: f, - Versions: &fakeVersions{requests: &f.versions, err: conflictErr}, + r := NewRecording(RecordingOptions{ + Client: testClient(f, conflictErr), DeploymentID: "stored-id", StatePath: testStatePath, Metadata: Metadata{TargetName: "dev", DisplayName: testDisplayName}, VersionType: VersionTypeDeploy, }) - require.NoError(t, r.PrepareDeployment(t.Context())) - err := r.CreateVersion(t.Context(), nil) + require.NoError(t, r.Prepare(t.Context())) + _, err := r.Start(t.Context(), nil) // Names the version that was taken and tells the user to retry, and keeps the // underlying ABORTED so callers can still match on it. @@ -322,30 +341,30 @@ func TestDeploymentIDFromName(t *testing.T) { assert.Error(t, err) } -func TestRecorderCreateVersionStagesOperations(t *testing.T) { +func TestRecordingStartStagesOperations(t *testing.T) { // The version fixes its operation set, so what the caller passes has to reach the wire // verbatim: the service has no API to add an operation later. f := &fakeDMS{assignedID: "dep-1"} - r := NewRecorder(RecorderOptions{Service: f, Versions: fakeVersions{requests: &f.versions}, StatePath: testStatePath, Metadata: Metadata{TargetName: "dev", DisplayName: testDisplayName}, VersionType: VersionTypeDeploy}) + r := NewRecording(RecordingOptions{Client: testClient(f, nil), StatePath: testStatePath, Metadata: Metadata{TargetName: "dev", DisplayName: testDisplayName}, VersionType: VersionTypeDeploy}) staged := []StagedOperation{ {ResourceKey: "jobs.foo", ActionType: bundledeployments.OperationActionTypeOperationActionTypeCreate}, {ResourceKey: "pipelines.bar", ActionType: bundledeployments.OperationActionTypeOperationActionTypeDelete}, } - require.NoError(t, r.CreateVersion(t.Context(), staged)) + startVersion(t, r, staged) require.Len(t, f.versions, 1) assert.Equal(t, staged, f.versions[0].body.Operations) } -func TestRecorderCreateVersionReportsTheOperationCap(t *testing.T) { +func TestRecordingStartReportsTheOperationCap(t *testing.T) { // A bundle past the service's per-version cap cannot be recorded at all, so say how many // resources it has rather than passing the raw quota error on. quotaErr := &apierr.APIError{StatusCode: 429, ErrorCode: "RESOURCE_EXHAUSTED"} f := &fakeDMS{assignedID: "dep-1"} - r := NewRecorder(RecorderOptions{Service: f, Versions: fakeVersions{requests: &f.versions, err: quotaErr}, StatePath: testStatePath, Metadata: Metadata{TargetName: "dev", DisplayName: testDisplayName}, VersionType: VersionTypeDeploy}) + r := NewRecording(RecordingOptions{Client: testClient(f, quotaErr), StatePath: testStatePath, Metadata: Metadata{TargetName: "dev", DisplayName: testDisplayName}, VersionType: VersionTypeDeploy}) - err := r.CreateVersion(t.Context(), []StagedOperation{ + _, err := r.Start(t.Context(), []StagedOperation{ {ResourceKey: "jobs.foo", ActionType: bundledeployments.OperationActionTypeOperationActionTypeCreate}, }) diff --git a/libs/dms/resources.go b/libs/dms/resources.go new file mode 100644 index 00000000000..304ab5e94ed --- /dev/null +++ b/libs/dms/resources.go @@ -0,0 +1,40 @@ +package dms + +import ( + "context" + "fmt" + + "github.com/databricks/databricks-sdk-go/service/bundledeployments" +) + +// Resource is what DMS holds for one resource of a deployment, as of the last operation +// that recorded it. +type Resource struct { + Key ResourceKey + ID string + + // State is the state the last operation recorded, as the opaque string the service + // stores, and empty when no operation recorded one. + State string +} + +// ListResources returns every resource DMS holds for the deployment. +func (c *Client) ListResources(ctx context.Context, deploymentID string) ([]Resource, error) { + it := c.Service.ListResources(ctx, bundledeployments.ListResourcesRequest{ + Parent: "deployments/" + deploymentID, + }) + + var out []Resource + for it.HasNext(ctx) { + res, err := it.Next(ctx) + if err != nil { + return nil, fmt.Errorf("listing resources from deployment metadata service: %w", err) + } + out = append(out, Resource{ + Key: ResourceKey(res.ResourceKey), + ID: res.ResourceId, + State: res.State, + }) + } + return out, nil +} diff --git a/libs/dms/writer.go b/libs/dms/writer.go new file mode 100644 index 00000000000..1051c379fce --- /dev/null +++ b/libs/dms/writer.go @@ -0,0 +1,56 @@ +package dms + +import ( + "context" + "sync" +) + +// stagedSequenceID is what CreateVersion leaves on every operation it stages, and so the +// precondition for the first update of a resource. +const stagedSequenceID = "0" + +// OperationWriter fills in the operations one version staged. Calls for different resources +// may run concurrently. +type OperationWriter interface { + Write(ctx context.Context, key ResourceKey, update OperationUpdate) error +} + +// operationWriter writes through the API, tracking the sequence id each resource is at. +type operationWriter struct { + ops OperationUpdater + deploymentID string + version int64 + + mu sync.Mutex + // sequenceIDs holds the token the last update for a resource returned. A resource + // absent from it has only what staging left, so its first update sends that. + sequenceIDs map[ResourceKey]string +} + +func (w *operationWriter) Write(ctx context.Context, key ResourceKey, update OperationUpdate) error { + w.mu.Lock() + sequenceID, written := w.sequenceIDs[key] + w.mu.Unlock() + if !written { + sequenceID = stagedSequenceID + } + + next, err := w.ops.UpdateOperation(ctx, w.deploymentID, w.version, key, sequenceID, update) + if err != nil { + return err + } + + // The next write for this resource echoes the sequence id this one earned. + w.mu.Lock() + w.sequenceIDs[key] = next + w.mu.Unlock() + + return nil +} + +// noopWriter is the writer of a disabled recording: nothing is staged, so nothing is written. +type noopWriter struct{} + +func (noopWriter) Write(context.Context, ResourceKey, OperationUpdate) error { + return nil +} diff --git a/libs/dms/writer_test.go b/libs/dms/writer_test.go new file mode 100644 index 00000000000..78b1f13cc8e --- /dev/null +++ b/libs/dms/writer_test.go @@ -0,0 +1,159 @@ +package dms + +import ( + "context" + "encoding/json" + "errors" + "sync" + "testing" + + "github.com/databricks/databricks-sdk-go/service/bundledeployments" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// updaterCall is one call the writer made to the operations API. +type updaterCall struct { + deploymentID string + version int64 + key ResourceKey + sequenceID string + update OperationUpdate +} + +// fakeUpdater reports sequence for every call, failing the one at index failOn. +type fakeUpdater struct { + mu sync.Mutex + calls []updaterCall + sequence string + failOn int +} + +func newFakeUpdater(sequence string) *fakeUpdater { + return &fakeUpdater{sequence: sequence, failOn: -1} +} + +func (f *fakeUpdater) UpdateOperation(ctx context.Context, deploymentID string, version int64, key ResourceKey, sequenceID string, update OperationUpdate) (string, error) { + f.mu.Lock() + defer f.mu.Unlock() + + callNum := len(f.calls) + f.calls = append(f.calls, updaterCall{ + deploymentID: deploymentID, + version: version, + key: key, + sequenceID: sequenceID, + update: update, + }) + if callNum == f.failOn { + return "", errors.New("injected error") + } + return f.sequence, nil +} + +// testWriter returns a writer for version 2 of dep-1, recording through f. +func testWriter(f OperationUpdater) OperationWriter { + return &operationWriter{ + ops: f, + deploymentID: "dep-1", + version: 2, + sequenceIDs: make(map[ResourceKey]string), + } +} + +func writeState(t *testing.T, w OperationWriter, key ResourceKey, resourceID string, state json.RawMessage) { + t.Helper() + update, err := NewStateUpdate(resourceID, state, false) + require.NoError(t, err) + require.NoError(t, w.Write(t.Context(), key, update)) +} + +func TestWriterFirstWriteUpdatesTheStagedOperation(t *testing.T) { + f := newFakeUpdater("1") + w := testWriter(f) + + writeState(t, w, "jobs.foo", "job-123", json.RawMessage(`{"state":{}}`)) + + require.Len(t, f.calls, 1) + c := f.calls[0] + // The version already staged this operation, so the first write updates it and echoes + // the sequence id staging left. + assert.Equal(t, "dep-1", c.deploymentID) + assert.Equal(t, int64(2), c.version) + assert.Equal(t, ResourceKey("jobs.foo"), c.key) + assert.Equal(t, stagedSequenceID, c.sequenceID) + assert.Equal(t, "job-123", c.update.ResourceID) +} + +func TestWriterSecondWriteEchoesTheServiceSequence(t *testing.T) { + // One operation per resource per version: the second write updates the same operation, + // echoing the sequence id the service returned as its precondition. + f := newFakeUpdater("7") + w := testWriter(f) + + writeState(t, w, "jobs.foo", "", nil) + writeState(t, w, "jobs.foo", "job-456", json.RawMessage(`{"state":{}}`)) + + require.Len(t, f.calls, 2) + assert.Equal(t, stagedSequenceID, f.calls[0].sequenceID) + assert.Equal(t, "7", f.calls[1].sequenceID) +} + +func TestWriterTracksSequencePerResource(t *testing.T) { + // Each resource has its own staged operation, so each one's first write echoes the staged + // sequence id rather than a sequence another resource earned. + f := newFakeUpdater("1") + w := testWriter(f) + + writeState(t, w, "jobs.foo", "id-1", json.RawMessage(`{"state":{}}`)) + writeState(t, w, "jobs.bar", "id-2", json.RawMessage(`{"state":{}}`)) + + require.Len(t, f.calls, 2) + assert.Equal(t, stagedSequenceID, f.calls[0].sequenceID) + assert.Equal(t, stagedSequenceID, f.calls[1].sequenceID) +} + +func TestWriterErrorKeepsTheSequence(t *testing.T) { + // A failed write returns its error and leaves the recorded sequence id alone, so a later + // write for the same resource still carries the precondition the service last gave us. + f := newFakeUpdater("9") + f.failOn = 1 + w := testWriter(f) + + writeState(t, w, "jobs.foo", "job-1", json.RawMessage(`{"state":{}}`)) + + second, err := NewStateUpdate("job-2", json.RawMessage(`{"state":{}}`), false) + require.NoError(t, err) + err = w.Write(t.Context(), "jobs.foo", second) + require.ErrorContains(t, err, "injected error") + + // The third write is what proves the sequence id survived the failure. + writeState(t, w, "jobs.foo", "job-3", json.RawMessage(`{"state":{}}`)) + + require.Len(t, f.calls, 3) + assert.Equal(t, "9", f.calls[2].sequenceID) +} + +func TestUpdateRequestSendsOnlyWhatTheMaskNames(t *testing.T) { + // A failure keeps whatever state an earlier write recorded, so it must send neither + // state nor resource_id: an empty state would drop the resource from the deployment. + failure := NewFailureUpdate("job-1", errors.New("boom")) + + body := newUpdateRequest(failure, "3") + + assert.Empty(t, body.State) + assert.Empty(t, body.ResourceId) + assert.Equal(t, "3", body.SequenceId) + assert.Equal(t, "boom", body.ErrorMessage) + assert.Equal(t, bundledeployments.OperationStatusOperationStatusFailed, body.Status) +} + +func TestUpdateRequestSendsStateWhenNamed(t *testing.T) { + update, err := NewStateUpdate("job-1", json.RawMessage(`{"state":{"name":"foo"}}`), false) + require.NoError(t, err) + + body := newUpdateRequest(update, stagedSequenceID) + + assert.JSONEq(t, `{"state":{"name":"foo"}}`, body.State) + assert.Equal(t, "job-1", body.ResourceId) +} diff --git a/libs/testserver/bundle_test.go b/libs/testserver/bundle_test.go new file mode 100644 index 00000000000..c9bfa82f533 --- /dev/null +++ b/libs/testserver/bundle_test.go @@ -0,0 +1,146 @@ +package testserver + +import ( + "encoding/json" + "net/url" + "testing" + + "github.com/databricks/databricks-sdk-go/service/bundledeployments" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// stageOperation creates a deployment with one version that stages an operation for +// resourceKey, and returns the deployment and version ids. +func stageOperation(t *testing.T, s *FakeWorkspace, resourceKey string) (string, string) { + t.Helper() + + parent := "/Users/" + TestUser.UserName + resp := s.CreateDeployment(Request{Body: []byte(`{"initial_parent_path":"` + parent + `"}`)}) + require.Equal(t, 0, resp.StatusCode, resp.Body) + var dep bundledeployments.Deployment + remarshal(t, resp.Body, &dep) + deploymentID := dep.Name[len("deployments/"):] + + body := `{"version_type":"VERSION_TYPE_DEPLOY","operations":[{"resource_key":"` + resourceKey + `","action_type":"OPERATION_ACTION_TYPE_UPDATE"}]}` + resp = s.CreateVersion(Request{ + Body: []byte(body), + URL: &url.URL{RawQuery: "version_id=1"}, + }, deploymentID) + require.Equal(t, 0, resp.StatusCode, resp.Body) + + return deploymentID, "1" +} + +// updateOperation applies one update, and returns the sequence id for the next one. +func updateOperation(t *testing.T, s *FakeWorkspace, deploymentID, versionID, resourceKey, mask, body string) string { + t.Helper() + + resp := s.UpdateOperation(Request{ + Body: []byte(body), + URL: &url.URL{RawQuery: "update_mask=" + url.QueryEscape(mask)}, + }, deploymentID, versionID, resourceKey) + require.Equal(t, 0, resp.StatusCode, resp.Body) + + // The response types sequence_id as a string, which is why the SDK cannot read it. + var raw map[string]any + remarshal(t, resp.Body, &raw) + return raw["sequence_id"].(string) +} + +func listResources(t *testing.T, s *FakeWorkspace, deploymentID string) map[string]bundledeployments.Resource { + t.Helper() + + resp := s.ListResources(deploymentID) + require.Equal(t, 0, resp.StatusCode, resp.Body) + + var listed bundledeployments.ListResourcesResponse + remarshal(t, resp.Body, &listed) + + out := map[string]bundledeployments.Resource{} + for _, r := range listed.Resources { + out[r.ResourceKey] = r + } + return out +} + +func remarshal(t *testing.T, from, into any) { + t.Helper() + raw, err := json.Marshal(from) + require.NoError(t, err) + require.NoError(t, json.Unmarshal(raw, into)) +} + +// TestUpdateOperationProjectionFollowsTheMask pins the rule the CLI's update masks are +// built around: the deployment-level resource - what the next plan reads - moves only when +// the mask names state. An update that leaves state out reports an outcome and must not +// disturb what the deployment already holds. +func TestUpdateOperationProjectionFollowsTheMask(t *testing.T) { + const key = "jobs.foo" + const state = `{\"state\":{\"name\":\"foo\"}}` + + tests := []struct { + name string + // updates are applied in order, as (mask, body) pairs. The sequence id is filled in. + updates [][2]string + // wantResource is the state the deployment holds afterwards, empty for no resource. + wantResource string + wantID string + }{ + { + name: "a write that names state records the resource", + updates: [][2]string{ + {"state,error_message,resource_id,status", `{"state":"` + state + `","resource_id":"job-1","status":"OPERATION_STATUS_SUCCEEDED"}`}, + }, + wantResource: `{"state":{"name":"foo"}}`, + wantID: "job-1", + }, + { + name: "naming state with no value removes the resource", + updates: [][2]string{ + {"state,error_message,resource_id,status", `{"state":"` + state + `","resource_id":"job-1","status":"OPERATION_STATUS_SUCCEEDED"}`}, + {"state,error_message,resource_id,status", `{"resource_id":"job-1","status":"OPERATION_STATUS_SUCCEEDED"}`}, + }, + wantResource: "", + }, + { + name: "a failure that leaves state out keeps the recorded resource", + updates: [][2]string{ + {"state,error_message,resource_id,status", `{"state":"` + state + `","resource_id":"job-1","status":"OPERATION_STATUS_SUCCEEDED"}`}, + {"error_message,status", `{"error_message":"boom","status":"OPERATION_STATUS_FAILED"}`}, + }, + wantResource: `{"state":{"name":"foo"}}`, + wantID: "job-1", + }, + { + name: "a failure before any write records no resource", + updates: [][2]string{ + {"error_message,status", `{"error_message":"boom","status":"OPERATION_STATUS_FAILED"}`}, + }, + wantResource: "", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := NewFakeWorkspace("http://localhost", "test-token") + deploymentID, versionID := stageOperation(t, s, key) + + sequenceID := "0" + for _, u := range tt.updates { + mask, body := u[0], u[1] + withSequence := body[:len(body)-1] + `,"sequence_id":"` + sequenceID + `"}` + sequenceID = updateOperation(t, s, deploymentID, versionID, key, mask, withSequence) + } + + resources := listResources(t, s, deploymentID) + if tt.wantResource == "" { + assert.NotContains(t, resources, key) + return + } + require.Contains(t, resources, key) + assert.JSONEq(t, tt.wantResource, resources[key].State) + assert.Equal(t, tt.wantID, resources[key].ResourceId) + }) + } +} From 78935397dc1b4371369dc903426ed427e37f20b2 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 18 Aug 2026 21:42:29 +0000 Subject: [PATCH 110/125] dstate: leave the state alone when a recorded envelope is malformed readDMSState replaced stateIDs before it had parsed everything, so a bad envelope half-way through left it holding some of the recorded ids while Data.State still held what the file loaded. Build both and assign together. Co-authored-by: Isaac --- bundle/direct/dstate/dms.go | 7 +++++-- bundle/direct/dstate/dms_test.go | 9 +++++++++ 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/bundle/direct/dstate/dms.go b/bundle/direct/dstate/dms.go index 7df599122eb..28ffe15a3a1 100644 --- a/bundle/direct/dstate/dms.go +++ b/bundle/direct/dstate/dms.go @@ -33,18 +33,21 @@ func (db *DeploymentState) readDMSState(ctx context.Context, src *DMSSource) err return err } + // Built first and assigned together, so a malformed envelope leaves the state as it was + // rather than half replaced. resources := make(map[string]ResourceEntry, len(recorded)) - db.stateIDs = make(map[string]string, len(recorded)) + stateIDs := make(map[string]string, len(recorded)) for _, res := range recorded { entry, err := stateEntry(res) if err != nil { return err } resources[res.Key.StateKey()] = entry - db.stateIDs[res.Key.StateKey()] = entry.ID + stateIDs[res.Key.StateKey()] = entry.ID } db.Data.State = resources + db.stateIDs = stateIDs return nil } diff --git a/bundle/direct/dstate/dms_test.go b/bundle/direct/dstate/dms_test.go index a2cab988c73..d08cdaf9917 100644 --- a/bundle/direct/dstate/dms_test.go +++ b/bundle/direct/dstate/dms_test.go @@ -65,13 +65,22 @@ func TestReadDMSStateUnwrapsEnvelope(t *testing.T) { } func TestReadDMSStateRejectsMalformedState(t *testing.T) { + // The good resource comes first, so the error lands mid-way: what the file loaded has to + // survive whole rather than end up half replaced. f := &fakeResourceLister{resources: []bundledeployments.Resource{ + {ResourceKey: "jobs.ok", ResourceId: "999", State: `{"state":{"name":"ok"}}`}, {ResourceKey: "jobs.foo", ResourceId: "123", State: "not json"}, }} var db DeploymentState + db.Data.State = map[string]ResourceEntry{"resources.jobs.bar": {ID: "file-id"}} + db.stateIDs = map[string]string{"resources.jobs.bar": "file-id"} + err := db.readDMSState(t.Context(), &DMSSource{Client: testClient(f), DeploymentID: "dep-1"}) assert.ErrorContains(t, err, "interpreting state recorded for resources.jobs.foo") + + assert.Equal(t, map[string]ResourceEntry{"resources.jobs.bar": {ID: "file-id"}}, db.Data.State) + assert.Equal(t, map[string]string{"resources.jobs.bar": "file-id"}, db.stateIDs) } func TestReadDMSStateReplacesLocalState(t *testing.T) { From d97c6a31c955d56a8b674cd630c051ff16e65abf Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 18 Aug 2026 21:53:01 +0000 Subject: [PATCH 111/125] libs/dms: put every call on the client, and send each field for its own reason Completing a version, the heartbeat and the three deployment calls still went through Client.Service from the lifecycle code, each formatting its own resource name - three copies of "deployments/%s/versions/%s" and four of the deployment form. They are Client methods now, the two name formats are declared once, and recording.go no longer touches Service or builds a name. newUpdateRequest gated resource_id on the mask naming state, which happens to be equivalent today because the two masks in use name both or neither. It said the wrong thing though: resource_id is sent because the mask names resource_id. Each field is now gated on its own entry. Drops the package doc; what it described is stated where it applies. Co-authored-by: Isaac --- .../mutator/initialize_deployment_history.go | 10 +- libs/dms/client.go | 102 ++++++++++++++++-- libs/dms/doc.go | 20 ---- libs/dms/operation.go | 2 +- libs/dms/recording.go | 59 +++------- libs/dms/resources.go | 2 +- libs/dms/writer_test.go | 17 +++ 7 files changed, 129 insertions(+), 83 deletions(-) delete mode 100644 libs/dms/doc.go diff --git a/bundle/config/mutator/initialize_deployment_history.go b/bundle/config/mutator/initialize_deployment_history.go index 6e1e47d2f9e..f8db461e9c0 100644 --- a/bundle/config/mutator/initialize_deployment_history.go +++ b/bundle/config/mutator/initialize_deployment_history.go @@ -7,7 +7,6 @@ import ( "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dms" - "github.com/databricks/databricks-sdk-go/service/bundledeployments" ) type initializeDeploymentHistory struct{} @@ -38,12 +37,15 @@ func (m *initializeDeploymentHistory) Apply(ctx context.Context, b *bundle.Bundl return nil } + client, err := dms.NewClient(w) + if err != nil { + return diag.FromErr(err) + } + // The ID came from a BUNDLE_DEPLOYMENT node that get-status returned, and by // design the service has a deployment for every such node, so this get does not // have a not-found case. last_version_id is empty until the first version. - dep, err := w.BundleDeployments.GetDeployment(ctx, bundledeployments.GetDeploymentRequest{ - Name: "deployments/" + deploymentID, - }) + dep, err := client.GetDeployment(ctx, deploymentID) if err != nil { return diag.FromErr(err) } diff --git a/libs/dms/client.go b/libs/dms/client.go index 922bc1e281f..b8502b8a93b 100644 --- a/libs/dms/client.go +++ b/libs/dms/client.go @@ -4,6 +4,8 @@ import ( "context" "fmt" "net/http" + "strconv" + "strings" "github.com/databricks/cli/libs/auth" "github.com/databricks/databricks-sdk-go" @@ -35,6 +37,82 @@ func NewClient(w *databricks.WorkspaceClient) (*Client, error) { return &Client{Service: w.BundleDeployments, Versions: raw, Operations: raw}, nil } +// deploymentName and versionName are the two resource-name formats the service uses. Every +// call builds its name here, so a caller only ever passes ids. +func deploymentName(deploymentID string) string { + return "deployments/" + deploymentID +} + +func versionName(deploymentID string, version int64) string { + return fmt.Sprintf("deployments/%s/versions/%d", deploymentID, version) +} + +// CreateDeployment registers a deployment under parentPath and returns the id the server +// assigned it, which is the id of the workspace node it creates there. +func (c *Client) CreateDeployment(ctx context.Context, parentPath, targetName string) (string, error) { + dep, err := c.Service.CreateDeployment(ctx, bundledeployments.CreateDeploymentRequest{ + Deployment: bundledeployments.Deployment{ + InitialParentPath: parentPath, + TargetName: targetName, + }, + }) + if err != nil { + return "", err + } + return deploymentIDFromName(dep.Name) +} + +// GetDeployment reads the deployment record, which carries the last version recorded under it. +func (c *Client) GetDeployment(ctx context.Context, deploymentID string) (*bundledeployments.Deployment, error) { + return c.Service.GetDeployment(ctx, bundledeployments.GetDeploymentRequest{ + Name: deploymentName(deploymentID), + }) +} + +// DeleteDeployment removes the deployment record, which a completed destroy does. +func (c *Client) DeleteDeployment(ctx context.Context, deploymentID string) error { + return c.Service.DeleteDeployment(ctx, bundledeployments.DeleteDeploymentRequest{ + Name: deploymentName(deploymentID), + }) +} + +// CreateVersion claims the version and stages the operations body carries. +func (c *Client) CreateVersion(ctx context.Context, deploymentID string, version int64, body CreateVersionRequest) (*bundledeployments.Version, error) { + return c.Versions.CreateVersion(ctx, deploymentID, strconv.FormatInt(version, 10), body) +} + +// CompleteVersion closes the version out, which is what stops the service expiring its lease. +func (c *Client) CompleteVersion(ctx context.Context, deploymentID string, version int64, reason bundledeployments.VersionComplete) error { + _, err := c.Service.CompleteVersion(ctx, bundledeployments.CompleteVersionRequest{ + Name: versionName(deploymentID, version), + CompletionReason: reason, + }) + return err +} + +// Heartbeat renews the version's lease. +func (c *Client) Heartbeat(ctx context.Context, deploymentID string, version int64) error { + _, err := c.Service.Heartbeat(ctx, bundledeployments.HeartbeatRequest{ + Name: versionName(deploymentID, version), + }) + return err +} + +// UpdateOperation fills in one operation the version staged; see OperationUpdater. +func (c *Client) UpdateOperation(ctx context.Context, deploymentID string, version int64, key ResourceKey, sequenceID string, update OperationUpdate) (string, error) { + return c.Operations.UpdateOperation(ctx, deploymentID, version, key, sequenceID, update) +} + +// deploymentIDFromName extracts the deployment ID from a DMS resource name of +// the form "deployments/{deployment_id}". +func deploymentIDFromName(name string) (string, error) { + id, ok := strings.CutPrefix(name, deploymentName("")) + if !ok || id == "" { + return "", fmt.Errorf("unexpected deployment name %q from deployment metadata service", name) + } + return id, nil +} + // VersionCreator creates a version under a deployment. Hand-written because the generated // struct has no previous_version_id, which the service needs as its concurrency check - // without it every deploy after the first is rejected. @@ -103,7 +181,7 @@ type rawClient struct { func (r *rawClient) CreateVersion(ctx context.Context, deploymentID, versionID string, body CreateVersionRequest) (*bundledeployments.Version, error) { var version bundledeployments.Version - path := fmt.Sprintf("/api/2.0/bundle/deployments/%s/versions", deploymentID) + path := "/api/2.0/bundle/" + deploymentName(deploymentID) + "/versions" err := r.client.Do(ctx, http.MethodPost, path, auth.WorkspaceIDHeaders(r.client.Config), map[string]any{"version_id": versionID}, @@ -114,19 +192,23 @@ func (r *rawClient) CreateVersion(ctx context.Context, deploymentID, versionID s return &version, nil } -// newUpdateRequest builds the request body for update. Only what the mask names is sent: -// the service would ignore the rest, and state is the largest field by far, so a failure -// that keeps the recorded state sends none. +// newUpdateRequest builds the request body for update. Each field is sent because the mask +// names it: the service ignores the rest, and state is the largest field by far, so a +// failure that keeps the recorded state sends none of it. func newUpdateRequest(update OperationUpdate, sequenceID string) updateOperationRequest { - body := updateOperationRequest{ - ErrorMessage: update.ErrorMessage, - Status: update.Status, - SequenceId: sequenceID, - } + body := updateOperationRequest{SequenceId: sequenceID} if update.Fields.Has(FieldState) { body.State = string(update.State) + } + if update.Fields.Has(FieldResourceID) { body.ResourceId = update.ResourceID } + if update.Fields.Has(FieldErrorMessage) { + body.ErrorMessage = update.ErrorMessage + } + if update.Fields.Has(FieldStatus) { + body.Status = update.Status + } return body } @@ -134,7 +216,7 @@ func (r *rawClient) UpdateOperation(ctx context.Context, deploymentID string, ve body := newUpdateRequest(update, sequenceID) var result operationResponse - path := fmt.Sprintf("/api/2.0/bundle/deployments/%s/versions/%d/operations/%s", deploymentID, version, key) + path := "/api/2.0/bundle/" + versionName(deploymentID, version) + "/operations/" + string(key) err := r.client.Do(ctx, http.MethodPatch, path, auth.WorkspaceIDHeaders(r.client.Config), map[string]any{"update_mask": update.Fields.Mask()}, diff --git a/libs/dms/doc.go b/libs/dms/doc.go deleted file mode 100644 index 1ae54848d0d..00000000000 --- a/libs/dms/doc.go +++ /dev/null @@ -1,20 +0,0 @@ -// Package dms records bundle deployment history with the deployment metadata service. -// -// One deploy or destroy is one version under one deployment. CreateVersion stages an -// operation for every resource the plan touches, in OPERATION_STATUS_PENDING at sequence -// id 0, and the CLI fills each one in with UpdateOperation as the resource is applied. -// The set is fixed at CreateVersion: the service has no call to add one later, and it caps -// how many a version may stage. -// -// An update is taken literally. The field mask decides what changes, and a field left out -// keeps the value it had. The service mirrors state and resource_id onto the -// deployment-level resource, which is what the next plan reads, so a mask that names state -// with no value is what removes a resource - and a failure that must not disturb the state -// an earlier write left names only error_message and status. -// -// Every update carries the sequence id the previous one returned, starting from the 0 that -// staging leaves, so a write from a stale deploy is rejected rather than applied. -// -// Two calls are written by hand rather than taken from the SDK, which is generated from an -// OpenAPI spec that trails the service; see client.go. -package dms diff --git a/libs/dms/operation.go b/libs/dms/operation.go index 6a08251303f..fd9752f10db 100644 --- a/libs/dms/operation.go +++ b/libs/dms/operation.go @@ -27,7 +27,7 @@ const StatusInProgress bundledeployments.OperationStatus = "OPERATION_STATUS_IN_ // resource that produced it rather than the upload at the end of apply. type OperationUpdate struct { // Fields is the mask to send. It is taken literally: a field named here is written, - // one left out keeps its value. See the package doc. + // one left out keeps the value it had. Fields Fields // State is the serialized state after the operation, and nil for a delete. diff --git a/libs/dms/recording.go b/libs/dms/recording.go index 7e94bff6e8b..6e28bfc9251 100644 --- a/libs/dms/recording.go +++ b/libs/dms/recording.go @@ -6,7 +6,6 @@ import ( "fmt" "net/http" "strconv" - "strings" "time" "github.com/databricks/cli/internal/build" @@ -174,12 +173,10 @@ func (r *recording) Start(ctx context.Context, staged []StagedOperation) (Operat } } - versionID := strconv.FormatInt(r.versionNum, 10) - - // The server rejects this unless versionID exceeds last_version_id and + // The server rejects this unless the version number exceeds last_version_id and // previous_version_id matches it, which is what makes claiming the number up front // safe: a deploy that took it in the meantime is reported, not overwritten. - version, err := r.client.Versions.CreateVersion(ctx, r.deploymentID, versionID, CreateVersionRequest{ + version, err := r.client.CreateVersion(ctx, r.deploymentID, r.versionNum, CreateVersionRequest{ CliVersion: build.GetInfo().Version, VersionType: r.versionType, TargetName: r.metadata.TargetName, @@ -199,13 +196,13 @@ func (r *recording) Start(ctx context.Context, staged []StagedOperation) (Operat // A 409 ABORTED means another deploy claimed this version number in between // Prepare and here. if isAbortedErr(err) { - return nil, fmt.Errorf("another deploy already claimed version %s of this deployment, try again: %w", versionID, err) + return nil, fmt.Errorf("another deploy already claimed version %d of this deployment, try again: %w", r.versionNum, err) } return nil, fmt.Errorf("failed to create deployment version: %w", err) } r.versionCreated = true - r.stopHeartbeat = startHeartbeat(ctx, r.client.Service, r.deploymentID, versionID) + r.stopHeartbeat = startHeartbeat(ctx, r.client, r.deploymentID, r.versionNum) log.Infof(ctx, "Created deployment version: deployment=%s version=%s", r.deploymentID, version.VersionId) return &operationWriter{ @@ -230,25 +227,15 @@ func (r *recording) Finish(ctx context.Context, success bool) error { r.stopHeartbeat() - versionIDStr := strconv.FormatInt(r.versionNum, 10) - versionName := fmt.Sprintf("deployments/%s/versions/%s", r.deploymentID, versionIDStr) - - _, err := r.client.Service.CompleteVersion(ctx, bundledeployments.CompleteVersionRequest{ - Name: versionName, - CompletionReason: reason, - }) - if err != nil { + if err := r.client.CompleteVersion(ctx, r.deploymentID, r.versionNum, reason); err != nil { return err } - log.Infof(ctx, "Completed deployment version: deployment=%s version=%s reason=%s", r.deploymentID, versionIDStr, reason) + log.Infof(ctx, "Completed deployment version: deployment=%s version=%d reason=%s", r.deploymentID, r.versionNum, reason) // For destroy operations, delete the deployment record after the version // completes successfully. if reason == bundledeployments.VersionCompleteVersionCompleteSuccess && r.versionType == VersionTypeDestroy { - err = r.client.Service.DeleteDeployment(ctx, bundledeployments.DeleteDeploymentRequest{ - Name: "deployments/" + r.deploymentID, - }) - if err != nil { + if err := r.client.DeleteDeployment(ctx, r.deploymentID); err != nil { return fmt.Errorf("failed to delete deployment: %w", err) } } @@ -263,9 +250,7 @@ func (r *recording) resolveNextVersion(ctx context.Context) (versionID string, e // The ID came from a BUNDLE_DEPLOYMENT node that get-status returned, and by // design the service has a deployment for every such node, so a not-found // here means that invariant is broken rather than anything the user did. - dep, getErr := r.client.Service.GetDeployment(ctx, bundledeployments.GetDeploymentRequest{ - Name: "deployments/" + r.deploymentID, - }) + dep, getErr := r.client.GetDeployment(ctx, r.deploymentID) switch { case errors.Is(getErr, apierr.ErrNotFound), errors.Is(getErr, apierr.ErrResourceDoesNotExist): return "", fmt.Errorf("internal error: no deployment found for the file with object id %s: %w", r.deploymentID, getErr) @@ -287,19 +272,10 @@ func (r *recording) resolveNextVersion(ctx context.Context) (versionID string, e // First deploy: create the deployment so the server assigns an ID. // initial_parent_path is required - the node the service creates under it is // what ResolveDeploymentID reads back later. - dep, createErr := r.client.Service.CreateDeployment(ctx, bundledeployments.CreateDeploymentRequest{ - Deployment: bundledeployments.Deployment{ - InitialParentPath: r.statePath, - TargetName: r.metadata.TargetName, - }, - }) + id, createErr := r.client.CreateDeployment(ctx, r.statePath, r.metadata.TargetName) if createErr != nil { return "", fmt.Errorf("failed to create deployment: %w", createErr) } - id, idErr := deploymentIDFromName(dep.Name) - if idErr != nil { - return "", idErr - } r.deploymentID = id versionID = "1" } @@ -307,21 +283,10 @@ func (r *recording) resolveNextVersion(ctx context.Context) (versionID string, e return versionID, nil } -// deploymentIDFromName extracts the deployment ID from a DMS resource name of -// the form "deployments/{deployment_id}". -func deploymentIDFromName(name string) (string, error) { - id, ok := strings.CutPrefix(name, "deployments/") - if !ok || id == "" { - return "", fmt.Errorf("unexpected deployment name %q from deployment metadata service", name) - } - return id, nil -} - // startHeartbeat starts a background goroutine that sends heartbeats to keep // the deployment version's lease alive. Returns a cancel function to stop it. -func startHeartbeat(ctx context.Context, svc bundledeployments.BundleDeploymentsInterface, deploymentID, versionID string) context.CancelFunc { +func startHeartbeat(ctx context.Context, client *Client, deploymentID string, version int64) context.CancelFunc { ctx, cancel := context.WithCancel(ctx) - versionName := fmt.Sprintf("deployments/%s/versions/%s", deploymentID, versionID) go func() { ticker := time.NewTicker(defaultHeartbeatInterval) @@ -332,7 +297,7 @@ func startHeartbeat(ctx context.Context, svc bundledeployments.BundleDeployments case <-ctx.Done(): return case <-ticker.C: - _, err := svc.Heartbeat(ctx, bundledeployments.HeartbeatRequest{Name: versionName}) + err := client.Heartbeat(ctx, deploymentID, version) if err != nil { // A 409 ABORTED is expected if the version was completed // between the ticker firing and the heartbeat. @@ -342,7 +307,7 @@ func startHeartbeat(ctx context.Context, svc bundledeployments.BundleDeployments } log.Warnf(ctx, "Failed to send deployment heartbeat: %v", err) } else { - log.Debugf(ctx, "Deployment heartbeat sent: deployment=%s version=%s", deploymentID, versionID) + log.Debugf(ctx, "Deployment heartbeat sent: deployment=%s version=%d", deploymentID, version) } } } diff --git a/libs/dms/resources.go b/libs/dms/resources.go index 304ab5e94ed..9bb2305fe43 100644 --- a/libs/dms/resources.go +++ b/libs/dms/resources.go @@ -21,7 +21,7 @@ type Resource struct { // ListResources returns every resource DMS holds for the deployment. func (c *Client) ListResources(ctx context.Context, deploymentID string) ([]Resource, error) { it := c.Service.ListResources(ctx, bundledeployments.ListResourcesRequest{ - Parent: "deployments/" + deploymentID, + Parent: deploymentName(deploymentID), }) var out []Resource diff --git a/libs/dms/writer_test.go b/libs/dms/writer_test.go index 78b1f13cc8e..bc3f4ebb0af 100644 --- a/libs/dms/writer_test.go +++ b/libs/dms/writer_test.go @@ -157,3 +157,20 @@ func TestUpdateRequestSendsStateWhenNamed(t *testing.T) { assert.JSONEq(t, `{"state":{"name":"foo"}}`, body.State) assert.Equal(t, "job-1", body.ResourceId) } + +func TestUpdateRequestSendsEachFieldOnItsOwnMaskEntry(t *testing.T) { + // resource_id does not travel with state: a mask that names one and not the other sends + // exactly that. + update := OperationUpdate{ + Fields: FieldResourceID | FieldStatus, + State: json.RawMessage(`{"state":{"name":"foo"}}`), + ResourceID: "job-1", + Status: bundledeployments.OperationStatusOperationStatusSucceeded, + } + + body := newUpdateRequest(update, "4") + + assert.Empty(t, body.State) + assert.Equal(t, "job-1", body.ResourceId) + assert.Equal(t, bundledeployments.OperationStatusOperationStatusSucceeded, body.Status) +} From a2aae6a92e815060aebb59ea0b6bfb85be24bb57 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 18 Aug 2026 22:26:55 +0000 Subject: [PATCH 112/125] libs/dms: one seam for the half the SDK cannot express The client had a field per hand-written call, which read as though Versions owned versions - but completing one goes through the generated client, so it did not. There are two halves, not three: Service for the generated calls and raw for the two requests that have to be written by hand, both behind one interface that says why each exists. The writer takes the client rather than a second interface, and the two test fakes for those calls become one. Also drops the testserver projection table test: the bundle/dms acceptance tests already drive that rule through the same fake. Co-authored-by: Isaac --- libs/dms/client.go | 46 +++++------ libs/dms/client_test.go | 107 ++++++++++++++++++++++++ libs/dms/recording.go | 2 +- libs/dms/recording_test.go | 73 +++++------------ libs/dms/writer.go | 4 +- libs/dms/writer_test.go | 75 ++++------------- libs/testserver/bundle_test.go | 146 --------------------------------- 7 files changed, 169 insertions(+), 284 deletions(-) create mode 100644 libs/dms/client_test.go delete mode 100644 libs/testserver/bundle_test.go diff --git a/libs/dms/client.go b/libs/dms/client.go index b8502b8a93b..4068fe40cd8 100644 --- a/libs/dms/client.go +++ b/libs/dms/client.go @@ -13,18 +13,15 @@ import ( "github.com/databricks/databricks-sdk-go/service/bundledeployments" ) -// Client is every call the CLI makes to DMS. Most go through the generated client; the two -// the SDK cannot express are written by hand below, each with its own interface so a test -// can capture what the CLI sends. +// Client is every call the CLI makes to DMS, as methods below. Each one goes out through one +// of two halves: the generated client for the calls it can express, and hand-written requests +// for the two it cannot. type Client struct { - // Service is the generated client, used for every call it can express. + // Service is the generated client. Service bundledeployments.BundleDeploymentsInterface - // Versions creates versions; see VersionCreator. - Versions VersionCreator - - // Operations fills in staged operations; see OperationUpdater. - Operations OperationUpdater + // raw sends what the generated client cannot; see requester. + raw requester } // NewClient returns a Client for the workspace w. @@ -33,8 +30,7 @@ func NewClient(w *databricks.WorkspaceClient) (*Client, error) { if err != nil { return nil, err } - raw := &rawClient{client: api} - return &Client{Service: w.BundleDeployments, Versions: raw, Operations: raw}, nil + return &Client{Service: w.BundleDeployments, raw: &rawClient{client: api}}, nil } // deploymentName and versionName are the two resource-name formats the service uses. Every @@ -78,7 +74,7 @@ func (c *Client) DeleteDeployment(ctx context.Context, deploymentID string) erro // CreateVersion claims the version and stages the operations body carries. func (c *Client) CreateVersion(ctx context.Context, deploymentID string, version int64, body CreateVersionRequest) (*bundledeployments.Version, error) { - return c.Versions.CreateVersion(ctx, deploymentID, strconv.FormatInt(version, 10), body) + return c.raw.CreateVersion(ctx, deploymentID, strconv.FormatInt(version, 10), body) } // CompleteVersion closes the version out, which is what stops the service expiring its lease. @@ -98,9 +94,10 @@ func (c *Client) Heartbeat(ctx context.Context, deploymentID string, version int return err } -// UpdateOperation fills in one operation the version staged; see OperationUpdater. +// UpdateOperation fills in one operation the version staged, and returns the sequence id the +// next update for that resource must send. func (c *Client) UpdateOperation(ctx context.Context, deploymentID string, version int64, key ResourceKey, sequenceID string, update OperationUpdate) (string, error) { - return c.Operations.UpdateOperation(ctx, deploymentID, version, key, sequenceID, update) + return c.raw.UpdateOperation(ctx, deploymentID, version, key, sequenceID, update) } // deploymentIDFromName extracts the deployment ID from a DMS resource name of @@ -113,19 +110,18 @@ func deploymentIDFromName(name string) (string, error) { return id, nil } -// VersionCreator creates a version under a deployment. Hand-written because the generated -// struct has no previous_version_id, which the service needs as its concurrency check - -// without it every deploy after the first is rejected. -type VersionCreator interface { +// requester sends the two requests the generated client cannot express, so a test can capture +// what the CLI puts on the wire. Both are TODO(DMS): drop them once the spec catches up. +type requester interface { + // CreateVersion is hand-written because the generated struct has no + // previous_version_id, which the service needs as its concurrency check - without it + // every deploy after the first is rejected. CreateVersion(ctx context.Context, deploymentID, versionID string, body CreateVersionRequest) (*bundledeployments.Version, error) -} -// OperationUpdater fills in an operation the version staged, and returns the sequence id the -// next update for that resource must send. Hand-written because the SDK types sequence_id as -// an int64 while the service sends a JSON string. TODO(DMS): drop once the spec agrees. -type OperationUpdater interface { - // sequenceID is the token the previous update for this resource returned, or 0 for the - // first, which is what staging leaves. + // UpdateOperation is hand-written because the SDK types sequence_id as an int64 while + // the service sends a JSON string, so it cannot read the response. sequenceID is the + // token the previous update for this resource returned, or 0 for the first, which is + // what staging leaves. UpdateOperation(ctx context.Context, deploymentID string, version int64, key ResourceKey, sequenceID string, update OperationUpdate) (next string, err error) } diff --git a/libs/dms/client_test.go b/libs/dms/client_test.go new file mode 100644 index 00000000000..6070c804d64 --- /dev/null +++ b/libs/dms/client_test.go @@ -0,0 +1,107 @@ +package dms + +import ( + "context" + "errors" + "sync" + "testing" + + "github.com/databricks/databricks-sdk-go/service/bundledeployments" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// fakeVersionRequest is a CreateVersion call captured by fakeRaw. +type fakeVersionRequest struct { + deploymentID string + versionID string + body CreateVersionRequest +} + +// updaterCall is an UpdateOperation call captured by fakeRaw. +type updaterCall struct { + deploymentID string + version int64 + key ResourceKey + sequenceID string + update OperationUpdate +} + +// fakeRaw stands in for the hand-written half of a Client, capturing what the CLI would put +// on the wire. +type fakeRaw struct { + mu sync.Mutex + + // versions collects CreateVersion calls, and versionErr fails them. + versions []fakeVersionRequest + versionErr error + + // updates collects UpdateOperation calls. sequence is what the service reports back, and + // the call at index failOn fails instead. + updates []updaterCall + sequence string + failOn int +} + +func newFakeRaw(sequence string) *fakeRaw { + return &fakeRaw{sequence: sequence, failOn: -1} +} + +func (f *fakeRaw) CreateVersion(ctx context.Context, deploymentID, versionID string, body CreateVersionRequest) (*bundledeployments.Version, error) { + f.mu.Lock() + defer f.mu.Unlock() + + f.versions = append(f.versions, fakeVersionRequest{deploymentID: deploymentID, versionID: versionID, body: body}) + if f.versionErr != nil { + return nil, f.versionErr + } + return &bundledeployments.Version{VersionId: versionID}, nil +} + +func (f *fakeRaw) UpdateOperation(ctx context.Context, deploymentID string, version int64, key ResourceKey, sequenceID string, update OperationUpdate) (string, error) { + f.mu.Lock() + defer f.mu.Unlock() + + callNum := len(f.updates) + f.updates = append(f.updates, updaterCall{ + deploymentID: deploymentID, + version: version, + key: key, + sequenceID: sequenceID, + update: update, + }) + if callNum == f.failOn { + return "", errors.New("injected error") + } + return f.sequence, nil +} + +func TestClientNamesEveryResourceTheSameWay(t *testing.T) { + // One format each, so a call only ever passes ids. + assert.Equal(t, "deployments/dep-1", deploymentName("dep-1")) + assert.Equal(t, "deployments/dep-1/versions/2", versionName("dep-1", 2)) +} + +func TestClientFormatsTheVersionIDForCreateVersion(t *testing.T) { + // The version is a number everywhere in the CLI; the request wants it as a string. + raw := newFakeRaw("1") + c := &Client{raw: raw} + + _, err := c.CreateVersion(t.Context(), "dep-1", 5, CreateVersionRequest{}) + require.NoError(t, err) + + require.Len(t, raw.versions, 1) + assert.Equal(t, "5", raw.versions[0].versionID) +} + +func TestDeploymentIDFromName(t *testing.T) { + id, err := deploymentIDFromName("deployments/abc-123") + require.NoError(t, err) + assert.Equal(t, "abc-123", id) + + _, err = deploymentIDFromName("abc-123") + assert.Error(t, err) + + _, err = deploymentIDFromName("deployments/") + assert.Error(t, err) +} diff --git a/libs/dms/recording.go b/libs/dms/recording.go index 6e28bfc9251..cdeafd06407 100644 --- a/libs/dms/recording.go +++ b/libs/dms/recording.go @@ -206,7 +206,7 @@ func (r *recording) Start(ctx context.Context, staged []StagedOperation) (Operat log.Infof(ctx, "Created deployment version: deployment=%s version=%s", r.deploymentID, version.VersionId) return &operationWriter{ - ops: r.client.Operations, + client: r.client, deploymentID: r.deploymentID, version: r.versionNum, sequenceIDs: make(map[ResourceKey]string), diff --git a/libs/dms/recording_test.go b/libs/dms/recording_test.go index 0b4c0689cdf..53624bce51a 100644 --- a/libs/dms/recording_test.go +++ b/libs/dms/recording_test.go @@ -34,32 +34,11 @@ type fakeDMS struct { // captured requests created []bundledeployments.CreateDeploymentRequest - versions []fakeVersionRequest completed []bundledeployments.CompleteVersionRequest deleted []string -} - -// fakeVersionRequest is a CreateVersion call captured by fakeVersions. -type fakeVersionRequest struct { - deploymentID string - versionID string - body CreateVersionRequest -} -// fakeVersions captures CreateVersion calls. It is separate from fakeDMS because -// the CLI does not create versions through the generated client (see -// CreateVersionRequest), so the two use different signatures. -type fakeVersions struct { - requests *[]fakeVersionRequest - err error -} - -func (f fakeVersions) CreateVersion(ctx context.Context, deploymentID, versionID string, body CreateVersionRequest) (*bundledeployments.Version, error) { - *f.requests = append(*f.requests, fakeVersionRequest{deploymentID: deploymentID, versionID: versionID, body: body}) - if f.err != nil { - return nil, f.err - } - return &bundledeployments.Version{VersionId: versionID}, nil + // raw captures what the hand-written half of the client sent; see fakeRaw. + raw *fakeRaw } func (f *fakeDMS) CreateDeployment(ctx context.Context, req bundledeployments.CreateDeploymentRequest) (*bundledeployments.Deployment, error) { @@ -90,7 +69,9 @@ func (f *fakeDMS) Heartbeat(ctx context.Context, req bundledeployments.Heartbeat // testClient wires a Recording to f, failing CreateVersion with versionErr when set. func testClient(f *fakeDMS, versionErr error) *Client { - return &Client{Service: f, Versions: fakeVersions{requests: &f.versions, err: versionErr}} + f.raw = newFakeRaw("1") + f.raw.versionErr = versionErr + return &Client{Service: f, raw: f.raw} } // startVersion creates the version and discards the writer, for a test that only asserts @@ -117,17 +98,17 @@ func TestRecordingFirstDeployCreatesDeploymentWithServerAssignedID(t *testing.T) assert.Equal(t, testStatePath, f.created[0].Deployment.InitialParentPath) // The first version is 1, parented under the assigned deployment. - require.Len(t, f.versions, 1) - assert.Equal(t, "1", f.versions[0].versionID) - assert.Equal(t, "server-generated-id", f.versions[0].deploymentID) + require.Len(t, f.raw.versions, 1) + assert.Equal(t, "1", f.raw.versions[0].versionID) + assert.Equal(t, "server-generated-id", f.raw.versions[0].deploymentID) assert.Equal(t, int64(1), r.Version()) // The service copies display_name onto the deployment's workspace node, which is // where GetDeployment reads it from; a version that omits it leaves the deployment // unnamed in the UI. - assert.Equal(t, testDisplayName, f.versions[0].body.DisplayName) + assert.Equal(t, testDisplayName, f.raw.versions[0].body.DisplayName) // A first version supersedes nothing, so previous_version_id is unset. - assert.Empty(t, f.versions[0].body.PreviousVersionId) + assert.Empty(t, f.raw.versions[0].body.PreviousVersionId) require.NoError(t, r.Finish(t.Context(), true)) require.Len(t, f.completed, 1) @@ -148,12 +129,12 @@ func TestRecordingSubsequentDeployReusesDeploymentAndIncrementsVersion(t *testin // No new deployment is created; the version increments to last_version_id + 1. assert.Empty(t, f.created) - require.Len(t, f.versions, 1) - assert.Equal(t, "5", f.versions[0].versionID) + require.Len(t, f.raw.versions, 1) + assert.Equal(t, "5", f.raw.versions[0].versionID) assert.Equal(t, "stored-id", r.DeploymentID()) // The version it supersedes is the concurrency check; without it the service // rejects every deploy after the first. - assert.Equal(t, "4", f.versions[0].body.PreviousVersionId) + assert.Equal(t, "4", f.raw.versions[0].body.PreviousVersionId) } func TestRecordingGetDeploymentErrorFailsDeploy(t *testing.T) { @@ -183,7 +164,7 @@ func TestRecordingMissingDeploymentIsInternalError(t *testing.T) { _, err := r.Start(t.Context(), nil) assert.ErrorContains(t, err, "internal error: no deployment found for the file with object id stored-id") assert.Empty(t, f.created) - assert.Empty(t, f.versions) + assert.Empty(t, f.raw.versions) } func TestRecordingDestroyDeletesDeploymentOnSuccess(t *testing.T) { @@ -195,7 +176,7 @@ func TestRecordingDestroyDeletesDeploymentOnSuccess(t *testing.T) { r := NewRecording(RecordingOptions{Client: testClient(f, nil), DeploymentID: "stored-id", StatePath: testStatePath, Metadata: Metadata{TargetName: "dev", DisplayName: testDisplayName}, VersionType: VersionTypeDestroy}) startVersion(t, r, nil) - assert.Equal(t, bundledeployments.VersionTypeVersionTypeDestroy, f.versions[0].body.VersionType) + assert.Equal(t, bundledeployments.VersionTypeVersionTypeDestroy, f.raw.versions[0].body.VersionType) require.NoError(t, r.Finish(t.Context(), true)) // A successful destroy deletes the deployment record. @@ -275,7 +256,7 @@ func TestRecordingPrepareClaimsNoVersion(t *testing.T) { require.NoError(t, r.Prepare(t.Context())) assert.Equal(t, int64(5), r.Version()) - assert.Empty(t, f.versions, "no version created") + assert.Empty(t, f.raw.versions, "no version created") require.NoError(t, r.Finish(t.Context(), true)) assert.Empty(t, f.completed, "nothing to complete") @@ -294,9 +275,9 @@ func TestRecordingStartUsesThePreparedNumber(t *testing.T) { // The version created is the one the plan was stamped with, and it reports the // version it supersedes so the service rejects a racing deploy. - require.Len(t, f.versions, 1) - assert.Equal(t, "5", f.versions[0].versionID) - assert.Equal(t, "4", f.versions[0].body.PreviousVersionId) + require.Len(t, f.raw.versions, 1) + assert.Equal(t, "5", f.raw.versions[0].versionID) + assert.Equal(t, "4", f.raw.versions[0].body.PreviousVersionId) assert.Equal(t, int64(5), r.Version()) } @@ -329,18 +310,6 @@ func TestRecordingStartDetectsAbortedConflict(t *testing.T) { assert.ErrorIs(t, err, conflictErr) } -func TestDeploymentIDFromName(t *testing.T) { - id, err := deploymentIDFromName("deployments/abc-123") - require.NoError(t, err) - assert.Equal(t, "abc-123", id) - - _, err = deploymentIDFromName("abc-123") - assert.Error(t, err) - - _, err = deploymentIDFromName("deployments/") - assert.Error(t, err) -} - func TestRecordingStartStagesOperations(t *testing.T) { // The version fixes its operation set, so what the caller passes has to reach the wire // verbatim: the service has no API to add an operation later. @@ -353,8 +322,8 @@ func TestRecordingStartStagesOperations(t *testing.T) { } startVersion(t, r, staged) - require.Len(t, f.versions, 1) - assert.Equal(t, staged, f.versions[0].body.Operations) + require.Len(t, f.raw.versions, 1) + assert.Equal(t, staged, f.raw.versions[0].body.Operations) } func TestRecordingStartReportsTheOperationCap(t *testing.T) { diff --git a/libs/dms/writer.go b/libs/dms/writer.go index 1051c379fce..668574cbd0d 100644 --- a/libs/dms/writer.go +++ b/libs/dms/writer.go @@ -17,7 +17,7 @@ type OperationWriter interface { // operationWriter writes through the API, tracking the sequence id each resource is at. type operationWriter struct { - ops OperationUpdater + client *Client deploymentID string version int64 @@ -35,7 +35,7 @@ func (w *operationWriter) Write(ctx context.Context, key ResourceKey, update Ope sequenceID = stagedSequenceID } - next, err := w.ops.UpdateOperation(ctx, w.deploymentID, w.version, key, sequenceID, update) + next, err := w.client.UpdateOperation(ctx, w.deploymentID, w.version, key, sequenceID, update) if err != nil { return err } diff --git a/libs/dms/writer_test.go b/libs/dms/writer_test.go index bc3f4ebb0af..fbe5aa5d05c 100644 --- a/libs/dms/writer_test.go +++ b/libs/dms/writer_test.go @@ -1,10 +1,8 @@ package dms import ( - "context" "encoding/json" "errors" - "sync" "testing" "github.com/databricks/databricks-sdk-go/service/bundledeployments" @@ -12,49 +10,10 @@ import ( "github.com/stretchr/testify/require" ) -// updaterCall is one call the writer made to the operations API. -type updaterCall struct { - deploymentID string - version int64 - key ResourceKey - sequenceID string - update OperationUpdate -} - -// fakeUpdater reports sequence for every call, failing the one at index failOn. -type fakeUpdater struct { - mu sync.Mutex - calls []updaterCall - sequence string - failOn int -} - -func newFakeUpdater(sequence string) *fakeUpdater { - return &fakeUpdater{sequence: sequence, failOn: -1} -} - -func (f *fakeUpdater) UpdateOperation(ctx context.Context, deploymentID string, version int64, key ResourceKey, sequenceID string, update OperationUpdate) (string, error) { - f.mu.Lock() - defer f.mu.Unlock() - - callNum := len(f.calls) - f.calls = append(f.calls, updaterCall{ - deploymentID: deploymentID, - version: version, - key: key, - sequenceID: sequenceID, - update: update, - }) - if callNum == f.failOn { - return "", errors.New("injected error") - } - return f.sequence, nil -} - -// testWriter returns a writer for version 2 of dep-1, recording through f. -func testWriter(f OperationUpdater) OperationWriter { +// testWriter returns a writer for version 2 of dep-1, recording through raw. +func testWriter(raw *fakeRaw) OperationWriter { return &operationWriter{ - ops: f, + client: &Client{raw: raw}, deploymentID: "dep-1", version: 2, sequenceIDs: make(map[ResourceKey]string), @@ -69,13 +28,13 @@ func writeState(t *testing.T, w OperationWriter, key ResourceKey, resourceID str } func TestWriterFirstWriteUpdatesTheStagedOperation(t *testing.T) { - f := newFakeUpdater("1") + f := newFakeRaw("1") w := testWriter(f) writeState(t, w, "jobs.foo", "job-123", json.RawMessage(`{"state":{}}`)) - require.Len(t, f.calls, 1) - c := f.calls[0] + require.Len(t, f.updates, 1) + c := f.updates[0] // The version already staged this operation, so the first write updates it and echoes // the sequence id staging left. assert.Equal(t, "dep-1", c.deploymentID) @@ -88,35 +47,35 @@ func TestWriterFirstWriteUpdatesTheStagedOperation(t *testing.T) { func TestWriterSecondWriteEchoesTheServiceSequence(t *testing.T) { // One operation per resource per version: the second write updates the same operation, // echoing the sequence id the service returned as its precondition. - f := newFakeUpdater("7") + f := newFakeRaw("7") w := testWriter(f) writeState(t, w, "jobs.foo", "", nil) writeState(t, w, "jobs.foo", "job-456", json.RawMessage(`{"state":{}}`)) - require.Len(t, f.calls, 2) - assert.Equal(t, stagedSequenceID, f.calls[0].sequenceID) - assert.Equal(t, "7", f.calls[1].sequenceID) + require.Len(t, f.updates, 2) + assert.Equal(t, stagedSequenceID, f.updates[0].sequenceID) + assert.Equal(t, "7", f.updates[1].sequenceID) } func TestWriterTracksSequencePerResource(t *testing.T) { // Each resource has its own staged operation, so each one's first write echoes the staged // sequence id rather than a sequence another resource earned. - f := newFakeUpdater("1") + f := newFakeRaw("1") w := testWriter(f) writeState(t, w, "jobs.foo", "id-1", json.RawMessage(`{"state":{}}`)) writeState(t, w, "jobs.bar", "id-2", json.RawMessage(`{"state":{}}`)) - require.Len(t, f.calls, 2) - assert.Equal(t, stagedSequenceID, f.calls[0].sequenceID) - assert.Equal(t, stagedSequenceID, f.calls[1].sequenceID) + require.Len(t, f.updates, 2) + assert.Equal(t, stagedSequenceID, f.updates[0].sequenceID) + assert.Equal(t, stagedSequenceID, f.updates[1].sequenceID) } func TestWriterErrorKeepsTheSequence(t *testing.T) { // A failed write returns its error and leaves the recorded sequence id alone, so a later // write for the same resource still carries the precondition the service last gave us. - f := newFakeUpdater("9") + f := newFakeRaw("9") f.failOn = 1 w := testWriter(f) @@ -130,8 +89,8 @@ func TestWriterErrorKeepsTheSequence(t *testing.T) { // The third write is what proves the sequence id survived the failure. writeState(t, w, "jobs.foo", "job-3", json.RawMessage(`{"state":{}}`)) - require.Len(t, f.calls, 3) - assert.Equal(t, "9", f.calls[2].sequenceID) + require.Len(t, f.updates, 3) + assert.Equal(t, "9", f.updates[2].sequenceID) } func TestUpdateRequestSendsOnlyWhatTheMaskNames(t *testing.T) { diff --git a/libs/testserver/bundle_test.go b/libs/testserver/bundle_test.go deleted file mode 100644 index c9bfa82f533..00000000000 --- a/libs/testserver/bundle_test.go +++ /dev/null @@ -1,146 +0,0 @@ -package testserver - -import ( - "encoding/json" - "net/url" - "testing" - - "github.com/databricks/databricks-sdk-go/service/bundledeployments" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -// stageOperation creates a deployment with one version that stages an operation for -// resourceKey, and returns the deployment and version ids. -func stageOperation(t *testing.T, s *FakeWorkspace, resourceKey string) (string, string) { - t.Helper() - - parent := "/Users/" + TestUser.UserName - resp := s.CreateDeployment(Request{Body: []byte(`{"initial_parent_path":"` + parent + `"}`)}) - require.Equal(t, 0, resp.StatusCode, resp.Body) - var dep bundledeployments.Deployment - remarshal(t, resp.Body, &dep) - deploymentID := dep.Name[len("deployments/"):] - - body := `{"version_type":"VERSION_TYPE_DEPLOY","operations":[{"resource_key":"` + resourceKey + `","action_type":"OPERATION_ACTION_TYPE_UPDATE"}]}` - resp = s.CreateVersion(Request{ - Body: []byte(body), - URL: &url.URL{RawQuery: "version_id=1"}, - }, deploymentID) - require.Equal(t, 0, resp.StatusCode, resp.Body) - - return deploymentID, "1" -} - -// updateOperation applies one update, and returns the sequence id for the next one. -func updateOperation(t *testing.T, s *FakeWorkspace, deploymentID, versionID, resourceKey, mask, body string) string { - t.Helper() - - resp := s.UpdateOperation(Request{ - Body: []byte(body), - URL: &url.URL{RawQuery: "update_mask=" + url.QueryEscape(mask)}, - }, deploymentID, versionID, resourceKey) - require.Equal(t, 0, resp.StatusCode, resp.Body) - - // The response types sequence_id as a string, which is why the SDK cannot read it. - var raw map[string]any - remarshal(t, resp.Body, &raw) - return raw["sequence_id"].(string) -} - -func listResources(t *testing.T, s *FakeWorkspace, deploymentID string) map[string]bundledeployments.Resource { - t.Helper() - - resp := s.ListResources(deploymentID) - require.Equal(t, 0, resp.StatusCode, resp.Body) - - var listed bundledeployments.ListResourcesResponse - remarshal(t, resp.Body, &listed) - - out := map[string]bundledeployments.Resource{} - for _, r := range listed.Resources { - out[r.ResourceKey] = r - } - return out -} - -func remarshal(t *testing.T, from, into any) { - t.Helper() - raw, err := json.Marshal(from) - require.NoError(t, err) - require.NoError(t, json.Unmarshal(raw, into)) -} - -// TestUpdateOperationProjectionFollowsTheMask pins the rule the CLI's update masks are -// built around: the deployment-level resource - what the next plan reads - moves only when -// the mask names state. An update that leaves state out reports an outcome and must not -// disturb what the deployment already holds. -func TestUpdateOperationProjectionFollowsTheMask(t *testing.T) { - const key = "jobs.foo" - const state = `{\"state\":{\"name\":\"foo\"}}` - - tests := []struct { - name string - // updates are applied in order, as (mask, body) pairs. The sequence id is filled in. - updates [][2]string - // wantResource is the state the deployment holds afterwards, empty for no resource. - wantResource string - wantID string - }{ - { - name: "a write that names state records the resource", - updates: [][2]string{ - {"state,error_message,resource_id,status", `{"state":"` + state + `","resource_id":"job-1","status":"OPERATION_STATUS_SUCCEEDED"}`}, - }, - wantResource: `{"state":{"name":"foo"}}`, - wantID: "job-1", - }, - { - name: "naming state with no value removes the resource", - updates: [][2]string{ - {"state,error_message,resource_id,status", `{"state":"` + state + `","resource_id":"job-1","status":"OPERATION_STATUS_SUCCEEDED"}`}, - {"state,error_message,resource_id,status", `{"resource_id":"job-1","status":"OPERATION_STATUS_SUCCEEDED"}`}, - }, - wantResource: "", - }, - { - name: "a failure that leaves state out keeps the recorded resource", - updates: [][2]string{ - {"state,error_message,resource_id,status", `{"state":"` + state + `","resource_id":"job-1","status":"OPERATION_STATUS_SUCCEEDED"}`}, - {"error_message,status", `{"error_message":"boom","status":"OPERATION_STATUS_FAILED"}`}, - }, - wantResource: `{"state":{"name":"foo"}}`, - wantID: "job-1", - }, - { - name: "a failure before any write records no resource", - updates: [][2]string{ - {"error_message,status", `{"error_message":"boom","status":"OPERATION_STATUS_FAILED"}`}, - }, - wantResource: "", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - s := NewFakeWorkspace("http://localhost", "test-token") - deploymentID, versionID := stageOperation(t, s, key) - - sequenceID := "0" - for _, u := range tt.updates { - mask, body := u[0], u[1] - withSequence := body[:len(body)-1] + `,"sequence_id":"` + sequenceID + `"}` - sequenceID = updateOperation(t, s, deploymentID, versionID, key, mask, withSequence) - } - - resources := listResources(t, s, deploymentID) - if tt.wantResource == "" { - assert.NotContains(t, resources, key) - return - } - require.Contains(t, resources, key) - assert.JSONEq(t, tt.wantResource, resources[key].State) - assert.Equal(t, tt.wantID, resources[key].ResourceId) - }) - } -} From c9d51e89c111611fb550a385ab1012d2f232fa79 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 18 Aug 2026 22:46:27 +0000 Subject: [PATCH 113/125] dstate: say why the sink comes back from the locked section Co-authored-by: Isaac --- bundle/direct/dstate/state.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bundle/direct/dstate/state.go b/bundle/direct/dstate/state.go index 20425bcb8af..cdd814ef295 100644 --- a/bundle/direct/dstate/state.go +++ b/bundle/direct/dstate/state.go @@ -145,9 +145,9 @@ func (db *DeploymentState) SaveState(ctx context.Context, key, newID string, sta return err } - // Recorded after the WAL write, so DMS never reports state the deploy failed to persist, - // and outside the lock because recording waits when the service is behind - waiting under - // db.mu would hold up every other resource's write. + // Recorded here, after the WAL write, so DMS never reports state the deploy failed to + // persist, and outside db.mu because recording waits when the service is behind - which is + // also why the sink comes back from the locked section rather than being read here. if sink != nil { sink.RecordOperation(ctx, key, false, newID, recorded) } From bd964ecd4d913b45927aa0435560d2c22774d05f Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 18 Aug 2026 22:53:22 +0000 Subject: [PATCH 114/125] direct: fix a stale comment about what a failed operation carries recordFailure took a priorState argument once; it does not now, so the comment described a hazard the code cannot have. Co-authored-by: Isaac --- bundle/direct/opsink_test.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/bundle/direct/opsink_test.go b/bundle/direct/opsink_test.go index dcabe8cbb8c..b3cba5fc600 100644 --- a/bundle/direct/opsink_test.go +++ b/bundle/direct/opsink_test.go @@ -174,8 +174,9 @@ func TestOperationSinkCoalescedFailureKeepsTheStateItReplaces(t *testing.T) { } func TestOperationSinkCoalescedFailureAfterADeleteKeepsTheResourceGone(t *testing.T) { - // The recreate's delete writes no state. When the create then fails, the failure takes - // that absent state rather than the pre-deploy one, so the resource stays gone. + // A recreate's delete writes no state, and then the create fails. The failure claims no + // state either, so the delete's emptiness survives the merge - and the merged mask still + // names state, which is what tells the service the resource really is gone. f := &fakeWriter{block: make(chan struct{}), started: make(chan dms.ResourceKey, 2)} s := newOperationSink(t.Context(), f) From 17437dc2ff6bf6c3ae1f45b4c9488f26b6d3a8cc Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 19 Aug 2026 08:49:18 +0000 Subject: [PATCH 115/125] direct: drop sink tests that only re-test the merge Four of them drove a write and a failure through the queue to assert which fields survived - which is OperationUpdate.Merge, tested directly in libs/dms. What is left is what only the sink does: coalescing behind an in-flight write, backpressure when the queue fills, and how a failure reaches the deploy. The request-body tests fold into one table: same values every case, so only the mask decides what is sent. Co-authored-by: Isaac --- bundle/direct/opsink_test.go | 169 ++--------------------------------- libs/dms/writer_test.go | 84 +++++++++-------- 2 files changed, 57 insertions(+), 196 deletions(-) diff --git a/bundle/direct/opsink_test.go b/bundle/direct/opsink_test.go index b3cba5fc600..281bcb928a9 100644 --- a/bundle/direct/opsink_test.go +++ b/bundle/direct/opsink_test.go @@ -12,7 +12,6 @@ import ( "github.com/databricks/cli/bundle/direct/dstate" "github.com/databricks/cli/libs/dms" - "github.com/databricks/databricks-sdk-go/service/bundledeployments" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -26,11 +25,8 @@ type fakeWriter struct { started chan dms.ResourceKey err error - mu sync.Mutex - writes []string - resourceIDs map[dms.ResourceKey]string - statuses map[dms.ResourceKey]bundledeployments.OperationStatus - errorMessages map[dms.ResourceKey]string + mu sync.Mutex + writes []string } func (f *fakeWriter) Write(ctx context.Context, key dms.ResourceKey, update dms.OperationUpdate) error { @@ -43,14 +39,6 @@ func (f *fakeWriter) Write(ctx context.Context, key dms.ResourceKey, update dms. f.mu.Lock() f.writes = append(f.writes, string(key)+"="+string(update.State)) - if f.resourceIDs == nil { - f.resourceIDs = map[dms.ResourceKey]string{} - f.statuses = map[dms.ResourceKey]bundledeployments.OperationStatus{} - f.errorMessages = map[dms.ResourceKey]string{} - } - f.resourceIDs[key] = update.ResourceID - f.statuses[key] = update.Status - f.errorMessages[key] = update.ErrorMessage f.mu.Unlock() return f.err @@ -62,24 +50,6 @@ func (f *fakeWriter) recorded() []string { return append([]string(nil), f.writes...) } -func (f *fakeWriter) resourceIDFor(key dms.ResourceKey) string { - f.mu.Lock() - defer f.mu.Unlock() - return f.resourceIDs[key] -} - -func (f *fakeWriter) statusFor(key dms.ResourceKey) bundledeployments.OperationStatus { - f.mu.Lock() - defer f.mu.Unlock() - return f.statuses[key] -} - -func (f *fakeWriter) errorMessageFor(key dms.ResourceKey) string { - f.mu.Lock() - defer f.mu.Unlock() - return f.errorMessages[key] -} - // envelope builds the serialized RecordedState the state DB hands the sink. func envelope(t *testing.T, name string) json.RawMessage { t.Helper() @@ -93,18 +63,6 @@ func recordState(t *testing.T, s *operationSink, resourceKey, name string) { s.RecordOperation(t.Context(), resourceKey, false, "id-1", envelope(t, name)) } -func TestOperationSinkWritesEachOperation(t *testing.T) { - f := &fakeWriter{} - s := newOperationSink(t.Context(), f) - - for i := range 20 { - recordState(t, s, "resources.jobs.job"+strconv.Itoa(i), "n") - } - require.NoError(t, s.close()) - - assert.Len(t, f.recorded(), 20) -} - func TestOperationSinkKeepsWritingAfterGoingIdle(t *testing.T) { // The writer parks on an empty queue instead of returning. Apply spends most of a // deploy inside resource CRUD, so the queue is empty far more often than not, and a @@ -144,103 +102,6 @@ func TestOperationSinkCoalescesWritesBehindAWrite(t *testing.T) { }, f.recorded()) } -func TestOperationSinkCoalescedFailureKeepsTheStateItReplaces(t *testing.T) { - // The create writes state and then fails before the write goes out. The failure must not - // replace that state with its own emptiness, which would drop the resource from the - // deployment. - f := &fakeWriter{block: make(chan struct{}), started: make(chan dms.ResourceKey, 2)} - s := newOperationSink(t.Context(), f) - - // Occupy the writer with an unrelated resource, so the two writes below both - // land in pending and coalesce. - recordState(t, s, "resources.jobs.busy", "v1") - assert.Equal(t, dms.ResourceKey("jobs.busy"), <-f.started) - - s.RecordOperation(t.Context(), "resources.job_runs.my_run", false, "run-1", envelope(t, "the run")) - // The id is empty: the resource was created in this deploy, so there is no pre-deploy - // record to report. - s.recordFailure("resources.job_runs.my_run", "", errors.New("run did not succeed: FAILED")) - - close(f.block) - require.NoError(t, s.close()) - - assert.Equal(t, []string{ - `jobs.busy={"state":{"name":"v1"}}`, - `job_runs.my_run={"state":{"name":"the run"}}`, - }, f.recorded()) - assert.Equal(t, "run-1", f.resourceIDFor("job_runs.my_run")) - assert.Equal(t, bundledeployments.OperationStatusOperationStatusFailed, f.statusFor("job_runs.my_run")) - assert.Equal(t, "run did not succeed: FAILED", f.errorMessageFor("job_runs.my_run")) -} - -func TestOperationSinkCoalescedFailureAfterADeleteKeepsTheResourceGone(t *testing.T) { - // A recreate's delete writes no state, and then the create fails. The failure claims no - // state either, so the delete's emptiness survives the merge - and the merged mask still - // names state, which is what tells the service the resource really is gone. - f := &fakeWriter{block: make(chan struct{}), started: make(chan dms.ResourceKey, 2)} - s := newOperationSink(t.Context(), f) - - recordState(t, s, "resources.jobs.busy", "v1") - assert.Equal(t, dms.ResourceKey("jobs.busy"), <-f.started) - - s.RecordOperation(t.Context(), "resources.schemas.foo", true, "old-id", nil) - s.recordFailure("resources.schemas.foo", "old-id", errors.New("Catalog 'other' does not exist")) - - close(f.block) - require.NoError(t, s.close()) - - assert.Equal(t, []string{ - `jobs.busy={"state":{"name":"v1"}}`, - `schemas.foo=`, - }, f.recorded()) - assert.Equal(t, bundledeployments.OperationStatusOperationStatusFailed, f.statusFor("schemas.foo")) -} - -func TestOperationSinkCoalescedFailureDoesNotRevertToPriorState(t *testing.T) { - // An update that succeeded and then failed waiting carries the pre-deploy id, which the - // write it supersedes has moved past. Reporting it would record the resource as it was - // before the deploy, and the next plan would read that back as current. - f := &fakeWriter{block: make(chan struct{}), started: make(chan dms.ResourceKey, 2)} - s := newOperationSink(t.Context(), f) - - recordState(t, s, "resources.jobs.busy", "v1") - assert.Equal(t, dms.ResourceKey("jobs.busy"), <-f.started) - - s.RecordOperation(t.Context(), "resources.jobs.foo", false, "id-new", envelope(t, "after the update")) - s.recordFailure("resources.jobs.foo", "id-old", errors.New("waiting after updating: timed out")) - - close(f.block) - require.NoError(t, s.close()) - - assert.Equal(t, []string{ - `jobs.busy={"state":{"name":"v1"}}`, - `jobs.foo={"state":{"name":"after the update"}}`, - }, f.recorded()) - assert.Equal(t, "id-new", f.resourceIDFor("jobs.foo")) - assert.Equal(t, bundledeployments.OperationStatusOperationStatusFailed, f.statusFor("jobs.foo")) -} - -func TestOperationSinkCoalescedDeleteStillClearsState(t *testing.T) { - // A delete legitimately carries no state, and coalescing must let it through: the - // resource is gone, and keeping the state it replaces would leave it listed. - f := &fakeWriter{block: make(chan struct{}), started: make(chan dms.ResourceKey, 2)} - s := newOperationSink(t.Context(), f) - - recordState(t, s, "resources.jobs.busy", "v1") - assert.Equal(t, dms.ResourceKey("jobs.busy"), <-f.started) - - s.RecordOperation(t.Context(), "resources.jobs.foo", false, "id-1", envelope(t, "before")) - s.RecordOperation(t.Context(), "resources.jobs.foo", false, "id-1", nil) - - close(f.block) - require.NoError(t, s.close()) - - assert.Equal(t, []string{ - `jobs.busy={"state":{"name":"v1"}}`, - `jobs.foo=`, - }, f.recorded()) -} - func TestOperationSinkRecordDuringWriteIsStillWritten(t *testing.T) { f := &fakeWriter{block: make(chan struct{}), started: make(chan dms.ResourceKey, 2)} s := newOperationSink(t.Context(), f) @@ -305,12 +166,16 @@ func TestOperationSinkReturnsWriteError(t *testing.T) { f := &fakeWriter{err: writeErr} s := newOperationSink(t.Context(), f) + assert.NoError(t, s.firstErr()) + recordState(t, s, "resources.jobs.foo", "v1") err := s.close() require.Error(t, err) assert.ErrorIs(t, err, writeErr) assert.ErrorContains(t, err, "resources.jobs.foo") + // Reported after the fact too, so apply can check between resources. + assert.Error(t, s.firstErr()) } func TestOperationSinkKeepsRecordingAfterWriteError(t *testing.T) { @@ -326,20 +191,6 @@ func TestOperationSinkKeepsRecordingAfterWriteError(t *testing.T) { assert.Len(t, f.recorded(), 2) } -func TestOperationSinkFirstErrIsWhatStopsTheDeploy(t *testing.T) { - f := &fakeWriter{err: errors.New("boom")} - s := newOperationSink(t.Context(), f) - - assert.NoError(t, s.firstErr()) - - recordState(t, s, "resources.jobs.foo", "v1") - require.Error(t, s.close()) - - // Reported after the fact too, so the caller can check once more before it - // completes the version. - assert.Error(t, s.firstErr()) -} - func TestOperationSinkFailsOnOversizedState(t *testing.T) { // The service will not take a state this large (the limit lives in libs/dms), so the // resource cannot be recorded. Failing here says so, where reporting nothing would leave @@ -367,14 +218,12 @@ func TestOperationSinkCloseIsIdempotent(t *testing.T) { } func TestNilOperationSinkIsNoOp(t *testing.T) { + // Recording off: the writer is nil, so the sink is too, and every method still works. + require.Nil(t, newOperationSink(t.Context(), nil)) + var s *operationSink s.RecordOperation(t.Context(), "resources.jobs.foo", false, "id-1", nil) s.recordFailure("resources.jobs.foo", "id-1", errors.New("boom")) assert.NoError(t, s.firstErr()) assert.NoError(t, s.close()) } - -func TestNewOperationSinkNilWriterIsNil(t *testing.T) { - // Recording off: the sink is nil so the state DB's nil check leaves it unset. - assert.Nil(t, newOperationSink(t.Context(), nil)) -} diff --git a/libs/dms/writer_test.go b/libs/dms/writer_test.go index fbe5aa5d05c..3e2e2c1738f 100644 --- a/libs/dms/writer_test.go +++ b/libs/dms/writer_test.go @@ -2,7 +2,6 @@ package dms import ( "encoding/json" - "errors" "testing" "github.com/databricks/databricks-sdk-go/service/bundledeployments" @@ -93,43 +92,56 @@ func TestWriterErrorKeepsTheSequence(t *testing.T) { assert.Equal(t, "9", f.updates[2].sequenceID) } -func TestUpdateRequestSendsOnlyWhatTheMaskNames(t *testing.T) { - // A failure keeps whatever state an earlier write recorded, so it must send neither - // state nor resource_id: an empty state would drop the resource from the deployment. - failure := NewFailureUpdate("job-1", errors.New("boom")) - - body := newUpdateRequest(failure, "3") - - assert.Empty(t, body.State) - assert.Empty(t, body.ResourceId) - assert.Equal(t, "3", body.SequenceId) - assert.Equal(t, "boom", body.ErrorMessage) - assert.Equal(t, bundledeployments.OperationStatusOperationStatusFailed, body.Status) -} - -func TestUpdateRequestSendsStateWhenNamed(t *testing.T) { - update, err := NewStateUpdate("job-1", json.RawMessage(`{"state":{"name":"foo"}}`), false) - require.NoError(t, err) - - body := newUpdateRequest(update, stagedSequenceID) - - assert.JSONEq(t, `{"state":{"name":"foo"}}`, body.State) - assert.Equal(t, "job-1", body.ResourceId) -} - -func TestUpdateRequestSendsEachFieldOnItsOwnMaskEntry(t *testing.T) { - // resource_id does not travel with state: a mask that names one and not the other sends - // exactly that. +func TestUpdateRequestSendsAFieldOnlyWhenTheMaskNamesIt(t *testing.T) { + // Every case carries the same values, so what reaches the body is decided by the mask + // alone. A failure sending state would drop the resource from the deployment, and + // resource_id does not ride along with state. update := OperationUpdate{ - Fields: FieldResourceID | FieldStatus, - State: json.RawMessage(`{"state":{"name":"foo"}}`), - ResourceID: "job-1", - Status: bundledeployments.OperationStatusOperationStatusSucceeded, + State: json.RawMessage(`{"state":{"name":"foo"}}`), + ResourceID: "job-1", + Status: bundledeployments.OperationStatusOperationStatusSucceeded, + ErrorMessage: "boom", } - body := newUpdateRequest(update, "4") + tests := []struct { + name string + fields Fields + want updateOperationRequest + }{ + { + name: "a write that describes the resource", + fields: DescribesResource, + want: updateOperationRequest{ + State: `{"state":{"name":"foo"}}`, + ResourceId: "job-1", + Status: bundledeployments.OperationStatusOperationStatusSucceeded, + ErrorMessage: "boom", + SequenceId: "3", + }, + }, + { + name: "a failure that keeps the recorded state", + fields: KeepsState, + want: updateOperationRequest{ + Status: bundledeployments.OperationStatusOperationStatusSucceeded, + ErrorMessage: "boom", + SequenceId: "3", + }, + }, + { + name: "resource_id without state", + fields: FieldResourceID, + want: updateOperationRequest{ + ResourceId: "job-1", + SequenceId: "3", + }, + }, + } - assert.Empty(t, body.State) - assert.Equal(t, "job-1", body.ResourceId) - assert.Equal(t, bundledeployments.OperationStatusOperationStatusSucceeded, body.Status) + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + update.Fields = tt.fields + assert.Equal(t, tt.want, newUpdateRequest(update, "3")) + }) + } } From 06fe6d70582930ef4bb0e66cca121c897d57cde7 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 19 Aug 2026 10:56:02 +0000 Subject: [PATCH 116/125] dstate: inline the state write helpers saveStateEntry and deleteStateEntry existed to hand the sink out from under a deferred unlock. One explicit Lock/Unlock pair does the same thing in one function: the marshal happens before the lock, the WAL write is the only fallible work under it, and the envelope is serialized after - so recording being off still costs nothing. Also stops a test comment claiming where apply spends its time. Co-authored-by: Isaac --- bundle/direct/dstate/state.go | 92 +++++++++++++---------------------- bundle/direct/opsink_test.go | 6 +-- 2 files changed, 37 insertions(+), 61 deletions(-) diff --git a/bundle/direct/dstate/state.go b/bundle/direct/dstate/state.go index cdd814ef295..42efdf03511 100644 --- a/bundle/direct/dstate/state.go +++ b/bundle/direct/dstate/state.go @@ -140,59 +140,44 @@ func NewDatabase(lineage string, serial int) Database { func (db *DeploymentState) SaveState(ctx context.Context, key, newID string, state any, dependsOn []deployplan.DependsOnEntry) error { db.AssertOpenedForWrite() - sink, recorded, err := db.saveStateEntry(key, newID, state, dependsOn) + jsonMessage, err := json.Marshal(state) if err != nil { return err } - - // Recorded here, after the WAL write, so DMS never reports state the deploy failed to - // persist, and outside db.mu because recording waits when the service is behind - which is - // also why the sink comes back from the locked section rather than being read here. - if sink != nil { - sink.RecordOperation(ctx, key, false, newID, recorded) + entry := ResourceEntry{ + ID: newID, + State: json.RawMessage(jsonMessage), + DependsOn: dependsOn, } - return nil -} - -// saveStateEntry writes the resource's state and returns the sink to report it to, -// along with the serialized envelope to report, or a nil sink when the bundle does not -// record deployment history. -func (db *DeploymentState) saveStateEntry(key, newID string, state any, dependsOn []deployplan.DependsOnEntry) (OperationSink, json.RawMessage, error) { db.mu.Lock() - defer db.mu.Unlock() - if db.Data.State == nil { db.Data.State = make(map[string]ResourceEntry) } - - jsonMessage, err := json.Marshal(state) - if err != nil { - return nil, nil, err - } - - entry := ResourceEntry{ - ID: newID, - State: json.RawMessage(jsonMessage), - DependsOn: dependsOn, + err = appendJSONLine(db.walFile, WALEntry{Key: key, Value: &entry}) + if err == nil { + db.stateIDs[key] = newID } + sink := db.sink + db.mu.Unlock() - err = appendJSONLine(db.walFile, WALEntry{Key: key, Value: &entry}) if err != nil { - return nil, nil, err + return err } - db.stateIDs[key] = newID - - if db.sink == nil { - return nil, nil, nil + if sink == nil { + return nil } - // Serialized here, while the entry the WAL took is still to hand. + // Recorded after the WAL write, so DMS never reports state the deploy failed to persist, + // and outside db.mu because recording waits when the service is behind - waiting under the + // lock would hold up every other resource's write. recorded, err := json.Marshal(RecordedState{State: entry.State, DependsOn: dependsOn}) if err != nil { - return nil, nil, err + return err } - return db.sink, recorded, nil + sink.RecordOperation(ctx, key, false, newID, recorded) + + return nil } // DeleteState drops the resource's state entry: the resource is gone. @@ -210,7 +195,20 @@ func (db *DeploymentState) DeleteStateForRecreate(ctx context.Context, key strin func (db *DeploymentState) deleteState(ctx context.Context, key string, inProgress bool) error { db.AssertOpenedForWrite() - sink, deletedID, err := db.deleteStateEntry(key) + db.mu.Lock() + if db.Data.State == nil { + db.mu.Unlock() + return nil + } + // Read before the delete: DMS needs the id to say which resource went away. + deletedID := db.stateIDs[key] + err := appendJSONLine(db.walFile, WALEntry{Key: key}) + if err == nil { + delete(db.stateIDs, key) + } + sink := db.sink + db.mu.Unlock() + if err != nil { return err } @@ -224,28 +222,6 @@ func (db *DeploymentState) deleteState(ctx context.Context, key string, inProgre return nil } -// deleteStateEntry drops the resource's state entry and returns the sink to report it -// to, along with the id it had, or a nil sink when there is nothing to report. -func (db *DeploymentState) deleteStateEntry(key string) (OperationSink, string, error) { - db.mu.Lock() - defer db.mu.Unlock() - - if db.Data.State == nil { - return nil, "", nil - } - - // Read before the delete: DMS needs the id to say which resource went away. - deletedID := db.stateIDs[key] - - err := appendJSONLine(db.walFile, WALEntry{Key: key}) - if err != nil { - return nil, "", err - } - delete(db.stateIDs, key) - - return db.sink, deletedID, nil -} - func (db *DeploymentState) GetResourceEntry(key string) (ResourceEntry, bool) { // Note, if opened for write, you get the state that you had at the beginning of deploy, not most recent one db.AssertOpenedForReadOrWrite() diff --git a/bundle/direct/opsink_test.go b/bundle/direct/opsink_test.go index 281bcb928a9..f8e8fb5ea99 100644 --- a/bundle/direct/opsink_test.go +++ b/bundle/direct/opsink_test.go @@ -64,9 +64,9 @@ func recordState(t *testing.T, s *operationSink, resourceKey, name string) { } func TestOperationSinkKeepsWritingAfterGoingIdle(t *testing.T) { - // The writer parks on an empty queue instead of returning. Apply spends most of a - // deploy inside resource CRUD, so the queue is empty far more often than not, and a - // writer that exited while idle would silently drop everything recorded after it. + // The writer parks on an empty queue instead of returning. Apply can spend long stretches + // inside resource CRUD with nothing to record, and a writer that exited while idle would + // silently drop everything recorded after it. f := &fakeWriter{} s := newOperationSink(t.Context(), f) From c9d95bd15e4fc4072720102bfbf997d4e511089a Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 19 Aug 2026 11:23:31 +0000 Subject: [PATCH 117/125] libs/dms: drop Enabled from Recording It asked a recording whether it was the disabled one, which is the branch the disabled implementation exists to avoid. Both callers can ask about data instead: Start returns no writer when nothing is recorded, so the deployment takes it unconditionally, and the stamp happens when there is a version number to stamp. The no-op writer goes with it. Also says plainly that a destroy creates a version too - it just stamps nothing, which is why only a deploy settles the deployment before the plan. Co-authored-by: Isaac --- bundle/phases/deploy.go | 4 ++-- bundle/phases/destroy.go | 2 +- bundle/phases/dms.go | 10 ---------- libs/dms/recording.go | 23 +++++++++-------------- libs/dms/recording_test.go | 5 ++--- libs/dms/writer.go | 7 ------- 6 files changed, 14 insertions(+), 37 deletions(-) diff --git a/bundle/phases/deploy.go b/bundle/phases/deploy.go index 475ae045e16..fed0de85608 100644 --- a/bundle/phases/deploy.go +++ b/bundle/phases/deploy.go @@ -285,7 +285,7 @@ func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHand logdiag.LogError(ctx, err) return } - if recording.Enabled() { + if recording.Version() != 0 { // The deployment ID is stamped earlier, when the state is opened; only the // version is new here. A first deploy has no ID until now, so stamp both. bundle.ApplySeqContext(ctx, b, @@ -363,7 +363,7 @@ func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHand logDeploymentVersion(ctx, b, recording) // Record operations under that version, so DMS holds the deployed resource state. - setOperationWriter(b, recording, writer) + b.DeploymentBundle.OpRec = writer deployCore(ctx, b, plan, stateEngine, requestedEngine) if logdiag.HasError(ctx) { diff --git a/bundle/phases/destroy.go b/bundle/phases/destroy.go index 0016d4adf8f..62fc0a7b096 100644 --- a/bundle/phases/destroy.go +++ b/bundle/phases/destroy.go @@ -282,7 +282,7 @@ func Destroy(ctx context.Context, b *bundle.Bundle, engine engine.EngineType) { logdiag.LogError(ctx, err) return } - setOperationWriter(b, recording, writer) + b.DeploymentBundle.OpRec = writer destroyCore(ctx, b, plan, engine, recording) } else { cmdio.LogString(ctx, "Destroy cancelled!") diff --git a/bundle/phases/dms.go b/bundle/phases/dms.go index b997f10fcec..cd48e54a116 100644 --- a/bundle/phases/dms.go +++ b/bundle/phases/dms.go @@ -88,16 +88,6 @@ func actionToSDK(a deployplan.ActionType) (bundledeployments.OperationActionType } } -// setOperationWriter has the state writes during apply recorded under the started version. -// A disabled recording leaves the writer unset, which is also what keeps the state DB from -// serializing an envelope for every write. -func setOperationWriter(b *bundle.Bundle, recording dms.Recording, writer dms.OperationWriter) { - if !recording.Enabled() { - return - } - b.DeploymentBundle.OpRec = writer -} - // logDeploymentVersion logs the deployment version URL. Workspace ID is omitted // so the page stays clickable in a terminal and redirects correctly without it. func logDeploymentVersion(ctx context.Context, b *bundle.Bundle, recording dms.Recording) { diff --git a/libs/dms/recording.go b/libs/dms/recording.go index cdeafd06407..5a240899b7d 100644 --- a/libs/dms/recording.go +++ b/libs/dms/recording.go @@ -30,10 +30,6 @@ const ( // stages, and their outcomes. A disabled recording is a no-op throughout, so callers do not // branch on whether recording is on. type Recording interface { - // Enabled reports whether anything is recorded. Only a caller that would otherwise do - // pointless work - serializing state on every write - needs to ask. - Enabled() bool - // Prepare settles the deployment and the version number this run will create, without // creating it. Both are needed before the plan, which the version number is stamped onto. Prepare(ctx context.Context) error @@ -46,8 +42,9 @@ type Recording interface { Version() int64 // Start creates the version, staging an operation for each resource, and returns the - // writer that fills them in. The staged set is fixed here: the service has no call to - // add one later, so a resource left out can never be recorded. + // writer that fills them in - nil when nothing is recorded, which is what leaves the state + // DB without a sink. The staged set is fixed here: the service has no call to add one + // later, so a resource left out can never be recorded. Start(ctx context.Context, staged []StagedOperation) (OperationWriter, error) // Finish completes the version. It is a no-op before Start, which is what lets a caller @@ -104,18 +101,17 @@ func Disabled() Recording { return disabled{} } -// disabled records nothing. Its Prepare leaves no deployment and no version, so DeploymentID -// and Version stay empty for a caller that stamps them onto resources. +// disabled records nothing. Its Prepare leaves no deployment and no version, and its Start no +// writer, so a caller that stamps a version or installs the writer finds nothing to install. type disabled struct{} -func (disabled) Enabled() bool { return false } func (disabled) Prepare(context.Context) error { return nil } func (disabled) DeploymentID() string { return "" } func (disabled) Version() int64 { return 0 } func (disabled) Finish(context.Context, bool) error { return nil } func (disabled) Start(context.Context, []StagedOperation) (OperationWriter, error) { - return noopWriter{}, nil + return nil, nil } // recording records with the service. @@ -142,8 +138,6 @@ type recording struct { completed bool } -func (r *recording) Enabled() bool { return true } - func (r *recording) DeploymentID() string { return r.deploymentID } func (r *recording) Version() int64 { return r.versionNum } @@ -165,8 +159,9 @@ func (r *recording) Prepare(ctx context.Context) error { // Start implements Recording. func (r *recording) Start(ctx context.Context, staged []StagedOperation) (OperationWriter, error) { - // A deploy calls Prepare first, because it needs the version number to stamp onto the - // plan. A destroy has no such need, so settle it here instead. + // A deploy calls Prepare itself, because the resources the plan is computed from are + // stamped with the version number. A destroy creates a version too, but stamps nothing, so + // it has no reason to settle the deployment any earlier than here. if r.versionNum == 0 { if err := r.Prepare(ctx); err != nil { return nil, err diff --git a/libs/dms/recording_test.go b/libs/dms/recording_test.go index 53624bce51a..5b05c36cfdb 100644 --- a/libs/dms/recording_test.go +++ b/libs/dms/recording_test.go @@ -227,11 +227,10 @@ func TestDisabledRecordingIsNoOp(t *testing.T) { require.NoError(t, err) require.NoError(t, r.Finish(t.Context(), true)) - assert.False(t, r.Enabled()) assert.Empty(t, r.DeploymentID()) assert.Zero(t, r.Version()) - // The writer it hands out records nothing, so a caller needs no nil check. - assert.NoError(t, writer.Write(t.Context(), "jobs.foo", OperationUpdate{})) + // No writer, which is what leaves the state DB without a sink and nothing to stamp. + assert.Nil(t, writer) } func TestRecordingFinishIsNoOpWithoutStart(t *testing.T) { diff --git a/libs/dms/writer.go b/libs/dms/writer.go index 668574cbd0d..6a59c64b45e 100644 --- a/libs/dms/writer.go +++ b/libs/dms/writer.go @@ -47,10 +47,3 @@ func (w *operationWriter) Write(ctx context.Context, key ResourceKey, update Ope return nil } - -// noopWriter is the writer of a disabled recording: nothing is staged, so nothing is written. -type noopWriter struct{} - -func (noopWriter) Write(context.Context, ResourceKey, OperationUpdate) error { - return nil -} From 13cb70000667f5b89e24c9bc99f08c1179be075d Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 19 Aug 2026 11:25:59 +0000 Subject: [PATCH 118/125] libs/dms: name what the version number is stamped onto Co-authored-by: Isaac --- libs/dms/recording.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/libs/dms/recording.go b/libs/dms/recording.go index 5a240899b7d..f717c7c0815 100644 --- a/libs/dms/recording.go +++ b/libs/dms/recording.go @@ -159,9 +159,9 @@ func (r *recording) Prepare(ctx context.Context) error { // Start implements Recording. func (r *recording) Start(ctx context.Context, staged []StagedOperation) (OperationWriter, error) { - // A deploy calls Prepare itself, because the resources the plan is computed from are - // stamped with the version number. A destroy creates a version too, but stamps nothing, so - // it has no reason to settle the deployment any earlier than here. + // A deploy calls Prepare itself, because the version number is stamped onto every job and + // pipeline before the plan is computed. A destroy creates a version too, but stamps + // nothing, so it has no reason to settle the deployment any earlier than here. if r.versionNum == 0 { if err := r.Prepare(ctx); err != nil { return nil, err From 615c03819d41a005960f15309078148ed92ddfd7 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 19 Aug 2026 11:28:19 +0000 Subject: [PATCH 119/125] libs/dms: keep the writer's tests about the sequence id The writer does one thing the client does not: remember where each resource is in the sequence. So its tests now cover only that - the staged id first, the service's id after, one chain per resource, and a failed write leaving the id alone - and the first case asserts the whole call, which covers the pass-through of ids, key and payload in one line. The update-mask table moves next to newUpdateRequest, which is what it tests. Co-authored-by: Isaac --- libs/dms/client_test.go | 55 +++++++++++++++++++ libs/dms/writer_test.go | 116 +++++++++------------------------------- 2 files changed, 80 insertions(+), 91 deletions(-) diff --git a/libs/dms/client_test.go b/libs/dms/client_test.go index 6070c804d64..f24db9f2197 100644 --- a/libs/dms/client_test.go +++ b/libs/dms/client_test.go @@ -2,6 +2,7 @@ package dms import ( "context" + "encoding/json" "errors" "sync" "testing" @@ -105,3 +106,57 @@ func TestDeploymentIDFromName(t *testing.T) { _, err = deploymentIDFromName("deployments/") assert.Error(t, err) } + +func TestUpdateRequestSendsAFieldOnlyWhenTheMaskNamesIt(t *testing.T) { + // Every case carries the same values, so what reaches the body is decided by the mask + // alone. A failure sending state would drop the resource from the deployment, and + // resource_id does not ride along with state. + update := OperationUpdate{ + State: json.RawMessage(`{"state":{"name":"foo"}}`), + ResourceID: "job-1", + Status: bundledeployments.OperationStatusOperationStatusSucceeded, + ErrorMessage: "boom", + } + + tests := []struct { + name string + fields Fields + want updateOperationRequest + }{ + { + name: "a write that describes the resource", + fields: DescribesResource, + want: updateOperationRequest{ + State: `{"state":{"name":"foo"}}`, + ResourceId: "job-1", + Status: bundledeployments.OperationStatusOperationStatusSucceeded, + ErrorMessage: "boom", + SequenceId: "3", + }, + }, + { + name: "a failure that keeps the recorded state", + fields: KeepsState, + want: updateOperationRequest{ + Status: bundledeployments.OperationStatusOperationStatusSucceeded, + ErrorMessage: "boom", + SequenceId: "3", + }, + }, + { + name: "resource_id without state", + fields: FieldResourceID, + want: updateOperationRequest{ + ResourceId: "job-1", + SequenceId: "3", + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + update.Fields = tt.fields + assert.Equal(t, tt.want, newUpdateRequest(update, "3")) + }) + } +} diff --git a/libs/dms/writer_test.go b/libs/dms/writer_test.go index 3e2e2c1738f..ba92475821b 100644 --- a/libs/dms/writer_test.go +++ b/libs/dms/writer_test.go @@ -4,7 +4,6 @@ import ( "encoding/json" "testing" - "github.com/databricks/databricks-sdk-go/service/bundledeployments" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -19,52 +18,43 @@ func testWriter(raw *fakeRaw) OperationWriter { } } -func writeState(t *testing.T, w OperationWriter, key ResourceKey, resourceID string, state json.RawMessage) { +func writeState(t *testing.T, w OperationWriter, key ResourceKey, resourceID string) { t.Helper() - update, err := NewStateUpdate(resourceID, state, false) + update, err := NewStateUpdate(resourceID, json.RawMessage(`{"state":{}}`), false) require.NoError(t, err) require.NoError(t, w.Write(t.Context(), key, update)) } -func TestWriterFirstWriteUpdatesTheStagedOperation(t *testing.T) { - f := newFakeRaw("1") - w := testWriter(f) - - writeState(t, w, "jobs.foo", "job-123", json.RawMessage(`{"state":{}}`)) - - require.Len(t, f.updates, 1) - c := f.updates[0] - // The version already staged this operation, so the first write updates it and echoes - // the sequence id staging left. - assert.Equal(t, "dep-1", c.deploymentID) - assert.Equal(t, int64(2), c.version) - assert.Equal(t, ResourceKey("jobs.foo"), c.key) - assert.Equal(t, stagedSequenceID, c.sequenceID) - assert.Equal(t, "job-123", c.update.ResourceID) -} - -func TestWriterSecondWriteEchoesTheServiceSequence(t *testing.T) { - // One operation per resource per version: the second write updates the same operation, - // echoing the sequence id the service returned as its precondition. +func TestWriterSendsTheStagedSequenceThenWhatTheServiceReturns(t *testing.T) { + // One operation per resource per version, so every write updates the same operation: the + // first at the sequence id staging left, each one after at the id the service returned. f := newFakeRaw("7") w := testWriter(f) - writeState(t, w, "jobs.foo", "", nil) - writeState(t, w, "jobs.foo", "job-456", json.RawMessage(`{"state":{}}`)) + update, err := NewStateUpdate("job-123", json.RawMessage(`{"state":{}}`), false) + require.NoError(t, err) + require.NoError(t, w.Write(t.Context(), "jobs.foo", update)) + writeState(t, w, "jobs.foo", "job-456") require.Len(t, f.updates, 2) - assert.Equal(t, stagedSequenceID, f.updates[0].sequenceID) + assert.Equal(t, updaterCall{ + deploymentID: "dep-1", + version: 2, + key: "jobs.foo", + sequenceID: stagedSequenceID, + update: update, + }, f.updates[0]) assert.Equal(t, "7", f.updates[1].sequenceID) } func TestWriterTracksSequencePerResource(t *testing.T) { // Each resource has its own staged operation, so each one's first write echoes the staged // sequence id rather than a sequence another resource earned. - f := newFakeRaw("1") + f := newFakeRaw("7") w := testWriter(f) - writeState(t, w, "jobs.foo", "id-1", json.RawMessage(`{"state":{}}`)) - writeState(t, w, "jobs.bar", "id-2", json.RawMessage(`{"state":{}}`)) + writeState(t, w, "jobs.foo", "id-1") + writeState(t, w, "jobs.bar", "id-2") require.Len(t, f.updates, 2) assert.Equal(t, stagedSequenceID, f.updates[0].sequenceID) @@ -72,76 +62,20 @@ func TestWriterTracksSequencePerResource(t *testing.T) { } func TestWriterErrorKeepsTheSequence(t *testing.T) { - // A failed write returns its error and leaves the recorded sequence id alone, so a later - // write for the same resource still carries the precondition the service last gave us. + // A failed write leaves the recorded sequence id alone, so the next write for that resource + // still carries the precondition the service last gave us rather than nothing. f := newFakeRaw("9") f.failOn = 1 w := testWriter(f) - writeState(t, w, "jobs.foo", "job-1", json.RawMessage(`{"state":{}}`)) + writeState(t, w, "jobs.foo", "job-1") - second, err := NewStateUpdate("job-2", json.RawMessage(`{"state":{}}`), false) + update, err := NewStateUpdate("job-2", json.RawMessage(`{"state":{}}`), false) require.NoError(t, err) - err = w.Write(t.Context(), "jobs.foo", second) - require.ErrorContains(t, err, "injected error") + require.ErrorContains(t, w.Write(t.Context(), "jobs.foo", update), "injected error") - // The third write is what proves the sequence id survived the failure. - writeState(t, w, "jobs.foo", "job-3", json.RawMessage(`{"state":{}}`)) + writeState(t, w, "jobs.foo", "job-3") require.Len(t, f.updates, 3) assert.Equal(t, "9", f.updates[2].sequenceID) } - -func TestUpdateRequestSendsAFieldOnlyWhenTheMaskNamesIt(t *testing.T) { - // Every case carries the same values, so what reaches the body is decided by the mask - // alone. A failure sending state would drop the resource from the deployment, and - // resource_id does not ride along with state. - update := OperationUpdate{ - State: json.RawMessage(`{"state":{"name":"foo"}}`), - ResourceID: "job-1", - Status: bundledeployments.OperationStatusOperationStatusSucceeded, - ErrorMessage: "boom", - } - - tests := []struct { - name string - fields Fields - want updateOperationRequest - }{ - { - name: "a write that describes the resource", - fields: DescribesResource, - want: updateOperationRequest{ - State: `{"state":{"name":"foo"}}`, - ResourceId: "job-1", - Status: bundledeployments.OperationStatusOperationStatusSucceeded, - ErrorMessage: "boom", - SequenceId: "3", - }, - }, - { - name: "a failure that keeps the recorded state", - fields: KeepsState, - want: updateOperationRequest{ - Status: bundledeployments.OperationStatusOperationStatusSucceeded, - ErrorMessage: "boom", - SequenceId: "3", - }, - }, - { - name: "resource_id without state", - fields: FieldResourceID, - want: updateOperationRequest{ - ResourceId: "job-1", - SequenceId: "3", - }, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - update.Fields = tt.fields - assert.Equal(t, tt.want, newUpdateRequest(update, "3")) - }) - } -} From 77bf1b4fefe7e2309b39ddc35a0fe4c3e25925cd Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 19 Aug 2026 11:41:34 +0000 Subject: [PATCH 120/125] libs/dms: leave the recorded wire format to the acceptance tests What a recorded deploy sends - the version it claims and the number it supersedes, the operations it stages, the metadata, the completion it reports - is asserted end to end by acceptance/bundle/dms, which prints the request bodies. Asserting it again against a fake proved nothing and pinned the wire format in two places. What only a unit test can reach is what the service refuses, so the four error translations become one table, and the destroy branch gets one: a completed destroy deletes the deployment, a failed one leaves it for the next deploy. The acceptance test for a destroy sends its requests to /dev/null and asserts the effect, so that branch had no coverage either way. 14 tests become 3, 373 lines become 190. Co-authored-by: Isaac --- libs/dms/recording_test.go | 373 +++++++++++-------------------------- 1 file changed, 111 insertions(+), 262 deletions(-) diff --git a/libs/dms/recording_test.go b/libs/dms/recording_test.go index 5b05c36cfdb..8339eda02f4 100644 --- a/libs/dms/recording_test.go +++ b/libs/dms/recording_test.go @@ -12,27 +12,17 @@ import ( "github.com/stretchr/testify/require" ) -// testStatePath is the bundle state directory the recorder registers the -// deployment node under; several tests assert it round-trips to the service. -const testStatePath = "/Workspace/Users/me/.bundle/proj/dev/state" +// What a recorded deploy puts on the wire - the version it claims, the operations it stages, +// the completion it reports - is asserted end to end by acceptance/bundle/dms. What is left +// here is what only an injected API error or a disabled recording can reach. -// testDisplayName is the bundle name the recorder sends as the version's display -// name; the service copies it onto the deployment's workspace node. -const testDisplayName = "proj" - -// fakeDMS records the calls the recorder makes and lets a test script the -// server-side responses. It embeds the SDK interface so it satisfies it while -// only overriding the methods the recorder uses. +// fakeDMS answers the generated calls a Recording makes. It embeds the SDK interface so it +// satisfies it while only overriding those. type fakeDMS struct { bundledeployments.BundleDeploymentsInterface - // scripted behavior getDeployment func(id string) (*bundledeployments.Deployment, error) - // assigned deployment ID for CreateDeployment (server-generated flow) - assignedID string - - // captured requests created []bundledeployments.CreateDeploymentRequest completed []bundledeployments.CompleteVersionRequest deleted []string @@ -43,14 +33,11 @@ type fakeDMS struct { func (f *fakeDMS) CreateDeployment(ctx context.Context, req bundledeployments.CreateDeploymentRequest) (*bundledeployments.Deployment, error) { f.created = append(f.created, req) - // The server always assigns the ID; it is the ID of the workspace node it - // creates under initial_parent_path. - return &bundledeployments.Deployment{Name: "deployments/" + f.assignedID}, nil + return &bundledeployments.Deployment{Name: "deployments/new-id"}, nil } func (f *fakeDMS) GetDeployment(ctx context.Context, req bundledeployments.GetDeploymentRequest) (*bundledeployments.Deployment, error) { - id := req.Name[len("deployments/"):] - return f.getDeployment(id) + return f.getDeployment(req.Name) } func (f *fakeDMS) CompleteVersion(ctx context.Context, req bundledeployments.CompleteVersionRequest) (*bundledeployments.Version, error) { @@ -67,156 +54,125 @@ func (f *fakeDMS) Heartbeat(ctx context.Context, req bundledeployments.Heartbeat return &bundledeployments.HeartbeatResponse{}, nil } -// testClient wires a Recording to f, failing CreateVersion with versionErr when set. -func testClient(f *fakeDMS, versionErr error) *Client { - f.raw = newFakeRaw("1") - f.raw.versionErr = versionErr - return &Client{Service: f, raw: f.raw} -} - -// startVersion creates the version and discards the writer, for a test that only asserts -// what reached the service. -func startVersion(t *testing.T, r Recording, staged []StagedOperation) { - t.Helper() - _, err := r.Start(t.Context(), staged) - require.NoError(t, err) +// deploymentAt answers GetDeployment with a deployment whose last version is lastVersion. +func deploymentAt(lastVersion string) func(string) (*bundledeployments.Deployment, error) { + return func(name string) (*bundledeployments.Deployment, error) { + return &bundledeployments.Deployment{Name: name, LastVersionId: lastVersion}, nil + } } -func TestRecordingFirstDeployCreatesDeploymentWithServerAssignedID(t *testing.T) { - f := &fakeDMS{assignedID: "server-generated-id"} - // A first deploy resolves no deployment ID from the workspace. - r := NewRecording(RecordingOptions{Client: testClient(f, nil), StatePath: testStatePath, Metadata: Metadata{TargetName: "dev", DisplayName: testDisplayName}, VersionType: VersionTypeDeploy}) - - startVersion(t, r, nil) - - // The server assigned the ID, and the recorder exposes it for the rest of the - // deploy (it parents the operations recorded under this version). - require.Len(t, f.created, 1) - assert.Equal(t, "server-generated-id", r.DeploymentID()) - // initial_parent_path is required: the service creates the deployment node - // under it, and that node is what ResolveDeploymentID looks up later. - assert.Equal(t, testStatePath, f.created[0].Deployment.InitialParentPath) - - // The first version is 1, parented under the assigned deployment. - require.Len(t, f.raw.versions, 1) - assert.Equal(t, "1", f.raw.versions[0].versionID) - assert.Equal(t, "server-generated-id", f.raw.versions[0].deploymentID) - assert.Equal(t, int64(1), r.Version()) - - // The service copies display_name onto the deployment's workspace node, which is - // where GetDeployment reads it from; a version that omits it leaves the deployment - // unnamed in the UI. - assert.Equal(t, testDisplayName, f.raw.versions[0].body.DisplayName) - // A first version supersedes nothing, so previous_version_id is unset. - assert.Empty(t, f.raw.versions[0].body.PreviousVersionId) - - require.NoError(t, r.Finish(t.Context(), true)) - require.Len(t, f.completed, 1) - assert.Equal(t, bundledeployments.VersionCompleteVersionCompleteSuccess, f.completed[0].CompletionReason) - assert.Empty(t, f.deleted) +// testRecording records the stored deployment through f, failing CreateVersion with +// versionErr when set. +func testRecording(f *fakeDMS, versionType VersionType, versionErr error) Recording { + f.raw = newFakeRaw("1") + f.raw.versionErr = versionErr + return NewRecording(RecordingOptions{ + Client: &Client{Service: f, raw: f.raw}, + DeploymentID: "stored-id", + VersionType: versionType, + }) } -func TestRecordingSubsequentDeployReusesDeploymentAndIncrementsVersion(t *testing.T) { - f := &fakeDMS{ - getDeployment: func(id string) (*bundledeployments.Deployment, error) { - return &bundledeployments.Deployment{Name: "deployments/" + id, LastVersionId: "4"}, nil +func TestRecordingStartReportsWhatTheServiceRefused(t *testing.T) { + aborted := &apierr.APIError{StatusCode: 409, ErrorCode: "ABORTED"} + exhausted := &apierr.APIError{StatusCode: 429, ErrorCode: "RESOURCE_EXHAUSTED"} + + tests := []struct { + name string + getDeployment func(string) (*bundledeployments.Deployment, error) + versionErr error + wantMessages []string + wantCause error + }{ + { + name: "the deployment cannot be read", + getDeployment: func(string) (*bundledeployments.Deployment, error) { + return nil, errors.New("boom") + }, + wantMessages: []string{"failed to get deployment"}, }, - } - // A subsequent deploy passes the stored deployment ID. - r := NewRecording(RecordingOptions{Client: testClient(f, nil), DeploymentID: "stored-id", StatePath: testStatePath, Metadata: Metadata{TargetName: "dev", DisplayName: testDisplayName}, VersionType: VersionTypeDeploy}) - - startVersion(t, r, nil) - - // No new deployment is created; the version increments to last_version_id + 1. - assert.Empty(t, f.created) - require.Len(t, f.raw.versions, 1) - assert.Equal(t, "5", f.raw.versions[0].versionID) - assert.Equal(t, "stored-id", r.DeploymentID()) - // The version it supersedes is the concurrency check; without it the service - // rejects every deploy after the first. - assert.Equal(t, "4", f.raw.versions[0].body.PreviousVersionId) -} - -func TestRecordingGetDeploymentErrorFailsDeploy(t *testing.T) { - f := &fakeDMS{ - getDeployment: func(id string) (*bundledeployments.Deployment, error) { - return nil, errors.New("boom") + { + // The service has a deployment for every BUNDLE_DEPLOYMENT node, so a not-found for + // a node get-status just returned is a broken invariant, not anything the user did. + name: "the deployment the workspace node names is gone", + getDeployment: func(string) (*bundledeployments.Deployment, error) { + return nil, fmt.Errorf("deployment: %w", apierr.ErrNotFound) + }, + wantMessages: []string{"internal error: no deployment found for the file with object id stored-id"}, }, - } - r := NewRecording(RecordingOptions{Client: testClient(f, nil), DeploymentID: "stored-id", StatePath: testStatePath, Metadata: Metadata{TargetName: "dev", DisplayName: testDisplayName}, VersionType: VersionTypeDeploy}) - - _, err := r.Start(t.Context(), nil) - assert.ErrorContains(t, err, "failed to get deployment") - assert.Empty(t, f.created) -} - -func TestRecordingMissingDeploymentIsInternalError(t *testing.T) { - // The service has a deployment for every BUNDLE_DEPLOYMENT node, so a not-found - // for a node get-status just returned is a broken invariant, not a state the - // deploy can recover from. - f := &fakeDMS{ - getDeployment: func(id string) (*bundledeployments.Deployment, error) { - return nil, fmt.Errorf("deployment: %w", apierr.ErrNotFound) + { + name: "another deploy claimed the version number", + getDeployment: deploymentAt("4"), + versionErr: aborted, + wantMessages: []string{"another deploy already claimed version 5", "try again"}, + wantCause: aborted, }, - } - r := NewRecording(RecordingOptions{Client: testClient(f, nil), DeploymentID: "stored-id", StatePath: testStatePath, Metadata: Metadata{TargetName: "dev", DisplayName: testDisplayName}, VersionType: VersionTypeDeploy}) - - _, err := r.Start(t.Context(), nil) - assert.ErrorContains(t, err, "internal error: no deployment found for the file with object id stored-id") - assert.Empty(t, f.created) - assert.Empty(t, f.raw.versions) -} - -func TestRecordingDestroyDeletesDeploymentOnSuccess(t *testing.T) { - f := &fakeDMS{ - getDeployment: func(id string) (*bundledeployments.Deployment, error) { - return &bundledeployments.Deployment{Name: "deployments/" + id, LastVersionId: "2"}, nil + { + name: "the bundle stages more operations than a version holds", + getDeployment: deploymentAt("4"), + versionErr: exhausted, + wantMessages: []string{"this bundle deploys 1 resources"}, + wantCause: exhausted, }, } - r := NewRecording(RecordingOptions{Client: testClient(f, nil), DeploymentID: "stored-id", StatePath: testStatePath, Metadata: Metadata{TargetName: "dev", DisplayName: testDisplayName}, VersionType: VersionTypeDestroy}) - startVersion(t, r, nil) - assert.Equal(t, bundledeployments.VersionTypeVersionTypeDestroy, f.raw.versions[0].body.VersionType) - - require.NoError(t, r.Finish(t.Context(), true)) - // A successful destroy deletes the deployment record. - require.Equal(t, []string{"deployments/stored-id"}, f.deleted) -} - -func TestRecordingFailedDestroyKeepsDeployment(t *testing.T) { - f := &fakeDMS{ - getDeployment: func(id string) (*bundledeployments.Deployment, error) { - return &bundledeployments.Deployment{Name: "deployments/" + id, LastVersionId: "2"}, nil - }, + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + f := &fakeDMS{getDeployment: tt.getDeployment} + r := testRecording(f, VersionTypeDeploy, tt.versionErr) + + _, err := r.Start(t.Context(), []StagedOperation{{ResourceKey: "jobs.foo"}}) + + require.Error(t, err) + for _, want := range tt.wantMessages { + assert.ErrorContains(t, err, want) + } + if tt.wantCause != nil { + // Wrapped, so a caller can still match on what the service said. + assert.ErrorIs(t, err, tt.wantCause) + } + assert.Empty(t, f.created, "the deployment already exists") + }) } - r := NewRecording(RecordingOptions{Client: testClient(f, nil), DeploymentID: "stored-id", StatePath: testStatePath, Metadata: Metadata{TargetName: "dev", DisplayName: testDisplayName}, VersionType: VersionTypeDestroy}) - - startVersion(t, r, nil) - require.NoError(t, r.Finish(t.Context(), false)) - - assert.Equal(t, bundledeployments.VersionCompleteVersionCompleteFailure, f.completed[0].CompletionReason) - // A failed destroy leaves the deployment in place. - assert.Empty(t, f.deleted) } -func TestRecordingFinishIsIdempotent(t *testing.T) { - // Destroy completes the version before deleting the remote files, because that - // deletes the deployment's node, and still defers Finish. The second - // call must not reach the server, which would fail with 404. - f := &fakeDMS{ - getDeployment: func(id string) (*bundledeployments.Deployment, error) { - return &bundledeployments.Deployment{Name: "deployments/" + id, LastVersionId: "2"}, nil - }, +func TestRecordingFinishDeletesTheDeploymentOnlyForACompletedDestroy(t *testing.T) { + // A failed destroy leaves resources behind, so its deployment has to stay for the next + // deploy to find. + tests := []struct { + name string + versionType VersionType + success bool + wantDeleted bool + }{ + {name: "a destroy that completed", versionType: VersionTypeDestroy, success: true, wantDeleted: true}, + {name: "a destroy that failed", versionType: VersionTypeDestroy, success: false}, + {name: "a deploy", versionType: VersionTypeDeploy, success: true}, } - r := NewRecording(RecordingOptions{Client: testClient(f, nil), DeploymentID: "stored-id", StatePath: testStatePath, Metadata: Metadata{TargetName: "dev", DisplayName: testDisplayName}, VersionType: VersionTypeDestroy}) - - startVersion(t, r, nil) - require.NoError(t, r.Finish(t.Context(), true)) - require.NoError(t, r.Finish(t.Context(), true)) - assert.Len(t, f.completed, 1) - // The destroy deletes the deployment record once, not once per call. - assert.Equal(t, []string{"deployments/stored-id"}, f.deleted) + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + f := &fakeDMS{getDeployment: deploymentAt("2")} + r := testRecording(f, tt.versionType, nil) + + _, err := r.Start(t.Context(), nil) + require.NoError(t, err) + require.NoError(t, r.Finish(t.Context(), tt.success)) + + wantReason := bundledeployments.VersionCompleteVersionCompleteSuccess + if !tt.success { + wantReason = bundledeployments.VersionCompleteVersionCompleteFailure + } + require.Len(t, f.completed, 1) + assert.Equal(t, wantReason, f.completed[0].CompletionReason) + + if tt.wantDeleted { + assert.Equal(t, []string{"deployments/stored-id"}, f.deleted) + } else { + assert.Empty(t, f.deleted) + } + }) + } } func TestDisabledRecordingIsNoOp(t *testing.T) { @@ -232,110 +188,3 @@ func TestDisabledRecordingIsNoOp(t *testing.T) { // No writer, which is what leaves the state DB without a sink and nothing to stamp. assert.Nil(t, writer) } - -func TestRecordingFinishIsNoOpWithoutStart(t *testing.T) { - f := &fakeDMS{} - r := NewRecording(RecordingOptions{Client: testClient(f, nil), DeploymentID: "stored-id", StatePath: testStatePath, Metadata: Metadata{TargetName: "dev", DisplayName: testDisplayName}, VersionType: VersionTypeDeploy}) - // Finish before Start is a no-op (nothing was claimed). - require.NoError(t, r.Finish(t.Context(), true)) - assert.Empty(t, f.completed) -} - -func TestRecordingPrepareClaimsNoVersion(t *testing.T) { - // A deploy the user declines prepares but never creates: the version number is - // known, so the plan can be stamped with it, but no version exists to complete and - // the number is left for the next deploy to take. - f := &fakeDMS{ - getDeployment: func(id string) (*bundledeployments.Deployment, error) { - return &bundledeployments.Deployment{Name: "deployments/" + id, LastVersionId: "4"}, nil - }, - } - r := NewRecording(RecordingOptions{Client: testClient(f, nil), DeploymentID: "stored-id", StatePath: testStatePath, Metadata: Metadata{TargetName: "dev", DisplayName: testDisplayName}, VersionType: VersionTypeDeploy}) - - require.NoError(t, r.Prepare(t.Context())) - - assert.Equal(t, int64(5), r.Version()) - assert.Empty(t, f.raw.versions, "no version created") - - require.NoError(t, r.Finish(t.Context(), true)) - assert.Empty(t, f.completed, "nothing to complete") -} - -func TestRecordingStartUsesThePreparedNumber(t *testing.T) { - f := &fakeDMS{ - getDeployment: func(id string) (*bundledeployments.Deployment, error) { - return &bundledeployments.Deployment{Name: "deployments/" + id, LastVersionId: "4"}, nil - }, - } - r := NewRecording(RecordingOptions{Client: testClient(f, nil), DeploymentID: "stored-id", StatePath: testStatePath, Metadata: Metadata{TargetName: "dev", DisplayName: testDisplayName}, VersionType: VersionTypeDeploy}) - - require.NoError(t, r.Prepare(t.Context())) - startVersion(t, r, nil) - - // The version created is the one the plan was stamped with, and it reports the - // version it supersedes so the service rejects a racing deploy. - require.Len(t, f.raw.versions, 1) - assert.Equal(t, "5", f.raw.versions[0].versionID) - assert.Equal(t, "4", f.raw.versions[0].body.PreviousVersionId) - assert.Equal(t, int64(5), r.Version()) -} - -func TestRecordingStartDetectsAbortedConflict(t *testing.T) { - f := &fakeDMS{ - getDeployment: func(id string) (*bundledeployments.Deployment, error) { - return &bundledeployments.Deployment{Name: "deployments/" + id, LastVersionId: "4"}, nil - }, - } - // Simulate concurrency conflict: another deploy claimed the version number. - conflictErr := &apierr.APIError{ - StatusCode: 409, - ErrorCode: "ABORTED", - } - r := NewRecording(RecordingOptions{ - Client: testClient(f, conflictErr), - DeploymentID: "stored-id", - StatePath: testStatePath, - Metadata: Metadata{TargetName: "dev", DisplayName: testDisplayName}, - VersionType: VersionTypeDeploy, - }) - - require.NoError(t, r.Prepare(t.Context())) - _, err := r.Start(t.Context(), nil) - - // Names the version that was taken and tells the user to retry, and keeps the - // underlying ABORTED so callers can still match on it. - assert.ErrorContains(t, err, "another deploy already claimed version 5") - assert.ErrorContains(t, err, "try again") - assert.ErrorIs(t, err, conflictErr) -} - -func TestRecordingStartStagesOperations(t *testing.T) { - // The version fixes its operation set, so what the caller passes has to reach the wire - // verbatim: the service has no API to add an operation later. - f := &fakeDMS{assignedID: "dep-1"} - r := NewRecording(RecordingOptions{Client: testClient(f, nil), StatePath: testStatePath, Metadata: Metadata{TargetName: "dev", DisplayName: testDisplayName}, VersionType: VersionTypeDeploy}) - - staged := []StagedOperation{ - {ResourceKey: "jobs.foo", ActionType: bundledeployments.OperationActionTypeOperationActionTypeCreate}, - {ResourceKey: "pipelines.bar", ActionType: bundledeployments.OperationActionTypeOperationActionTypeDelete}, - } - startVersion(t, r, staged) - - require.Len(t, f.raw.versions, 1) - assert.Equal(t, staged, f.raw.versions[0].body.Operations) -} - -func TestRecordingStartReportsTheOperationCap(t *testing.T) { - // A bundle past the service's per-version cap cannot be recorded at all, so say how many - // resources it has rather than passing the raw quota error on. - quotaErr := &apierr.APIError{StatusCode: 429, ErrorCode: "RESOURCE_EXHAUSTED"} - f := &fakeDMS{assignedID: "dep-1"} - r := NewRecording(RecordingOptions{Client: testClient(f, quotaErr), StatePath: testStatePath, Metadata: Metadata{TargetName: "dev", DisplayName: testDisplayName}, VersionType: VersionTypeDeploy}) - - _, err := r.Start(t.Context(), []StagedOperation{ - {ResourceKey: "jobs.foo", ActionType: bundledeployments.OperationActionTypeOperationActionTypeCreate}, - }) - - assert.ErrorContains(t, err, "this bundle deploys 1 resources") - assert.ErrorIs(t, err, quotaErr) -} From a8e846b1164eba680f6c6793eae0e89d3866eef6 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 19 Aug 2026 12:10:46 +0000 Subject: [PATCH 121/125] acceptance/dms: print the recorded requests in the order they happened --sort was hiding the sequence. A one-resource deploy has one possible order, and so does a deploy whose second resource waits on the first, so the golden can show it: create the version, fill in the operation, complete the version. Sorted alphabetically, the operation update came out before the version that staged it. depends-on gains the most: parent now precedes child, which is what the test is named after. multiple-resources and no-drift keep --sort, since resources with no edge between them are applied in parallel. Co-authored-by: Isaac --- .../bundle/dms/declined-deploy/output.txt | 30 +++---- acceptance/bundle/dms/declined-deploy/script | 4 +- acceptance/bundle/dms/depends-on/output.txt | 10 +-- acceptance/bundle/dms/depends-on/script | 2 +- .../bundle/dms/emptied-resource/output.txt | 80 +++++++++---------- acceptance/bundle/dms/emptied-resource/script | 4 +- acceptance/bundle/dms/provenance/output.txt | 28 +++---- acceptance/bundle/dms/provenance/script | 2 +- .../bundle/dms/record-failure/output.txt | 26 +++--- acceptance/bundle/dms/record-failure/script | 2 +- 10 files changed, 94 insertions(+), 94 deletions(-) diff --git a/acceptance/bundle/dms/declined-deploy/output.txt b/acceptance/bundle/dms/declined-deploy/output.txt index 554d4541650..0412b4b2f79 100644 --- a/acceptance/bundle/dms/declined-deploy/output.txt +++ b/acceptance/bundle/dms/declined-deploy/output.txt @@ -6,20 +6,7 @@ Created schemas.foo Files: 4 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged ->>> print_requests.py --dms //api/2.0/bundle --sort -{ - "method": "PATCH", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/schemas.foo", - "q": { - "update_mask": "state,error_message,resource_id,status" - }, - "body": { - "state": "{\"state\":{\"catalog_name\":\"main\",\"name\":\"dms_declined_deploy_schema\"}}", - "resource_id": "main.dms_declined_deploy_schema", - "status": "OPERATION_STATUS_SUCCEEDED", - "sequence_id": "0" - } -} +>>> print_requests.py --dms //api/2.0/bundle { "method": "POST", "path": "/api/2.0/bundle/deployments", @@ -51,6 +38,19 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged ] } } +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/schemas.foo", + "q": { + "update_mask": "state,error_message,resource_id,status" + }, + "body": { + "state": "{\"state\":{\"catalog_name\":\"main\",\"name\":\"dms_declined_deploy_schema\"}}", + "resource_id": "main.dms_declined_deploy_schema", + "status": "OPERATION_STATUS_SUCCEEDED", + "sequence_id": "0" + } +} { "method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/complete", @@ -74,7 +74,7 @@ To proceed, use --auto-approve after reviewing the plan above. Files: 3 uploaded, 0 deleted === Nothing was recorded for the declined deploy - no version, so none to abort ->>> print_requests.py --dms //api/2.0/bundle --sort +>>> print_requests.py --dms //api/2.0/bundle >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: diff --git a/acceptance/bundle/dms/declined-deploy/script b/acceptance/bundle/dms/declined-deploy/script index 8609f7e1203..49aa81d79f2 100644 --- a/acceptance/bundle/dms/declined-deploy/script +++ b/acceptance/bundle/dms/declined-deploy/script @@ -1,6 +1,6 @@ title "Deploy a schema, so the deployment and its first version exist" trace $CLI bundle deploy -trace print_requests.py --dms //api/2.0/bundle --sort +trace print_requests.py --dms //api/2.0/bundle title "A destructive change without --auto-approve is declined: this console cannot prompt" # Changing the catalog recreates the schema, which needs approval. @@ -10,7 +10,7 @@ trace musterr $CLI bundle deploy title "Nothing was recorded for the declined deploy - no version, so none to abort" # The version number it would have used is left for the next deploy to take, so the # history has no entry that reads like a deploy which failed or did nothing. -trace print_requests.py --dms //api/2.0/bundle --sort +trace print_requests.py --dms //api/2.0/bundle trace $CLI bundle destroy --auto-approve rm -f out.requests.txt diff --git a/acceptance/bundle/dms/depends-on/output.txt b/acceptance/bundle/dms/depends-on/output.txt index ccf622262b9..ff24ec209f0 100644 --- a/acceptance/bundle/dms/depends-on/output.txt +++ b/acceptance/bundle/dms/depends-on/output.txt @@ -7,15 +7,15 @@ Created jobs.parent Files: 4 uploaded, 0 deleted Resources: 2 created, 0 changed, 0 deleted, 0 unchanged ->>> print_requests.py --dms //versions/1/operations --sort +>>> print_requests.py --dms //versions/1/operations { "method": "PATCH", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.child", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.parent", "q": { "update_mask": "state,error_message,resource_id,status" }, "body": { - "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-depends-on/default/state/metadata.json\",\"version_id\":\"1\"},\"description\":\"depends on [NUMID]\",\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"child\",\"queue\":{\"enabled\":true}},\"depends_on\":[{\"node\":\"resources.jobs.parent\",\"label\":\"${resources.jobs.parent.id}\"}]}", + "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-depends-on/default/state/metadata.json\",\"version_id\":\"1\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"parent\",\"queue\":{\"enabled\":true}}}", "resource_id": "[NUMID]", "status": "OPERATION_STATUS_SUCCEEDED", "sequence_id": "0" @@ -23,12 +23,12 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged } { "method": "PATCH", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.parent", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.child", "q": { "update_mask": "state,error_message,resource_id,status" }, "body": { - "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-depends-on/default/state/metadata.json\",\"version_id\":\"1\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"parent\",\"queue\":{\"enabled\":true}}}", + "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-depends-on/default/state/metadata.json\",\"version_id\":\"1\"},\"description\":\"depends on [NUMID]\",\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"child\",\"queue\":{\"enabled\":true}},\"depends_on\":[{\"node\":\"resources.jobs.parent\",\"label\":\"${resources.jobs.parent.id}\"}]}", "resource_id": "[NUMID]", "status": "OPERATION_STATUS_SUCCEEDED", "sequence_id": "0" diff --git a/acceptance/bundle/dms/depends-on/script b/acceptance/bundle/dms/depends-on/script index f2bcf0cc411..a8f179cc8b6 100644 --- a/acceptance/bundle/dms/depends-on/script +++ b/acceptance/bundle/dms/depends-on/script @@ -1,6 +1,6 @@ title "Deploy a job that references another: depends_on is recorded alongside the config, since the reference is resolved to a literal in the config itself" trace $CLI bundle deploy -trace print_requests.py --dms //versions/1/operations --sort +trace print_requests.py --dms //versions/1/operations trace $CLI bundle destroy --auto-approve rm -f out.requests.txt diff --git a/acceptance/bundle/dms/emptied-resource/output.txt b/acceptance/bundle/dms/emptied-resource/output.txt index eb8e5285d48..cbbea18ae89 100644 --- a/acceptance/bundle/dms/emptied-resource/output.txt +++ b/acceptance/bundle/dms/emptied-resource/output.txt @@ -7,33 +7,7 @@ Created schemas.foo.grants Files: 4 uploaded, 0 deleted Resources: 2 created, 0 changed, 0 deleted, 0 unchanged ->>> print_requests.py --dms //api/2.0/bundle --sort -{ - "method": "PATCH", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/schemas.foo", - "q": { - "update_mask": "state,error_message,resource_id,status" - }, - "body": { - "state": "{\"state\":{\"catalog_name\":\"main\",\"name\":\"dms_emptied_resource\"}}", - "resource_id": "main.dms_emptied_resource", - "status": "OPERATION_STATUS_SUCCEEDED", - "sequence_id": "0" - } -} -{ - "method": "PATCH", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/schemas.foo.grants", - "q": { - "update_mask": "state,error_message,resource_id,status" - }, - "body": { - "state": "{\"state\":{\"securable_type\":\"schema\",\"full_name\":\"main.dms_emptied_resource\",\"__embed__\":[{\"principal\":\"someone@example.com\",\"privileges\":[\"USE_SCHEMA\"]}]},\"depends_on\":[{\"node\":\"resources.schemas.foo\",\"label\":\"${resources.schemas.foo.id}\"}]}", - "resource_id": "schema/main.dms_emptied_resource", - "status": "OPERATION_STATUS_SUCCEEDED", - "sequence_id": "0" - } -} +>>> print_requests.py --dms //api/2.0/bundle { "method": "POST", "path": "/api/2.0/bundle/deployments", @@ -69,6 +43,32 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged ] } } +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/schemas.foo", + "q": { + "update_mask": "state,error_message,resource_id,status" + }, + "body": { + "state": "{\"state\":{\"catalog_name\":\"main\",\"name\":\"dms_emptied_resource\"}}", + "resource_id": "main.dms_emptied_resource", + "status": "OPERATION_STATUS_SUCCEEDED", + "sequence_id": "0" + } +} +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/schemas.foo.grants", + "q": { + "update_mask": "state,error_message,resource_id,status" + }, + "body": { + "state": "{\"state\":{\"securable_type\":\"schema\",\"full_name\":\"main.dms_emptied_resource\",\"__embed__\":[{\"principal\":\"someone@example.com\",\"privileges\":[\"USE_SCHEMA\"]}]},\"depends_on\":[{\"node\":\"resources.schemas.foo\",\"label\":\"${resources.schemas.foo.id}\"}]}", + "resource_id": "schema/main.dms_emptied_resource", + "status": "OPERATION_STATUS_SUCCEEDED", + "sequence_id": "0" + } +} { "method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/complete", @@ -86,19 +86,7 @@ Updated schemas.foo.grants Files: 3 uploaded, 0 deleted Resources: 0 created, 1 changed, 0 deleted, 1 unchanged ->>> print_requests.py --dms //api/2.0/bundle --sort -{ - "method": "PATCH", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations/schemas.foo.grants", - "q": { - "update_mask": "state,error_message,resource_id,status" - }, - "body": { - "resource_id": "schema/main.dms_emptied_resource", - "status": "OPERATION_STATUS_SUCCEEDED", - "sequence_id": "0" - } -} +>>> print_requests.py --dms //api/2.0/bundle { "method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", @@ -123,6 +111,18 @@ Resources: 0 created, 1 changed, 0 deleted, 1 unchanged ] } } +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations/schemas.foo.grants", + "q": { + "update_mask": "state,error_message,resource_id,status" + }, + "body": { + "resource_id": "schema/main.dms_emptied_resource", + "status": "OPERATION_STATUS_SUCCEEDED", + "sequence_id": "0" + } +} { "method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/complete", diff --git a/acceptance/bundle/dms/emptied-resource/script b/acceptance/bundle/dms/emptied-resource/script index 3c7d9512ee0..9bb72aceb26 100644 --- a/acceptance/bundle/dms/emptied-resource/script +++ b/acceptance/bundle/dms/emptied-resource/script @@ -1,6 +1,6 @@ title "Deploy a schema with one grant: the grants node is recorded with its state" trace $CLI bundle deploy -trace print_requests.py --dms //api/2.0/bundle --sort +trace print_requests.py --dms //api/2.0/bundle title "Revoke the grant, so the grants node empties out" trace update_file.py databricks.yml 'grants: [{principal: someone@example.com, privileges: [USE_SCHEMA]}]' 'grants: []' @@ -8,7 +8,7 @@ trace $CLI bundle deploy # The emptied node records as a delete, not an update. The service only drops # a resource on delete; otherwise it stays listed with no state. -trace print_requests.py --dms //api/2.0/bundle --sort +trace print_requests.py --dms //api/2.0/bundle title "Plan again: reading state back from the service works and reports no work" trace $CLI bundle plan | contains.py "Plan: 0 to add, 0 to change, 0 to delete" "!unexpected end of JSON input" diff --git a/acceptance/bundle/dms/provenance/output.txt b/acceptance/bundle/dms/provenance/output.txt index 38ae16991e7..61723573fc2 100644 --- a/acceptance/bundle/dms/provenance/output.txt +++ b/acceptance/bundle/dms/provenance/output.txt @@ -6,20 +6,7 @@ Created jobs.foo Files: 5 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged ->>> print_requests.py --dms //versions --sort -{ - "method": "PATCH", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.foo", - "q": { - "update_mask": "state,error_message,resource_id,status" - }, - "body": { - "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-provenance/dev/state/metadata.json\",\"version_id\":\"1\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":4,\"name\":\"[dev [USERNAME]] foo\",\"queue\":{\"enabled\":true},\"tags\":{\"dev\":\"[USERNAME]\"}}}", - "resource_id": "[NUMID]", - "status": "OPERATION_STATUS_SUCCEEDED", - "sequence_id": "0" - } -} +>>> print_requests.py --dms //versions { "method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", @@ -49,6 +36,19 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged ] } } +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.foo", + "q": { + "update_mask": "state,error_message,resource_id,status" + }, + "body": { + "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-provenance/dev/state/metadata.json\",\"version_id\":\"1\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":4,\"name\":\"[dev [USERNAME]] foo\",\"queue\":{\"enabled\":true},\"tags\":{\"dev\":\"[USERNAME]\"}}}", + "resource_id": "[NUMID]", + "status": "OPERATION_STATUS_SUCCEEDED", + "sequence_id": "0" + } +} { "method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/complete", diff --git a/acceptance/bundle/dms/provenance/script b/acceptance/bundle/dms/provenance/script index 46eecbf26de..a1359d6e9e3 100644 --- a/acceptance/bundle/dms/provenance/script +++ b/acceptance/bundle/dms/provenance/script @@ -4,7 +4,7 @@ git remote add origin https://github.com/databricks/bundle-examples.git trace $CLI bundle deploy # The commit SHA changes every run, so assert it is a 40-char hex string and drop it. add_repl.py "$(git rev-parse HEAD)" COMMIT -trace print_requests.py --dms //versions --sort +trace print_requests.py --dms //versions title "The service denormalizes them onto the deployment, so a reader gets the provenance of the latest version" deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-provenance/dev/state/resources.deployment.json" -o json | python3 -c 'import sys,json; print(json.load(sys.stdin)["object_id"])') diff --git a/acceptance/bundle/dms/record-failure/output.txt b/acceptance/bundle/dms/record-failure/output.txt index 62ff21c81e8..efea62c8a27 100644 --- a/acceptance/bundle/dms/record-failure/output.txt +++ b/acceptance/bundle/dms/record-failure/output.txt @@ -11,19 +11,7 @@ API message: cluster spec is invalid Files: 5 uploaded, 0 deleted ->>> print_requests.py --dms //api/2.0/bundle --sort -{ - "method": "PATCH", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.doomed", - "q": { - "update_mask": "error_message,status" - }, - "body": { - "error_message": "cluster spec is invalid (400 INVALID_PARAMETER_VALUE)", - "status": "OPERATION_STATUS_FAILED", - "sequence_id": "0" - } -} +>>> print_requests.py --dms //api/2.0/bundle { "method": "POST", "path": "/api/2.0/bundle/deployments", @@ -55,6 +43,18 @@ Files: 5 uploaded, 0 deleted ] } } +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.doomed", + "q": { + "update_mask": "error_message,status" + }, + "body": { + "error_message": "cluster spec is invalid (400 INVALID_PARAMETER_VALUE)", + "status": "OPERATION_STATUS_FAILED", + "sequence_id": "0" + } +} { "method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/complete", diff --git a/acceptance/bundle/dms/record-failure/script b/acceptance/bundle/dms/record-failure/script index b8d387834b0..fbbe784b03c 100644 --- a/acceptance/bundle/dms/record-failure/script +++ b/acceptance/bundle/dms/record-failure/script @@ -1,6 +1,6 @@ title "A resource that fails to apply is recorded as a failed operation carrying the error, so the deployment history says why rather than omitting the resource" trace musterr $CLI bundle deploy -trace print_requests.py --dms //api/2.0/bundle --sort +trace print_requests.py --dms //api/2.0/bundle title "The failed resource is not listed at all: state is what projects a resource, and a failed create records none, so a later deploy plans to create it rather than treating it as already deployed" # The deployment ID is the workspace node's ID; read it back the way the CLI does. From e96181ab1820232686c36c8b44cf097e58ffe01c Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 19 Aug 2026 12:26:08 +0000 Subject: [PATCH 122/125] acceptance/dms: correct what drops an emptied resource The comment said the emptied node records as a delete and that the service only drops a resource on delete. Neither is true: the action type is staged from the plan, which says update, and what drops the resource is the update naming state with no value. Co-authored-by: Isaac --- acceptance/bundle/dms/emptied-resource/script | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/acceptance/bundle/dms/emptied-resource/script b/acceptance/bundle/dms/emptied-resource/script index 9bb72aceb26..fe3a6255e39 100644 --- a/acceptance/bundle/dms/emptied-resource/script +++ b/acceptance/bundle/dms/emptied-resource/script @@ -6,8 +6,9 @@ title "Revoke the grant, so the grants node empties out" trace update_file.py databricks.yml 'grants: [{principal: someone@example.com, privileges: [USE_SCHEMA]}]' 'grants: []' trace $CLI bundle deploy -# The emptied node records as a delete, not an update. The service only drops -# a resource on delete; otherwise it stays listed with no state. +# The action type stays update: it is staged from the plan, before apply knows the state +# comes back empty. What drops the resource from the deployment is the update naming state +# with no value. trace print_requests.py --dms //api/2.0/bundle title "Plan again: reading state back from the service works and reports no work" From 6fad8a66d3f72af4c47e5840c5e6b2ce0538e883 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 19 Aug 2026 12:34:19 +0000 Subject: [PATCH 123/125] acceptance/dms: show the error a refused operation gives a user operation-upload-fails deploys eight jobs, so which ones were reached before the background upload failed is not deterministic and its errors go to a LOG file the diff ignores. That left the message itself unasserted. One resource fixes the order, so this records what the user actually reads: which resource could not be recorded, that the deployment metadata service refused it, and the endpoint and status behind that. Co-authored-by: Isaac --- .../dms/operation-upload-message/databricks.yml | 10 ++++++++++ .../out.test.toml | 0 .../bundle/dms/operation-upload-message/output.txt | 14 ++++++++++++++ .../bundle/dms/operation-upload-message/script | 2 ++ .../bundle/dms/operation-upload-message/test.toml | 9 +++++++++ .../databricks.yml | 0 .../bundle/dms/resource-lifecycle/out.test.toml | 3 +++ .../output.txt | 0 .../{partial-update => resource-lifecycle}/script | 0 9 files changed, 38 insertions(+) create mode 100644 acceptance/bundle/dms/operation-upload-message/databricks.yml rename acceptance/bundle/dms/{partial-update => operation-upload-message}/out.test.toml (100%) create mode 100644 acceptance/bundle/dms/operation-upload-message/output.txt create mode 100644 acceptance/bundle/dms/operation-upload-message/script create mode 100644 acceptance/bundle/dms/operation-upload-message/test.toml rename acceptance/bundle/dms/{partial-update => resource-lifecycle}/databricks.yml (100%) create mode 100644 acceptance/bundle/dms/resource-lifecycle/out.test.toml rename acceptance/bundle/dms/{partial-update => resource-lifecycle}/output.txt (100%) rename acceptance/bundle/dms/{partial-update => resource-lifecycle}/script (100%) diff --git a/acceptance/bundle/dms/operation-upload-message/databricks.yml b/acceptance/bundle/dms/operation-upload-message/databricks.yml new file mode 100644 index 00000000000..e3c4d8cdd83 --- /dev/null +++ b/acceptance/bundle/dms/operation-upload-message/databricks.yml @@ -0,0 +1,10 @@ +bundle: + name: dms-operation-upload-message + +experimental: + record_deployment_history: true + +resources: + jobs: + foo: + name: foo diff --git a/acceptance/bundle/dms/partial-update/out.test.toml b/acceptance/bundle/dms/operation-upload-message/out.test.toml similarity index 100% rename from acceptance/bundle/dms/partial-update/out.test.toml rename to acceptance/bundle/dms/operation-upload-message/out.test.toml diff --git a/acceptance/bundle/dms/operation-upload-message/output.txt b/acceptance/bundle/dms/operation-upload-message/output.txt new file mode 100644 index 00000000000..ec964d501f6 --- /dev/null +++ b/acceptance/bundle/dms/operation-upload-message/output.txt @@ -0,0 +1,14 @@ + +=== What a user sees when an operation cannot be recorded +>>> errcode [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-operation-upload-message/default/files... +Error: recording operation for resources.jobs.foo with the deployment metadata service: Internal error (500 INTERNAL_ERROR) + +Endpoint: PATCH [DATABRICKS_URL]/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.foo?update_mask=state%2Cerror_message%2Cresource_id%2Cstatus +HTTP Status: 500 Internal Server Error +API error_code: INTERNAL_ERROR +API message: Internal error + +Files: 4 uploaded, 0 deleted + +Exit code: 1 diff --git a/acceptance/bundle/dms/operation-upload-message/script b/acceptance/bundle/dms/operation-upload-message/script new file mode 100644 index 00000000000..f53c0360b89 --- /dev/null +++ b/acceptance/bundle/dms/operation-upload-message/script @@ -0,0 +1,2 @@ +title "What a user sees when an operation cannot be recorded" +trace errcode $CLI bundle deploy diff --git a/acceptance/bundle/dms/operation-upload-message/test.toml b/acceptance/bundle/dms/operation-upload-message/test.toml new file mode 100644 index 00000000000..baa605dc106 --- /dev/null +++ b/acceptance/bundle/dms/operation-upload-message/test.toml @@ -0,0 +1,9 @@ +# One resource, so which operation is refused is fixed and the error a user sees can be +# asserted. operation-upload-fails covers a deploy of several resources, where which ones +# were reached before the failure landed is not. +RecordRequests = false + +[[Server]] +Pattern = "PATCH /api/2.0/bundle/deployments/{deployment_id}/versions/{version_id}/operations/{resource_key}" +Response.StatusCode = 500 +Response.Body = '''{"error_code": "INTERNAL_ERROR", "message": "Internal error"}''' diff --git a/acceptance/bundle/dms/partial-update/databricks.yml b/acceptance/bundle/dms/resource-lifecycle/databricks.yml similarity index 100% rename from acceptance/bundle/dms/partial-update/databricks.yml rename to acceptance/bundle/dms/resource-lifecycle/databricks.yml diff --git a/acceptance/bundle/dms/resource-lifecycle/out.test.toml b/acceptance/bundle/dms/resource-lifecycle/out.test.toml new file mode 100644 index 00000000000..7daaf6fd56a --- /dev/null +++ b/acceptance/bundle/dms/resource-lifecycle/out.test.toml @@ -0,0 +1,3 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/dms/partial-update/output.txt b/acceptance/bundle/dms/resource-lifecycle/output.txt similarity index 100% rename from acceptance/bundle/dms/partial-update/output.txt rename to acceptance/bundle/dms/resource-lifecycle/output.txt diff --git a/acceptance/bundle/dms/partial-update/script b/acceptance/bundle/dms/resource-lifecycle/script similarity index 100% rename from acceptance/bundle/dms/partial-update/script rename to acceptance/bundle/dms/resource-lifecycle/script From 438788afacc74e92c7ed3feb38c6e9163f8c45ee Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 19 Aug 2026 12:36:46 +0000 Subject: [PATCH 124/125] acceptance/dms: fold redeploy-after-destroy into record, and name resource-lifecycle for what it walks record already deployed, redeployed and destroyed the same bundle, and its golden shows the deployment being deleted. redeploy-after-destroy repeated all of that to reach the one thing record stopped short of: with the node gone, the next deploy has nothing to resolve, so it creates a fresh deployment at version 1. That is four lines appended to record, so the separate bundle goes. partial-update was never about a partial update. It walks one resource through create, recreate - where two state writes land on one operation, IN_PROGRESS then SUCCEEDED - and destroy, so it is resource-lifecycle now. Co-authored-by: Isaac --- acceptance/bundle/dms/record/output.txt | 69 +++++++++++++++ acceptance/bundle/dms/record/script | 6 ++ .../dms/redeploy-after-destroy/databricks.yml | 10 --- .../dms/redeploy-after-destroy/out.test.toml | 3 - .../dms/redeploy-after-destroy/output.txt | 84 ------------------- .../bundle/dms/redeploy-after-destroy/script | 11 --- .../dms/resource-lifecycle/databricks.yml | 4 +- .../bundle/dms/resource-lifecycle/output.txt | 38 ++++----- 8 files changed, 96 insertions(+), 129 deletions(-) delete mode 100644 acceptance/bundle/dms/redeploy-after-destroy/databricks.yml delete mode 100644 acceptance/bundle/dms/redeploy-after-destroy/out.test.toml delete mode 100644 acceptance/bundle/dms/redeploy-after-destroy/output.txt delete mode 100644 acceptance/bundle/dms/redeploy-after-destroy/script diff --git a/acceptance/bundle/dms/record/output.txt b/acceptance/bundle/dms/record/output.txt index 7ffe2b2913e..4f7a6f8bff7 100644 --- a/acceptance/bundle/dms/record/output.txt +++ b/acceptance/bundle/dms/record/output.txt @@ -159,3 +159,72 @@ Destroy: 1 deleted "method": "DELETE", "path": "/api/2.0/bundle/deployments/[NUMID]" } + +=== Deploy again: the destroy took the node with it, so there is nothing to resolve and a fresh deployment starts at version 1 +>>> MSYS_NO_PATHCONV=1 musterr [CLI] workspace get-status /Workspace/Users/[USERNAME]/.bundle/dms-record/default/state/resources.deployment.json +Error: Path (/Workspace/Users/[USERNAME]/.bundle/dms-record/default/state/resources.deployment.json) doesn't exist. + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-record/default/files... +Created jobs.foo +Files: 4 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +>>> MSYS_NO_PATHCONV=1 [CLI] workspace get-status /Workspace/Users/[USERNAME]/.bundle/dms-record/default/state/resources.deployment.json +{ + "object_type": "FILE", + "path": "/Workspace/Users/[USERNAME]/.bundle/dms-record/default/state/resources.deployment.json" +} + +>>> print_requests.py --dms //api/2.0/bundle +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments", + "body": { + "initial_parent_path": "/Workspace/Users/[USERNAME]/.bundle/dms-record/default/state", + "target_name": "default" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions", + "q": { + "version_id": "1" + }, + "body": { + "cli_version": "[CLI_VERSION]", + "version_type": "VERSION_TYPE_DEPLOY", + "target_name": "default", + "display_name": "dms-record", + "workspace_info": { + "file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-record/default/files", + "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-record/default" + }, + "operations": [ + { + "resource_key": "jobs.foo", + "action_type": "OPERATION_ACTION_TYPE_CREATE" + } + ] + } +} +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.foo", + "q": { + "update_mask": "state,error_message,resource_id,status" + }, + "body": { + "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-record/default/state/metadata.json\",\"version_id\":\"1\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"foo\",\"queue\":{\"enabled\":true}}}", + "resource_id": "[NUMID]", + "status": "OPERATION_STATUS_SUCCEEDED", + "sequence_id": "0" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/complete", + "body": { + "completion_reason": "VERSION_COMPLETE_SUCCESS" + } +} diff --git a/acceptance/bundle/dms/record/script b/acceptance/bundle/dms/record/script index 448454e97eb..583f777cadc 100644 --- a/acceptance/bundle/dms/record/script +++ b/acceptance/bundle/dms/record/script @@ -16,3 +16,9 @@ trace print_requests.py --dms //api/2.0/bundle title "Destroy: a destroy version and delete operation are recorded, then the deployment is deleted" trace $CLI bundle destroy --auto-approve trace print_requests.py --dms //api/2.0/bundle + +title "Deploy again: the destroy took the node with it, so there is nothing to resolve and a fresh deployment starts at version 1" +trace MSYS_NO_PATHCONV=1 musterr $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-record/default/state/resources.deployment.json" +trace $CLI bundle deploy +trace MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-record/default/state/resources.deployment.json" | jq '{object_type,path}' +trace print_requests.py --dms //api/2.0/bundle diff --git a/acceptance/bundle/dms/redeploy-after-destroy/databricks.yml b/acceptance/bundle/dms/redeploy-after-destroy/databricks.yml deleted file mode 100644 index 8f79a2c0381..00000000000 --- a/acceptance/bundle/dms/redeploy-after-destroy/databricks.yml +++ /dev/null @@ -1,10 +0,0 @@ -bundle: - name: dms-redeploy-after-destroy - -experimental: - record_deployment_history: true - -resources: - jobs: - foo: - name: foo diff --git a/acceptance/bundle/dms/redeploy-after-destroy/out.test.toml b/acceptance/bundle/dms/redeploy-after-destroy/out.test.toml deleted file mode 100644 index 7daaf6fd56a..00000000000 --- a/acceptance/bundle/dms/redeploy-after-destroy/out.test.toml +++ /dev/null @@ -1,3 +0,0 @@ -Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = [""] diff --git a/acceptance/bundle/dms/redeploy-after-destroy/output.txt b/acceptance/bundle/dms/redeploy-after-destroy/output.txt deleted file mode 100644 index 94dc3c83374..00000000000 --- a/acceptance/bundle/dms/redeploy-after-destroy/output.txt +++ /dev/null @@ -1,84 +0,0 @@ - -=== Deploy, then destroy: the deployment record and its workspace node are both deleted, so nothing points at the destroyed deployment ->>> [CLI] bundle deploy -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-redeploy-after-destroy/default/files... -Created jobs.foo -Files: 4 uploaded, 0 deleted -Resources: 1 created, 0 changed, 0 deleted, 0 unchanged - ->>> [CLI] bundle destroy --auto-approve -The following resources will be deleted: - delete resources.jobs.foo - -All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/dms-redeploy-after-destroy/default - -Destroy: 1 deleted - ->>> MSYS_NO_PATHCONV=1 musterr [CLI] workspace get-status /Workspace/Users/[USERNAME]/.bundle/dms-redeploy-after-destroy/default/state/resources.deployment.json -Error: Path (/Workspace/Users/[USERNAME]/.bundle/dms-redeploy-after-destroy/default/state/resources.deployment.json) doesn't exist. - -=== Deploy again: with no node to resolve, a fresh deployment is created at version 1, with its own workspace node ->>> [CLI] bundle deploy -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-redeploy-after-destroy/default/files... -Created jobs.foo -Files: 4 uploaded, 0 deleted -Resources: 1 created, 0 changed, 0 deleted, 0 unchanged - ->>> MSYS_NO_PATHCONV=1 [CLI] workspace get-status /Workspace/Users/[USERNAME]/.bundle/dms-redeploy-after-destroy/default/state/resources.deployment.json -{ - "object_type": "FILE", - "path": "/Workspace/Users/[USERNAME]/.bundle/dms-redeploy-after-destroy/default/state/resources.deployment.json" -} - ->>> print_requests.py --dms //api/2.0/bundle --get -{ - "method": "POST", - "path": "/api/2.0/bundle/deployments", - "body": { - "initial_parent_path": "/Workspace/Users/[USERNAME]/.bundle/dms-redeploy-after-destroy/default/state", - "target_name": "default" - } -} -{ - "method": "POST", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions", - "q": { - "version_id": "1" - }, - "body": { - "cli_version": "[CLI_VERSION]", - "version_type": "VERSION_TYPE_DEPLOY", - "target_name": "default", - "display_name": "dms-redeploy-after-destroy", - "workspace_info": { - "file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-redeploy-after-destroy/default/files", - "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-redeploy-after-destroy/default" - }, - "operations": [ - { - "resource_key": "jobs.foo", - "action_type": "OPERATION_ACTION_TYPE_CREATE" - } - ] - } -} -{ - "method": "PATCH", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.foo", - "q": { - "update_mask": "state,error_message,resource_id,status" - }, - "body": { - "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-redeploy-after-destroy/default/state/metadata.json\",\"version_id\":\"1\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"foo\",\"queue\":{\"enabled\":true}}}", - "resource_id": "[NUMID]", - "status": "OPERATION_STATUS_SUCCEEDED", - "sequence_id": "0" - } -} -{ - "method": "POST", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/complete", - "body": { - "completion_reason": "VERSION_COMPLETE_SUCCESS" - } -} diff --git a/acceptance/bundle/dms/redeploy-after-destroy/script b/acceptance/bundle/dms/redeploy-after-destroy/script deleted file mode 100644 index edd58e11507..00000000000 --- a/acceptance/bundle/dms/redeploy-after-destroy/script +++ /dev/null @@ -1,11 +0,0 @@ -title "Deploy, then destroy: the deployment record and its workspace node are both deleted, so nothing points at the destroyed deployment" -trace $CLI bundle deploy -trace $CLI bundle destroy --auto-approve -print_requests.py --dms //api/2.0/bundle --get > /dev/null - -trace MSYS_NO_PATHCONV=1 musterr $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-redeploy-after-destroy/default/state/resources.deployment.json" - -title "Deploy again: with no node to resolve, a fresh deployment is created at version 1, with its own workspace node" -trace $CLI bundle deploy -trace MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-redeploy-after-destroy/default/state/resources.deployment.json" | jq '{object_type,path}' -trace print_requests.py --dms //api/2.0/bundle --get diff --git a/acceptance/bundle/dms/resource-lifecycle/databricks.yml b/acceptance/bundle/dms/resource-lifecycle/databricks.yml index fc61c680fb1..9b948e1261b 100644 --- a/acceptance/bundle/dms/resource-lifecycle/databricks.yml +++ b/acceptance/bundle/dms/resource-lifecycle/databricks.yml @@ -1,5 +1,5 @@ bundle: - name: dms-partial-update + name: dms-resource-lifecycle experimental: record_deployment_history: true @@ -7,6 +7,6 @@ experimental: resources: schemas: foo: - name: dms_partial_update_schema + name: dms_resource_lifecycle_schema catalog_name: main comment: v1 diff --git a/acceptance/bundle/dms/resource-lifecycle/output.txt b/acceptance/bundle/dms/resource-lifecycle/output.txt index 7ac3648d4f1..7bdb9677c46 100644 --- a/acceptance/bundle/dms/resource-lifecycle/output.txt +++ b/acceptance/bundle/dms/resource-lifecycle/output.txt @@ -1,7 +1,7 @@ === Deploy: the state write records the resource >>> [CLI] bundle deploy -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-partial-update/default/files... +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-resource-lifecycle/default/files... Created schemas.foo Files: 4 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged @@ -11,7 +11,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "method": "POST", "path": "/api/2.0/bundle/deployments", "body": { - "initial_parent_path": "/Workspace/Users/[USERNAME]/.bundle/dms-partial-update/default/state", + "initial_parent_path": "/Workspace/Users/[USERNAME]/.bundle/dms-resource-lifecycle/default/state", "target_name": "default" } } @@ -25,10 +25,10 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "cli_version": "[CLI_VERSION]", "version_type": "VERSION_TYPE_DEPLOY", "target_name": "default", - "display_name": "dms-partial-update", + "display_name": "dms-resource-lifecycle", "workspace_info": { - "file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-partial-update/default/files", - "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-partial-update/default" + "file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-resource-lifecycle/default/files", + "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-resource-lifecycle/default" }, "operations": [ { @@ -45,8 +45,8 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "update_mask": "state,error_message,resource_id,status" }, "body": { - "state": "{\"state\":{\"catalog_name\":\"main\",\"comment\":\"v1\",\"name\":\"dms_partial_update_schema\"}}", - "resource_id": "main.dms_partial_update_schema", + "state": "{\"state\":{\"catalog_name\":\"main\",\"comment\":\"v1\",\"name\":\"dms_resource_lifecycle_schema\"}}", + "resource_id": "main.dms_resource_lifecycle_schema", "status": "OPERATION_STATUS_SUCCEEDED", "sequence_id": "0" } @@ -63,7 +63,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged >>> update_file.py databricks.yml catalog_name: main catalog_name: other >>> [CLI] bundle deploy --auto-approve -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-partial-update/default/files... +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-resource-lifecycle/default/files... This action will result in the deletion or recreation of the following UC schemas. Any underlying data may be lost: recreate resources.schemas.foo @@ -82,11 +82,11 @@ Resources: 1 created, 0 changed, 1 deleted, 0 unchanged "cli_version": "[CLI_VERSION]", "version_type": "VERSION_TYPE_DEPLOY", "target_name": "default", - "display_name": "dms-partial-update", + "display_name": "dms-resource-lifecycle", "previous_version_id": "1", "workspace_info": { - "file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-partial-update/default/files", - "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-partial-update/default" + "file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-resource-lifecycle/default/files", + "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-resource-lifecycle/default" }, "operations": [ { @@ -103,7 +103,7 @@ Resources: 1 created, 0 changed, 1 deleted, 0 unchanged "update_mask": "state,error_message,resource_id,status" }, "body": { - "resource_id": "main.dms_partial_update_schema", + "resource_id": "main.dms_resource_lifecycle_schema", "status": "OPERATION_STATUS_IN_PROGRESS", "sequence_id": "0" } @@ -115,8 +115,8 @@ Resources: 1 created, 0 changed, 1 deleted, 0 unchanged "update_mask": "state,error_message,resource_id,status" }, "body": { - "state": "{\"state\":{\"catalog_name\":\"other\",\"comment\":\"v1\",\"name\":\"dms_partial_update_schema\"}}", - "resource_id": "other.dms_partial_update_schema", + "state": "{\"state\":{\"catalog_name\":\"other\",\"comment\":\"v1\",\"name\":\"dms_resource_lifecycle_schema\"}}", + "resource_id": "other.dms_resource_lifecycle_schema", "status": "OPERATION_STATUS_SUCCEEDED", "sequence_id": "1" } @@ -137,7 +137,7 @@ The following resources will be deleted: This action will result in the deletion of the following UC schemas. Any underlying data may be lost: delete resources.schemas.foo -All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/dms-partial-update/default +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/dms-resource-lifecycle/default Destroy: 1 deleted @@ -152,11 +152,11 @@ Destroy: 1 deleted "cli_version": "[CLI_VERSION]", "version_type": "VERSION_TYPE_DESTROY", "target_name": "default", - "display_name": "dms-partial-update", + "display_name": "dms-resource-lifecycle", "previous_version_id": "2", "workspace_info": { - "file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-partial-update/default/files", - "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-partial-update/default" + "file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-resource-lifecycle/default/files", + "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-resource-lifecycle/default" }, "operations": [ { @@ -173,7 +173,7 @@ Destroy: 1 deleted "update_mask": "state,error_message,resource_id,status" }, "body": { - "resource_id": "other.dms_partial_update_schema", + "resource_id": "other.dms_resource_lifecycle_schema", "status": "OPERATION_STATUS_SUCCEEDED", "sequence_id": "0" } From d01927575b53ce2e33715ea14597b66672b2a2df Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 19 Aug 2026 13:16:41 +0000 Subject: [PATCH 125/125] acceptance: record the deployment-history variant for three tests from main config-remote-sync/variable_reference_parent, job_runs/on_bundle_deploy and state/newer_cli_version arrived while this branch was open, so their out.test.toml never listed the matrix key it adds. All three pass with recording on. Co-authored-by: Isaac --- .../config-remote-sync/variable_reference_parent/out.test.toml | 1 + .../bundle/resources/job_runs/on_bundle_deploy/out.test.toml | 1 + acceptance/bundle/state/newer_cli_version/out.test.toml | 1 + 3 files changed, 3 insertions(+) diff --git a/acceptance/bundle/config-remote-sync/variable_reference_parent/out.test.toml b/acceptance/bundle/config-remote-sync/variable_reference_parent/out.test.toml index a55b29471a4..287cde8ab5c 100644 --- a/acceptance/bundle/config-remote-sync/variable_reference_parent/out.test.toml +++ b/acceptance/bundle/config-remote-sync/variable_reference_parent/out.test.toml @@ -1,3 +1,4 @@ Cloud = false GOOS.windows = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] diff --git a/acceptance/bundle/resources/job_runs/on_bundle_deploy/out.test.toml b/acceptance/bundle/resources/job_runs/on_bundle_deploy/out.test.toml index 57b0f616850..71b97f1370d 100644 --- a/acceptance/bundle/resources/job_runs/on_bundle_deploy/out.test.toml +++ b/acceptance/bundle/resources/job_runs/on_bundle_deploy/out.test.toml @@ -1,3 +1,4 @@ Cloud = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"] EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/state/newer_cli_version/out.test.toml b/acceptance/bundle/state/newer_cli_version/out.test.toml index 0938e678987..2c6699da193 100644 --- a/acceptance/bundle/state/newer_cli_version/out.test.toml +++ b/acceptance/bundle/state/newer_cli_version/out.test.toml @@ -1,2 +1,3 @@ Cloud = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY = ["", "true"]