From f0673efa12866457321cee27dad46df94d75f85f Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Thu, 3 Sep 2026 20:36:52 +0000 Subject: [PATCH 01/46] bundle: record and read deployment state via DMS Record each deploy's resource state to the deployment metadata service and read it back, making the service the source of truth for a recorded bundle: the local state file becomes a tombstone and a leftover WAL is discarded. Gated on experimental.record_deployment_history (direct engine only), and supports deploy --plan. Co-authored-by: Isaac --- acceptance/bin/dms_resources.py | 105 +++ acceptance/bin/nostamp | 41 -- acceptance/bin/nostamp.py | 79 +++ acceptance/bin/print_state.py | 31 +- acceptance/bin/read_id.py | 21 +- acceptance/bin/read_state.py | 19 +- acceptance/bin/replace_ids.py | 18 +- .../whl_dynamic/out.plan_create.direct.json | 2 +- .../whl_dynamic/out.plan_update.direct.json | 2 +- .../bundle/artifacts/whl_dynamic/script | 6 +- .../artifacts/whl_implicit_custom_path/script | 2 +- .../artifacts/whl_via_environment_key/script | 2 +- .../whl_via_environment_key_extras/script | 2 +- acceptance/bundle/bundle_tag/id/script | 2 +- acceptance/bundle/bundle_tag/url/script | 2 +- .../bundle/deploy/experimental-python/script | 2 +- acceptance/bundle/deploy/immutable/script | 2 +- .../bundle/deploy/python-notebook/script | 2 +- .../readplan/basic/out.plan_create.json | 2 +- .../deploy/readplan/basic/out.plan_skip.json | 2 +- .../bundle/deploy/readplan/basic/script | 4 +- .../readplan/cli-version-mismatch/plan.json | 2 +- .../readplan/lineage-mismatch/plan.json | 2 +- .../readplan/plan-version-mismatch/output.txt | 2 +- acceptance/bundle/deploy/readplan/test.toml | 4 +- .../readplan/unknown-field/out.plan.json | 2 +- .../deploy/readplan/unknown-field/script | 2 +- .../readplan/whl-not-uploaded/out.plan.json | 2 +- .../deploy/wal/chain-3-jobs/out.test.toml | 2 +- .../bundle/deploy/wal/chain-3-jobs/script | 2 +- .../wal/crash-after-create/out.test.toml | 2 +- .../deploy/wal/crash-after-create/script | 2 +- .../bundle/deploy/wal/empty-wal/out.test.toml | 2 +- .../wal/failed-plan-no-wal/out.test.toml | 2 +- .../wal/future-serial-wal/out.test.toml | 2 +- .../deploy/wal/header-only-wal/out.test.toml | 2 +- .../deploy/wal/header-only-wal/output.txt | 2 +- .../bundle/deploy/wal/header-only-wal/script | 2 +- .../deploy/wal/lineage-mismatch/out.test.toml | 2 +- acceptance/bundle/deploy/wal/test.toml | 6 + .../deploy/wal/wal-with-delete/out.test.toml | 2 +- .../bundle/deployment/bind/experiment/script | 4 +- .../bundle/destroy/jobs-and-pipeline/script | 2 +- .../dms/declined-deploy/databricks.yml.tmpl | 10 + .../dms/declined-deploy/out.requests.txt | 7 + .../bundle/dms/declined-deploy/out.test.toml | 3 + .../bundle/dms/declined-deploy/output.txt | 88 +++ acceptance/bundle/dms/declined-deploy/script | 22 + .../bundle/dms/depends-on/databricks.yml.tmpl | 12 + .../bundle/dms/depends-on/out.test.toml | 3 + acceptance/bundle/dms/depends-on/output.txt | 47 ++ acceptance/bundle/dms/depends-on/script | 8 + .../databricks.yml.tmpl | 22 + .../deployment-metadata-change/out.test.toml | 3 + .../dms/deployment-metadata-change/output.txt | 144 ++++ .../dms/deployment-metadata-change/script | 14 + .../dms/emptied-resource/databricks.yml.tmpl | 11 + .../bundle/dms/emptied-resource/out.test.toml | 3 + .../bundle/dms/emptied-resource/output.txt | 142 ++++ acceptance/bundle/dms/emptied-resource/script | 20 + .../dms/existing-state/databricks.yml.tmpl | 7 + .../bundle/dms/existing-state/out.test.toml | 3 + .../bundle/dms/existing-state/output.txt | 78 +++ acceptance/bundle/dms/existing-state/script | 27 + .../bundle/dms/existing-state/test.toml | 3 + .../dms/failed-delete/databricks.yml.tmpl | 9 + .../bundle/dms/failed-delete/out.test.toml | 3 + .../bundle/dms/failed-delete/output.txt | 149 +++++ acceptance/bundle/dms/failed-delete/script | 16 + acceptance/bundle/dms/failed-delete/test.toml | 3 + .../dms/failed-recreate/databricks.yml.tmpl | 11 + .../dms/failed-recreate/out.requests.txt | 7 + .../bundle/dms/failed-recreate/out.test.toml | 3 + .../bundle/dms/failed-recreate/output.txt | 87 +++ acceptance/bundle/dms/failed-recreate/script | 32 + .../bundle/dms/failed-recreate/test.toml | 5 + .../databricks.yml.tmpl | 12 + .../failed-update-permissions/out.test.toml | 3 + .../dms/failed-update-permissions/output.txt | 29 + .../dms/failed-update-permissions/script | 12 + .../dms/failed-update-permissions/test.toml | 5 + .../dms/failed-update/databricks.yml.tmpl | 11 + .../bundle/dms/failed-update/out.test.toml | 3 + .../bundle/dms/failed-update/output.txt | 86 +++ acceptance/bundle/dms/failed-update/script | 28 + acceptance/bundle/dms/failed-update/test.toml | 3 + .../multiple-resources/databricks.yml.tmpl | 17 + .../dms/multiple-resources/out.test.toml | 3 + .../bundle/dms/multiple-resources/output.txt | 28 + .../bundle/dms/multiple-resources/script | 9 + .../bundle/dms/no-drift/databricks.yml.tmpl | 20 + acceptance/bundle/dms/no-drift/out.test.toml | 3 + acceptance/bundle/dms/no-drift/output.txt | 126 ++++ acceptance/bundle/dms/no-drift/script | 26 + acceptance/bundle/dms/no-drift/transform.py | 1 + .../dms/no-resources/databricks.yml.tmpl | 4 + .../bundle/dms/no-resources/out.test.toml | 3 + acceptance/bundle/dms/no-resources/output.txt | 80 +++ acceptance/bundle/dms/no-resources/script | 10 + .../databricks.yml.tmpl | 11 + .../out.requests.txt | 7 + .../operation-upload-fails-once/out.test.toml | 3 + .../operation-upload-fails-once/output.txt | 79 +++ .../dms/operation-upload-fails-once/script | 15 + .../dms/operation-upload-fails-once/test.toml | 3 + .../databricks.yml.tmpl | 23 + .../dms/operation-upload-fails/out.test.toml | 3 + .../dms/operation-upload-fails/output.txt | 4 + .../bundle/dms/operation-upload-fails/script | 9 + .../dms/operation-upload-fails/test.toml | 10 + .../databricks.yml.tmpl | 9 + .../operation-upload-message/out.test.toml | 3 + .../dms/operation-upload-message/output.txt | 16 + .../dms/operation-upload-message/script | 5 + .../dms/operation-upload-message/test.toml | 8 + .../bundle/dms/provenance/databricks.yml.tmpl | 14 + .../bundle/dms/provenance/out.test.toml | 3 + acceptance/bundle/dms/provenance/output.txt | 68 ++ acceptance/bundle/dms/provenance/script | 19 + acceptance/bundle/dms/provenance/test.toml | 4 + .../bundle/dms/readplan/databricks.yml.tmpl | 19 + acceptance/bundle/dms/readplan/out.test.toml | 3 + acceptance/bundle/dms/readplan/output.txt | 22 + acceptance/bundle/dms/readplan/script | 13 + acceptance/bundle/dms/readplan/transform.py | 1 + .../dms/record-failure/databricks.yml.tmpl | 9 + .../bundle/dms/record-failure/out.test.toml | 3 + .../bundle/dms/record-failure/output.txt | 75 +++ acceptance/bundle/dms/record-failure/script | 21 + .../bundle/dms/record-failure/test.toml | 5 + .../bundle/dms/record/databricks.yml.tmpl | 9 + acceptance/bundle/dms/record/out.test.toml | 3 + acceptance/bundle/dms/record/output.txt | 84 +++ acceptance/bundle/dms/record/script | 16 + .../dms/serialized-plan/databricks.yml.tmpl | 9 + .../bundle/dms/serialized-plan/out.test.toml | 3 + .../bundle/dms/serialized-plan/output.txt | 41 ++ acceptance/bundle/dms/serialized-plan/script | 20 + .../bundle/dms/stale-plan/databricks.yml.tmpl | 9 + .../bundle/dms/stale-plan/out.test.toml | 3 + acceptance/bundle/dms/stale-plan/output.txt | 21 + acceptance/bundle/dms/stale-plan/script | 13 + .../successful-recreate/databricks.yml.tmpl | 11 + .../dms/successful-recreate/out.requests.txt | 7 + .../dms/successful-recreate/out.test.toml | 3 + .../bundle/dms/successful-recreate/output.txt | 180 +++++ .../bundle/dms/successful-recreate/script | 20 + .../bundle/dms/summary/databricks.yml.tmpl | 9 + acceptance/bundle/dms/summary/out.test.toml | 3 + acceptance/bundle/dms/summary/output.txt | 37 ++ acceptance/bundle/dms/summary/script | 17 + acceptance/bundle/dms/test.toml | 19 + .../dms/version-never-created/databricks.yml | 10 + .../dms/version-never-created/out.test.toml | 3 + .../dms/version-never-created/output.txt | 38 ++ .../bundle/dms/version-never-created/script | 8 + .../dms/version-never-created/test.toml | 12 + acceptance/bundle/empty_string_dropped/script | 2 +- .../bundle/environments/dependencies/script | 8 +- .../bundle/escaped_refs/out.plan.direct.json | 2 +- acceptance/bundle/escaped_refs/out.test.toml | 2 +- acceptance/bundle/escaped_refs/output.txt | 2 +- acceptance/bundle/escaped_refs/script | 6 +- acceptance/bundle/escaped_refs/test.toml | 4 + .../generate/dashboard-inplace/out.test.toml | 2 +- .../generate/dashboard-inplace/test.toml | 4 + .../genie_space_inplace/out.test.toml | 2 +- .../generate/genie_space_inplace/test.toml | 3 + .../out.plan_update.direct.json | 2 +- .../bundle/migrate/auto-migrate-clean/script | 2 +- .../migrate/auto-migrate-clean/test.toml | 4 + .../migrate/auto-migrate-direct-only/script | 2 +- .../bundle/migrate/basic/out.plan_update.json | 2 +- acceptance/bundle/migrate/basic/script | 2 +- .../dashboards/out.plan_after_migrate.json | 2 +- acceptance/bundle/migrate/dashboards/script | 4 +- .../default-python/out.plan_after_deploy.json | 2 +- .../out.plan_after_migration.json | 2 +- .../bundle/migrate/default-python/script | 4 +- acceptance/bundle/migrate/runas/out.plan.json | 2 +- acceptance/bundle/migrate/runas/script | 2 +- .../resource_deps/escaped_ref/output.txt | 2 +- .../bundle/resource_deps/escaped_ref/script | 4 +- .../grant_ref/out.plan_create.direct.json | 2 +- .../bundle/resource_deps/grant_ref/script | 2 +- .../id_chain/out.plan_create.direct.json | 2 +- .../id_chain/out.plan_skip.direct.json | 2 +- .../id_chain/out.plan_update.direct.json | 2 +- .../bundle/resource_deps/id_chain/script | 6 +- .../id_star/out.plan_create.direct.json | 2 +- .../bundle/resource_deps/id_star/script | 2 +- .../job_id/out.plan_create.direct.json | 2 +- .../job_id/out.plan_delete.direct.json | 2 +- acceptance/bundle/resource_deps/job_id/script | 4 +- .../out.plan_delete.direct.json | 2 +- .../resource_deps/job_id_delete_bar/script | 2 +- .../out.plan_delete.direct.json | 2 +- .../resource_deps/job_id_delete_foo/script | 2 +- .../bundle/resource_deps/jobs_update/script | 4 +- .../out.plan_create.direct.json | 2 +- .../out.plan_update.direct.json | 2 +- .../resource_deps/jobs_update_remote/script | 4 +- .../resource_deps/missing_string_field/script | 2 +- .../out.plan_create.direct.json | 2 +- .../resource_deps/permission_ref/script | 2 +- .../out.plan_create.direct.json | 2 +- .../out.plan_noop.direct.json | 2 +- .../out.plan_update.direct.json | 2 +- .../resource_deps/pipelines_recreate/script | 10 +- .../out.plan_create.direct.json | 2 +- .../remote_pipeline/out.plan_skip.direct.json | 2 +- .../resource_deps/remote_pipeline/script | 4 +- .../resources_var/out.plan.direct.json | 2 +- .../bundle/resource_deps/resources_var/script | 2 +- .../out.plan.direct.json | 2 +- .../script | 2 +- .../tf_path_renames/out.plan.direct.json | 2 +- .../resource_deps/tf_path_renames/script | 2 +- .../out.plan.direct.json | 2 +- .../volume_path_contains_id/script | 2 +- .../volume_path_job_ref/out.plan.direct.json | 2 +- .../volume_path_job_ref/out.test.toml | 2 +- .../resource_deps/volume_path_job_ref/script | 2 +- .../volume_path_job_ref/test.toml | 4 + .../apps/config-drift-stopped/script | 4 +- .../bundle/resources/apps/config-drift/script | 6 +- .../apps/config-no-deployment/script | 2 +- .../apps/git-source-no-deployment/script | 2 +- .../apps/lifecycle-started-omitted/script | 2 +- .../resources/apps/uc-securable-drift/script | 2 +- .../catalogs/comment_out_of_band/script | 2 +- .../catalogs/drift/managed_properties/script | 2 +- .../out.plan_.direct.json | 10 +- .../deploy/update-and-resize-autoscale/script | 12 +- .../update-and-resize/out.plan_.direct.json | 8 +- .../clusters/deploy/update-and-resize/script | 10 +- .../out.plan.direct.json | 2 +- .../change-serialized-inline/out.test.toml | 2 +- .../change-serialized-inline/script | 2 +- .../change-serialized-inline/test.toml | 4 + .../out.plan.direct.json | 2 +- .../dashboards/dataset-catalog-schema/script | 2 +- .../out.plan.direct.json | 2 +- .../delete-trashed-out-of-band/script | 2 +- .../detect-change/out.plan.direct.json | 2 +- .../resources/dashboards/detect-change/script | 2 +- .../dashboards/generate_inplace/out.test.toml | 2 +- .../dashboards/generate_inplace/test.toml | 3 + .../publish-failure-stale-content/script | 4 +- .../republish-after-draft-update/script | 4 +- .../dashboards/simple/out.plan.direct.json | 2 +- .../bundle/resources/dashboards/simple/script | 2 +- .../out.plan.direct.json | 2 +- .../out.plan_initial.direct.json | 2 +- .../dashboards/unpublish-out-of-band/script | 4 +- .../genie_spaces/inline/out.plan.json | 2 +- .../resources/genie_spaces/inline/script | 2 +- .../genie_spaces/version_migration/script | 2 +- .../grants/catalogs/out.plan1.direct.json | 2 +- .../grants/catalogs/out.plan2.direct.json | 2 +- .../bundle/resources/grants/catalogs/script | 4 +- .../registered_models/out.plan1.direct.json | 2 +- .../resources/grants/registered_models/script | 2 +- .../change_privilege/out.plan1.direct.json | 2 +- .../change_privilege/out.plan2.direct.json | 2 +- .../grants/schemas/change_privilege/script | 4 +- .../duplicate_principals/out.plan.direct.json | 2 +- .../schemas/duplicate_principals/script | 2 +- .../out.plan.direct.json | 2 +- .../out.plan2.direct.json | 2 +- .../schemas/out_of_band_principal/script | 4 +- .../grants/volumes/out.plan1.direct.json | 2 +- .../grants/volumes/out.plan2.direct.json | 2 +- .../bundle/resources/grants/volumes/script | 4 +- .../resources/job_runs/failed_run/script | 2 +- .../bundle/resources/job_runs/redeploy/script | 2 +- .../bundle/resources/job_runs/wait/script | 2 +- .../jobs/big_id/out.plan.direct.json | 2 +- .../bundle/resources/jobs/big_id/script | 2 +- .../bundle/resources/jobs/big_id/test.toml | 4 +- .../jobs/delete_job/out.plan.direct.json | 2 +- .../bundle/resources/jobs/delete_job/script | 2 +- .../delete_task/out.plan_create.direct.json | 2 +- .../delete_task/out.plan_update.direct.json | 2 +- .../bundle/resources/jobs/delete_task/script | 4 +- .../resources/jobs/delete_task/test.toml | 4 +- .../jobs/on_failure_empty_slice/script | 2 +- .../out.plan_post_update.direct.json | 2 +- .../resources/jobs/remote_add_tag/script | 2 +- .../remote_delete/deploy/out.plan.direct.json | 2 +- .../jobs/remote_delete/deploy/test.toml | 4 +- .../out.plan.direct.json | 2 +- .../jobs/remote_matches_config/script | 2 +- .../resources/jobs/tags_empty_map/script | 2 +- .../jobs/update/out.plan_create.direct.json | 2 +- .../jobs/update/out.plan_skip.direct.json | 2 +- .../jobs/update/out.plan_update.direct.json | 2 +- .../bundle/resources/jobs/update/script | 6 +- .../bundle/resources/jobs/update/test.toml | 4 +- .../out.plan_create.direct.json | 2 +- .../out.plan_skip.direct.json | 2 +- .../out.plan_update.direct.json | 2 +- .../resources/jobs/update_single_node/script | 6 +- .../jobs/webhook-reorder-remote/script | 2 +- .../basic/out.first-plan.direct.txt | 2 +- .../basic/out.second-plan.direct.txt | 2 +- .../model_serving_endpoints/basic/script | 4 +- .../drift/telemetry_config_unmanaged/script | 2 +- .../drift/write_only/script | 2 +- .../catalog-name/out.first-plan.direct.json | 2 +- .../catalog-name/out.second-plan.direct.json | 2 +- .../recreate/catalog-name/script | 4 +- .../name-change/out.first-plan.direct.json | 2 +- .../name-change/out.second-plan.direct.json | 2 +- .../recreate/name-change/script | 4 +- .../out.first-plan.direct.json | 2 +- .../out.second-plan.direct.json | 2 +- .../recreate/route-optimized/script | 4 +- .../schema-name/out.first-plan.direct.json | 2 +- .../schema-name/out.second-plan.direct.json | 2 +- .../recreate/schema-name/script | 4 +- .../table-prefix/out.first-plan.direct.json | 2 +- .../table-prefix/out.second-plan.direct.json | 2 +- .../recreate/table-prefix/script | 4 +- .../running-endpoint/out.plan.direct.txt | 2 +- .../running-endpoint/script | 2 +- .../update/ai-gateway/out.plan.direct.json | 2 +- .../update/ai-gateway/script | 2 +- .../out.plan.direct.json | 2 +- .../update/both_gateway_and_tags/script | 2 +- .../update/config/out.plan.direct.json | 2 +- .../update/config/script | 2 +- .../email-notifications/out.plan.direct.json | 2 +- .../update/email-notifications/script | 2 +- .../update/tags/out.plan.direct.json | 2 +- .../update/tags/script | 2 +- .../bundle/resources/permissions/_script | 4 +- .../current_can_manage/out.plan.direct.json | 2 +- .../other_can_manage/out.plan.direct.json | 2 +- .../current_can_manage/out.plan.direct.json | 2 +- .../dashboards/create/out.plan.direct.json | 2 +- .../permissions/dashboards/create/script | 2 +- .../current_can_manage/out.plan.direct.json | 2 +- .../current_can_manage/out.plan.direct.json | 2 +- .../current_can_manage/out.plan.direct.json | 2 +- .../genie_spaces/current_can_manage/script | 2 +- .../jobs/added_remotely/out.plan.direct.json | 2 +- .../permissions/jobs/added_remotely/script | 2 +- .../current_can_manage/out.plan.direct.json | 2 +- .../out.plan.direct.txt | 2 +- .../jobs/current_can_manage_run/script | 4 +- .../current_is_owner/out.plan.direct.json | 2 +- .../delete_one/out.plan_create.direct.json | 2 +- .../permissions/jobs/delete_one/script | 2 +- .../out.plan_create.direct.json | 2 +- .../out.plan_restore.direct.json | 2 +- .../permissions/jobs/deleted_remotely/script | 4 +- .../jobs/destroy_without_mgmtperms/test.toml | 5 +- .../jobs/empty_list/out.plan.direct.json | 2 +- .../other_can_manage/out.plan.direct.json | 2 +- .../jobs/other_is_owner/out.plan.direct.json | 2 +- .../jobs/update/out.plan_create.direct.json | 2 +- .../update/out.plan_delete_all.direct.json | 2 +- .../update/out.plan_delete_one.direct.json | 2 +- .../update/out.plan_post_create.direct.json | 2 +- .../jobs/update/out.plan_restore.direct.json | 2 +- .../update/out.plan_set_empty.direct.json | 2 +- .../jobs/update/out.plan_update.direct.json | 2 +- .../resources/permissions/jobs/update/script | 14 +- .../jobs/viewers/out.plan.direct.json | 2 +- .../current_can_manage/out.plan.direct.json | 2 +- .../models/current_can_manage/script | 2 +- .../current_can_manage/out.plan.direct.json | 2 +- .../current_is_owner/out.plan.direct.json | 2 +- .../pipelines/empty_list/out.plan.direct.json | 2 +- .../other_can_manage/out.plan.direct.json | 2 +- .../other_is_owner/out.plan.direct.json | 2 +- .../update/out.plan_create.direct.json | 2 +- .../update/out.plan_delete_all.direct.json | 2 +- .../update/out.plan_delete_one.direct.json | 2 +- .../update/out.plan_restore.direct.json | 2 +- .../update/out.plan_update.direct.json | 2 +- .../permissions/pipelines/update/script | 10 +- .../current_can_manage/out.plan.direct.json | 2 +- .../current_can_manage/out.plan.direct.json | 2 +- .../target_permissions/out.plan.direct.json | 2 +- .../permissions/target_permissions/script | 2 +- .../current_can_manage/out.plan.direct.json | 2 +- .../resources/pipelines/auto-approve/script | 8 +- .../pipelines/lakeflow-pipeline/script | 2 +- .../pipelines/num-workers-zero/script | 2 +- .../resources/pipelines/recreate-keys/_script | 8 +- .../out.plan_create.direct.json | 2 +- .../out.plan_recreate.direct.json | 2 +- .../out.plan_create.direct.json | 2 +- .../out.plan_recreate.direct.json | 2 +- .../resources/pipelines/recreate/script | 4 +- .../out.plan.direct.json | 2 +- .../pipelines/remote_matches_config/script | 2 +- .../bundle/resources/pipelines/update/script | 2 +- .../purge_on_delete_transitions/script | 2 +- .../recreate_source_branch_time/script | 2 +- .../update_expire_time/script | 2 +- .../postgres_branches/update_protected/script | 8 +- .../postgres_branches/update_ttl/script | 2 +- .../postgres_databases/update/script | 8 +- .../remove_suspend_timeout/script | 4 +- .../update_autoscaling/script | 8 +- .../update_pg_settings/script | 2 +- .../update_suspend_timeout/script | 2 +- .../purge_on_delete_transitions/script | 2 +- .../remove_history_retention/script | 4 +- .../update_default_endpoint_suspend/script | 2 +- .../update_display_name/script | 8 +- .../update_history_retention/script | 2 +- .../inherited-role-bind/out.test.toml | 2 +- .../inherited-role-bind/test.toml | 4 + .../resources/postgres_roles/update/script | 8 +- .../change_assets_dir/out.plan.direct.json | 2 +- .../change_assets_dir/out.test.toml | 2 +- .../change_assets_dir/output.txt | 2 +- .../change_assets_dir/test.toml | 3 + .../out.plan.direct.json | 2 +- .../change_output_schema_name/script | 2 +- .../change_table_name/out.plan.direct.json | 2 +- .../change_table_name/out.test.toml | 2 +- .../change_table_name/test.toml | 4 + .../create/out.plan_create.direct.json | 2 +- .../create/out.plan_noop.direct.json | 2 +- .../resources/quality_monitors/create/script | 4 +- .../registered_models/aliases_converge/script | 2 +- .../resources/registered_models/basic/script | 2 +- .../drift/browse_only/script | 2 +- .../resources/schemas/auto-approve/script | 4 +- .../schemas/comment_out_of_band/script | 2 +- .../schemas/drift/managed_properties/script | 2 +- .../secret_scopes/basic/out.plan1.direct.txt | 2 +- .../secret_scopes/basic/out.plan2.direct.txt | 2 +- .../basic/out.plan_verify_no_drift.direct.txt | 2 +- .../resources/secret_scopes/basic/script | 6 +- .../secrets/remove-expire-time/script | 4 +- .../secrets/update-expire-time/script | 2 +- .../resources/secrets/update-value/output.txt | 26 +- .../resources/secrets/update-value/script | 6 +- .../drift/target_qps/script | 2 +- .../recreate/with_endpoint/output.txt | 10 +- .../recreate/with_endpoint/script | 2 +- .../volumes/catalog-var-ref/output.txt | 2 +- .../resources/volumes/catalog-var-ref/script | 2 +- .../volumes/change-name/out.plan.direct.json | 2 +- .../resources/volumes/change-name/script | 2 +- .../volumes/comment_out_of_band/script | 2 +- .../out.plan_create.direct.json | 2 +- .../volumes/remote-change-name/script | 2 +- .../resources/volumes/uppercase-name/script | 2 +- acceptance/bundle/run_as/pipelines/_script | 4 +- ....plan_t_service_principal_name.direct.json | 2 +- ...rvice_principal_name_different.direct.json | 2 +- .../out.plan_t_user_name.direct.json | 2 +- ...out.plan_t_user_name_different.direct.json | 2 +- ....plan_t_service_principal_name.direct.json | 2 +- ...rvice_principal_name_different.direct.json | 2 +- .../out.plan_t_user_name.direct.json | 2 +- ...out.plan_t_user_name_different.direct.json | 2 +- acceptance/bundle/select/basic/out.test.toml | 2 +- acceptance/bundle/select/basic/script | 4 +- acceptance/bundle/select/basic/test.toml | 4 + .../select/grants_permissions/out.test.toml | 2 +- .../select/grants_permissions/test.toml | 4 + .../bundle/state/feature_flags/out.test.toml | 2 +- .../bundle/state/feature_flags/output.txt | 8 +- .../resources.empty_features.json | 2 +- .../feature_flags/resources.with_feature.json | 2 +- acceptance/bundle/state/feature_flags/script | 4 +- .../bundle/state/feature_flags/test.toml | 3 + .../bundle/state/future_version/output.txt | 2 +- .../out.plan.direct.json | 2 +- .../state/permission_level_migration/script | 4 +- .../bundle/summary/modified_status/script | 8 +- .../out.plan_after_deploy_dev.direct.json | 2 +- .../out.plan_after_deploy_prod.direct.json | 2 +- .../classic/out.plan_dev.direct.json | 2 +- .../classic/out.plan_prod.direct.json | 2 +- .../templates/default-python/classic/script | 8 +- .../out.plan_dev.direct.json | 2 +- .../out.plan_prod.direct.json | 2 +- .../default-python/integration_classic/script | 4 +- .../out.plan_after_deploy_dev.direct.json | 2 +- .../out.plan_after_deploy_prod.direct.json | 2 +- .../serverless/out.plan_dev.direct.json | 2 +- .../serverless/out.plan_prod.direct.json | 2 +- .../default-python/serverless/script | 8 +- .../record-deployment-history/out.test.toml | 2 +- .../record-deployment-history/test.toml | 4 + acceptance/bundle/test.toml | 8 - bundle/bundle.go | 8 + bundle/config/deployment.go | 16 + bundle/config/experimental.go | 3 + bundle/configsync/diff.go | 2 +- bundle/configsync/variables.go | 2 +- .../metadata/annotate_deployment_version.go | 65 ++ .../annotate_deployment_version_test.go | 50 ++ bundle/deployplan/plan.go | 21 +- bundle/direct/apply.go | 28 +- bundle/direct/bind.go | 22 +- bundle/direct/bundle_apply.go | 28 +- bundle/direct/bundle_plan.go | 59 +- bundle/direct/dstate/dms.go | 58 ++ bundle/direct/dstate/dms_test.go | 100 +++ bundle/direct/dstate/migrate.go | 20 - bundle/direct/dstate/state.go | 321 +++++++-- bundle/direct/dstate/state_test.go | 80 +-- bundle/direct/pkg.go | 8 + bundle/env/dms.go | 22 + bundle/env/dms_test.go | 41 ++ bundle/migrate/build_state.go | 2 +- bundle/phases/bind.go | 16 +- bundle/phases/deploy.go | 78 ++- bundle/phases/destroy.go | 48 +- bundle/phases/dms.go | 249 +++++++ bundle/phases/dms_test.go | 81 +++ bundle/statemgmt/direct_migration.go | 3 +- cmd/bundle/generate/dashboard.go | 2 +- cmd/bundle/generate/genie_space.go | 2 +- cmd/bundle/utils/process.go | 116 +++- libs/dms/client.go | 250 +++++++ libs/dms/client_test.go | 67 ++ libs/dms/fields.go | 59 ++ libs/dms/fields_test.go | 23 + libs/dms/metadata.go | 95 +++ libs/dms/metadata_test.go | 25 + libs/dms/operation.go | 139 ++++ libs/dms/operation_buffer.go | 197 ++++++ libs/dms/operation_buffer_test.go | 116 ++++ libs/dms/operation_test.go | 83 +++ libs/dms/resources.go | 41 ++ libs/testserver/bundledeployments.go | 615 ++++++++++++++++++ libs/testserver/fake_workspace.go | 11 + libs/testserver/handlers.go | 29 + libs/workspaceurls/urls.go | 17 + libs/workspaceurls/urls_test.go | 49 ++ 541 files changed, 6619 insertions(+), 748 deletions(-) create mode 100755 acceptance/bin/dms_resources.py delete mode 100755 acceptance/bin/nostamp create mode 100755 acceptance/bin/nostamp.py create mode 100644 acceptance/bundle/dms/declined-deploy/databricks.yml.tmpl create mode 100644 acceptance/bundle/dms/declined-deploy/out.requests.txt create mode 100644 acceptance/bundle/dms/declined-deploy/out.test.toml create mode 100644 acceptance/bundle/dms/declined-deploy/output.txt create mode 100644 acceptance/bundle/dms/declined-deploy/script create mode 100644 acceptance/bundle/dms/depends-on/databricks.yml.tmpl create mode 100644 acceptance/bundle/dms/depends-on/out.test.toml create mode 100644 acceptance/bundle/dms/depends-on/output.txt create mode 100644 acceptance/bundle/dms/depends-on/script create mode 100644 acceptance/bundle/dms/deployment-metadata-change/databricks.yml.tmpl create mode 100644 acceptance/bundle/dms/deployment-metadata-change/out.test.toml create mode 100644 acceptance/bundle/dms/deployment-metadata-change/output.txt create mode 100644 acceptance/bundle/dms/deployment-metadata-change/script create mode 100644 acceptance/bundle/dms/emptied-resource/databricks.yml.tmpl create mode 100644 acceptance/bundle/dms/emptied-resource/out.test.toml create mode 100644 acceptance/bundle/dms/emptied-resource/output.txt create mode 100644 acceptance/bundle/dms/emptied-resource/script create mode 100644 acceptance/bundle/dms/existing-state/databricks.yml.tmpl create mode 100644 acceptance/bundle/dms/existing-state/out.test.toml create mode 100644 acceptance/bundle/dms/existing-state/output.txt create mode 100644 acceptance/bundle/dms/existing-state/script create mode 100644 acceptance/bundle/dms/existing-state/test.toml create mode 100644 acceptance/bundle/dms/failed-delete/databricks.yml.tmpl create mode 100644 acceptance/bundle/dms/failed-delete/out.test.toml create mode 100644 acceptance/bundle/dms/failed-delete/output.txt create mode 100644 acceptance/bundle/dms/failed-delete/script create mode 100644 acceptance/bundle/dms/failed-delete/test.toml create mode 100644 acceptance/bundle/dms/failed-recreate/databricks.yml.tmpl create mode 100644 acceptance/bundle/dms/failed-recreate/out.requests.txt create mode 100644 acceptance/bundle/dms/failed-recreate/out.test.toml create mode 100644 acceptance/bundle/dms/failed-recreate/output.txt create mode 100644 acceptance/bundle/dms/failed-recreate/script create mode 100644 acceptance/bundle/dms/failed-recreate/test.toml create mode 100644 acceptance/bundle/dms/failed-update-permissions/databricks.yml.tmpl create mode 100644 acceptance/bundle/dms/failed-update-permissions/out.test.toml create mode 100644 acceptance/bundle/dms/failed-update-permissions/output.txt create mode 100644 acceptance/bundle/dms/failed-update-permissions/script create mode 100644 acceptance/bundle/dms/failed-update-permissions/test.toml create mode 100644 acceptance/bundle/dms/failed-update/databricks.yml.tmpl create mode 100644 acceptance/bundle/dms/failed-update/out.test.toml create mode 100644 acceptance/bundle/dms/failed-update/output.txt create mode 100644 acceptance/bundle/dms/failed-update/script create mode 100644 acceptance/bundle/dms/failed-update/test.toml create mode 100644 acceptance/bundle/dms/multiple-resources/databricks.yml.tmpl create mode 100644 acceptance/bundle/dms/multiple-resources/out.test.toml create mode 100644 acceptance/bundle/dms/multiple-resources/output.txt create mode 100644 acceptance/bundle/dms/multiple-resources/script create mode 100644 acceptance/bundle/dms/no-drift/databricks.yml.tmpl create mode 100644 acceptance/bundle/dms/no-drift/out.test.toml create mode 100644 acceptance/bundle/dms/no-drift/output.txt create mode 100644 acceptance/bundle/dms/no-drift/script create mode 100644 acceptance/bundle/dms/no-drift/transform.py create mode 100644 acceptance/bundle/dms/no-resources/databricks.yml.tmpl create mode 100644 acceptance/bundle/dms/no-resources/out.test.toml create mode 100644 acceptance/bundle/dms/no-resources/output.txt create mode 100644 acceptance/bundle/dms/no-resources/script create mode 100644 acceptance/bundle/dms/operation-upload-fails-once/databricks.yml.tmpl create mode 100644 acceptance/bundle/dms/operation-upload-fails-once/out.requests.txt create mode 100644 acceptance/bundle/dms/operation-upload-fails-once/out.test.toml create mode 100644 acceptance/bundle/dms/operation-upload-fails-once/output.txt create mode 100644 acceptance/bundle/dms/operation-upload-fails-once/script create mode 100644 acceptance/bundle/dms/operation-upload-fails-once/test.toml create mode 100644 acceptance/bundle/dms/operation-upload-fails/databricks.yml.tmpl create mode 100644 acceptance/bundle/dms/operation-upload-fails/out.test.toml create mode 100644 acceptance/bundle/dms/operation-upload-fails/output.txt create mode 100644 acceptance/bundle/dms/operation-upload-fails/script create mode 100644 acceptance/bundle/dms/operation-upload-fails/test.toml create mode 100644 acceptance/bundle/dms/operation-upload-message/databricks.yml.tmpl create mode 100644 acceptance/bundle/dms/operation-upload-message/out.test.toml create mode 100644 acceptance/bundle/dms/operation-upload-message/output.txt create mode 100644 acceptance/bundle/dms/operation-upload-message/script create mode 100644 acceptance/bundle/dms/operation-upload-message/test.toml create mode 100644 acceptance/bundle/dms/provenance/databricks.yml.tmpl create mode 100644 acceptance/bundle/dms/provenance/out.test.toml create mode 100644 acceptance/bundle/dms/provenance/output.txt create mode 100644 acceptance/bundle/dms/provenance/script create mode 100644 acceptance/bundle/dms/provenance/test.toml create mode 100644 acceptance/bundle/dms/readplan/databricks.yml.tmpl create mode 100644 acceptance/bundle/dms/readplan/out.test.toml create mode 100644 acceptance/bundle/dms/readplan/output.txt create mode 100644 acceptance/bundle/dms/readplan/script create mode 100644 acceptance/bundle/dms/readplan/transform.py create mode 100644 acceptance/bundle/dms/record-failure/databricks.yml.tmpl create mode 100644 acceptance/bundle/dms/record-failure/out.test.toml create mode 100644 acceptance/bundle/dms/record-failure/output.txt create mode 100644 acceptance/bundle/dms/record-failure/script create mode 100644 acceptance/bundle/dms/record-failure/test.toml create mode 100644 acceptance/bundle/dms/record/databricks.yml.tmpl create mode 100644 acceptance/bundle/dms/record/out.test.toml create mode 100644 acceptance/bundle/dms/record/output.txt create mode 100644 acceptance/bundle/dms/record/script create mode 100644 acceptance/bundle/dms/serialized-plan/databricks.yml.tmpl create mode 100644 acceptance/bundle/dms/serialized-plan/out.test.toml create mode 100644 acceptance/bundle/dms/serialized-plan/output.txt create mode 100644 acceptance/bundle/dms/serialized-plan/script create mode 100644 acceptance/bundle/dms/stale-plan/databricks.yml.tmpl create mode 100644 acceptance/bundle/dms/stale-plan/out.test.toml create mode 100644 acceptance/bundle/dms/stale-plan/output.txt create mode 100644 acceptance/bundle/dms/stale-plan/script create mode 100644 acceptance/bundle/dms/successful-recreate/databricks.yml.tmpl create mode 100644 acceptance/bundle/dms/successful-recreate/out.requests.txt create mode 100644 acceptance/bundle/dms/successful-recreate/out.test.toml create mode 100644 acceptance/bundle/dms/successful-recreate/output.txt create mode 100644 acceptance/bundle/dms/successful-recreate/script create mode 100644 acceptance/bundle/dms/summary/databricks.yml.tmpl create mode 100644 acceptance/bundle/dms/summary/out.test.toml create mode 100644 acceptance/bundle/dms/summary/output.txt create mode 100644 acceptance/bundle/dms/summary/script create mode 100644 acceptance/bundle/dms/test.toml create mode 100644 acceptance/bundle/dms/version-never-created/databricks.yml create mode 100644 acceptance/bundle/dms/version-never-created/out.test.toml create mode 100644 acceptance/bundle/dms/version-never-created/output.txt create mode 100644 acceptance/bundle/dms/version-never-created/script create mode 100644 acceptance/bundle/dms/version-never-created/test.toml create mode 100644 acceptance/bundle/resources/quality_monitors/change_assets_dir/test.toml create mode 100644 bundle/deploy/metadata/annotate_deployment_version.go create mode 100644 bundle/deploy/metadata/annotate_deployment_version_test.go create mode 100644 bundle/direct/dstate/dms.go create mode 100644 bundle/direct/dstate/dms_test.go create mode 100644 bundle/env/dms.go create mode 100644 bundle/env/dms_test.go create mode 100644 bundle/phases/dms.go create mode 100644 bundle/phases/dms_test.go create mode 100644 libs/dms/client.go create mode 100644 libs/dms/client_test.go create mode 100644 libs/dms/fields.go create mode 100644 libs/dms/fields_test.go create mode 100644 libs/dms/metadata.go create mode 100644 libs/dms/metadata_test.go create mode 100644 libs/dms/operation.go create mode 100644 libs/dms/operation_buffer.go create mode 100644 libs/dms/operation_buffer_test.go create mode 100644 libs/dms/operation_test.go create mode 100644 libs/dms/resources.go create mode 100644 libs/testserver/bundledeployments.go diff --git a/acceptance/bin/dms_resources.py b/acceptance/bin/dms_resources.py new file mode 100755 index 00000000000..25e469c5bad --- /dev/null +++ b/acceptance/bin/dms_resources.py @@ -0,0 +1,105 @@ +#!/usr/bin/env python3 +""" +Read resource ids and state from the deployment metadata service. + +While a bundle records deployment history the service owns the resource set, so the state file is +not where ids and state come from. The service is asked instead, which takes two lookups: the CLI +keeps no deployment id locally, and the id is the object id of the workspace node the service +registers under /resources.deployment.json (see libs/dms/resolve.go). +""" + +import functools +import glob +import json +import os +import posixpath +import subprocess +import sys + +sys.path.insert(0, os.path.dirname(__file__)) +from print_state import get_state_file + +CLI = os.environ.get("CLI", "databricks") + +# Must match dms.DeploymentNodeName. +DEPLOYMENT_NODE_NAME = "resources.deployment.json" + + +def run_json(cmd, allow_failure=False): + """Run cmd and parse its stdout, or return None if it fails and allow_failure is set. stderr is + captured rather than inherited: these lookups are plumbing, and a CLI warning like "no files to + sync" would otherwise land in the test output.""" + result = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, encoding="utf-8") + if result.returncode != 0: + if allow_failure: + return None + raise SystemExit(f"{cmd} failed with code {result.returncode}\n{result.stdout}{result.stderr}".strip()) + return json.loads(result.stdout) + + +def records_deployment_history(): + """Whether this run records deployment history, so the service is what to ask.""" + return os.environ.get("DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY") == "true" + + +def get_remote_state_path(target): + """The bundle's remote state directory. + + Preferred source is the sync snapshot, because it needs no CLI call: re-running the config + load would need whatever --var and flags the test deployed with, which a helper cannot know. + A bundle with no files to sync writes no snapshot, so fall back to asking the CLI - those + bundles are the ones with nothing to parameterize.""" + target_dir = os.path.dirname(get_state_file(target, False)) + snapshots = glob.glob(f"{target_dir}/sync-snapshots/*.json") + if snapshots: + # One snapshot per remote path, so a test that moved its root leaves several: the newest + # is the one the last deploy used. + newest = max(snapshots, key=os.path.getmtime) + remote_path = json.loads(open(newest).read())["remote_path"] + # state and files are siblings under the bundle root. + return posixpath.join(posixpath.dirname(remote_path), "state") + + args = [CLI, "bundle", "validate", "--output", "json"] + if target: + args += ["-t", target] + return run_json(args)["workspace"]["state_path"] + + +@functools.cache +def get_resources(target): + """Map every recorded resource key ("jobs.foo") to its {"id", "state"}. + + Empty when the bundle has no deployment recorded yet. Cached because a lookup costs three + round trips and a script asks for one resource at a time. + """ + state_path = get_remote_state_path(target) + if not state_path: + return {} + + # No node means nothing has been recorded, the conclusion dms.resolveDeploymentID also draws + # from a 404 - the deployment is gone once the bundle is destroyed. + node = run_json([CLI, "workspace", "get-status", f"{state_path}/{DEPLOYMENT_NODE_NAME}"], allow_failure=True) + if not node or not node.get("object_id"): + return {} + deployment_id = node["object_id"] + + result = {} + # The service pages at 50 resources; the local fake returns everything at once. + page_token = None + while True: + url = f"/api/2.0/bundle/deployments/{deployment_id}/resources" + if page_token: + url += f"?page_token={page_token}" + listed = run_json([CLI, "api", "get", url]) + for resource in listed.get("resources") or []: + # The service stores state as the opaque envelope the CLI wrote (dstate.RecordedState), + # so unwrap it to the resource state itself. + envelope = json.loads(resource["state"]) if resource.get("state") else {} + result[resource["resource_key"]] = { + "id": resource.get("resource_id"), + "state": envelope.get("state") or {}, + "depends_on": envelope.get("depends_on") or [], + } + page_token = listed.get("next_page_token") + if not page_token: + return result diff --git a/acceptance/bin/nostamp b/acceptance/bin/nostamp deleted file mode 100755 index 1827400bece..00000000000 --- a/acceptance/bin/nostamp +++ /dev/null @@ -1,41 +0,0 @@ -#!/usr/bin/env bash - -# Reads JSON on stdin, writes it back with the deployment stamp removed. -# -# Deployment history recording adds deployment_id and version_id to every job and -# pipeline. Acceptance tests compare output byte for byte, so those two extra fields -# would fail every test in the DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY=true run -# (see bundle/test.toml). -# Pipe a plan, a state dump, or a resource payload through this and the test asserts one -# golden file either way. Tests under bundle/dms assert the stamp itself and must not. -# -# Three passes, because the stamp shows up in three shapes: -# -# 1. Nested in a deployment block, as printed by `jobs get` / `pipelines get`: -# "deployment": {"deployment_id": "87..", "kind": "BUNDLE", -# "metadata_file_path": "/x", "version_id": "1"} -# -> both keys dropped, "kind" and "metadata_file_path" kept. -# -# 2. Flat in a plan's "changes", keyed by field path: -# "changes": {"deployment.version_id": {"action": "skip", ...}, "name": {...}} -# -> the stamp entries dropped, real changes kept. -# -# 3. A "changes" object that pass 2 emptied to {} - it existed only because of -# recording, so the key goes too. -# -# Two things it deliberately keeps: -# -# - A deployment_id anywhere else. Pass 1 requires "kind" and "metadata_file_path" as -# neighbours, a pair unique to the deployment block, so an unrelated field of the -# same name is untouched. -# - "version_id": "". A terraform state dump carries that for a job it never stamped, -# and it is the test's own expected output, so `.value == ""` keeps it. -# -# Arguments are passed to jq, for callers whose input is not formatted the way jq -# formats by default: a state dump is printed verbatim from disk, so it needs -# --indent 1 to come back out unchanged. -jq "$@" '((.. | objects | select(has("kind") and has("metadata_file_path"))) - |= with_entries(select((.key | IN("deployment_id", "version_id")) == false or .value == ""))) - | ((.. | objects | .changes? | objects) - |= with_entries(select(.key | IN("deployment.deployment_id", "deployment.version_id") | not))) - | del(.. | objects | select(.changes == {}) | .changes)' diff --git a/acceptance/bin/nostamp.py b/acceptance/bin/nostamp.py new file mode 100755 index 00000000000..a4306fe401a --- /dev/null +++ b/acceptance/bin/nostamp.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python3 +"""Read JSON on stdin, write it back with the DMS deployment stamp removed. + +Deployment history recording (DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY=true; see +bundle/test.toml) adds a deployment stamp to plans, states, and resource payloads. Pipe any of +those through this so an acceptance golden compares equal whether or not recording is on. Output +is a 2-space indent, keys in input order, <>& left unescaped, and integers at full precision. +Tests under bundle/dms assert the stamp itself and must not use it. + +Removed in three shapes plus the plan header: + 1. deployment_id/version_id nested in a deployment block, recognized by the + neighbouring "kind" and "metadata_file_path", which are kept; version_id + == "" is kept (a Terraform state dump carries that for an unstamped job). + 2. changes entries keyed "deployment.deployment_id" / "deployment.version_id". + 3. a changes object emptied by (2), or already empty. + 4. the recorded plan header fields deployment_id, next_version_id, last_version_id. +""" + +import argparse +import json +import sys + +_STAMP_KEYS = ("deployment_id", "version_id") +_CHANGE_KEYS = ("deployment.deployment_id", "deployment.version_id") +_PLAN_HEADER_KEYS = ("deployment_id", "next_version_id", "last_version_id") + + +def scrub(node): + if isinstance(node, dict): + # 1. A deployment block is the only object carrying both of these. + if "kind" in node and "metadata_file_path" in node: + for k in _STAMP_KEYS: + if node.get(k, "") != "": + node.pop(k, None) + out = {} + for k, v in node.items(): + if k == "changes" and isinstance(v, dict): + v = {ck: scrub(cv) for ck, cv in v.items() if ck not in _CHANGE_KEYS} + if not v: + continue # 3. drop a changes object left (or already) empty + out[k] = v + else: + out[k] = scrub(v) + return out + if isinstance(node, list): + return [scrub(x) for x in node] + return node + + +def render(data, indent): + data = scrub(data) + if isinstance(data, dict): + for k in _PLAN_HEADER_KEYS: + data.pop(k, None) # 4. plan header, present only at the root + return json.dumps(data, indent=indent, ensure_ascii=False, separators=(",", ": ")) + + +def main(): + parser = argparse.ArgumentParser() + # A state dump is printed with a single-space indent; plans use two. + parser.add_argument("--indent", type=int, default=2) + args = parser.parse_args() + + # Input is one JSON value (a plan or state dump) or a whitespace-separated stream of them (each + # request emitted by a print_requests filter) - jq accepted both here, so this must too. + text = sys.stdin.read() + decoder = json.JSONDecoder() + idx, n = 0, len(text) + while idx < n: + while idx < n and text[idx].isspace(): + idx += 1 + if idx >= n: + break + data, idx = decoder.raw_decode(text, idx) + sys.stdout.write(render(data, args.indent) + "\n") + + +if __name__ == "__main__": + main() diff --git a/acceptance/bin/print_state.py b/acceptance/bin/print_state.py index 7e82f7b3d16..2959a76f81b 100755 --- a/acceptance/bin/print_state.py +++ b/acceptance/bin/print_state.py @@ -8,9 +8,14 @@ import argparse import glob +import json import os +def records_deployment_history(): + return os.environ.get("DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY") == "true" + + def print_file(filename): data = open(filename).read() print(data, end="") @@ -53,6 +58,26 @@ def get_state_file(target, backup): return filtered[0] if filtered else result[0] +def print_recorded_state(filename, target): + """Print the state file with its resources filled in from the deployment metadata service. + + While recording, the file itself carries only the header - the service holds the resources - so + printing it raw would show an empty state and differ from the same test's non-recording run. + """ + # Imported here rather than at module level: dms_resources reads get_state_file from this module. + from dms_resources import get_resources + + data = json.loads(open(filename).read()) + state = {} + for key, value in sorted(get_resources(target).items()): + entry = {"__id__": value["id"], "state": value["state"]} + if value["depends_on"]: + entry["depends_on"] = value["depends_on"] + state[f"resources.{key}"] = entry + data["state"] = state + print(json.dumps(data, indent=1)) + + def main(): parser = argparse.ArgumentParser() parser.add_argument("-t", "--target") @@ -60,7 +85,11 @@ def main(): args = parser.parse_args() for filename in get_state_files(args.target, args.backup): - if os.path.exists(filename): + if not os.path.exists(filename): + continue + if filename.endswith("resources.json") and records_deployment_history(): + print_recorded_state(filename, args.target) + else: print_file(filename) diff --git a/acceptance/bin/read_id.py b/acceptance/bin/read_id.py index 87cd2954bdf..06e7b69281e 100755 --- a/acceptance/bin/read_id.py +++ b/acceptance/bin/read_id.py @@ -15,6 +15,7 @@ sys.path.insert(0, str(Path(__file__).parent)) from add_repl import add_repl +from dms_resources import get_resources, records_deployment_history from print_state import get_state_file @@ -34,6 +35,15 @@ def get_id_terraform(filename, name): print(f"Cannot find resource with {name=}. Available: {available}", file=sys.stderr) +def get_id_recorded(target, name): + resources = get_resources(target) + for key, value in resources.items(): + if key.split(".")[1] == name: + return value["id"] + + print(f"Cannot find recorded resource with {name=}. Available: {list(resources)}", file=sys.stderr) + + def get_id_direct(filename, name): raw = open(filename).read() data = json.loads(raw) @@ -53,11 +63,14 @@ def main(): parser.add_argument("name") args = parser.parse_args() - filename = get_state_file(args.target, args.backup) - if filename.endswith(".tfstate"): - id = get_id_terraform(filename, args.name) + if records_deployment_history(): + id = get_id_recorded(args.target, args.name) else: - id = get_id_direct(filename, args.name) + filename = get_state_file(args.target, args.backup) + if filename.endswith(".tfstate"): + id = get_id_terraform(filename, args.name) + else: + id = get_id_direct(filename, args.name) if id: print(id) diff --git a/acceptance/bin/read_state.py b/acceptance/bin/read_state.py index 0166bf9abbb..4d9bf84186e 100755 --- a/acceptance/bin/read_state.py +++ b/acceptance/bin/read_state.py @@ -9,6 +9,9 @@ import os import sys +sys.path.insert(0, os.path.dirname(__file__)) +from dms_resources import get_resources, records_deployment_history + def print_resource_terraform(group, name, *attrs): resource_type = "databricks_" + group[:-1] @@ -50,7 +53,21 @@ def print_resource_direct(group, name, *attrs): print(group, name, " ".join(values)) -if os.environ.get("DATABRICKS_BUNDLE_ENGINE", "").startswith("direct"): +def print_resource_recorded(group, name, *attrs): + result = get_resources(None).get(f"{group}.{name}") + if result is None: + print(f"State not found for {group}.{name}") + return + + state = dict(result["state"]) + state.setdefault("id", result["id"]) + values = [f"{x}={state.get(x)!r}" for x in attrs] + print(group, name, " ".join(values)) + + +if records_deployment_history(): + print_resource_recorded(*sys.argv[1:]) +elif os.environ.get("DATABRICKS_BUNDLE_ENGINE", "").startswith("direct"): print_resource_direct(*sys.argv[1:]) else: print_resource_terraform(*sys.argv[1:]) diff --git a/acceptance/bin/replace_ids.py b/acceptance/bin/replace_ids.py index e0ebf1cd0e6..2f5d8165032 100755 --- a/acceptance/bin/replace_ids.py +++ b/acceptance/bin/replace_ids.py @@ -10,6 +10,7 @@ sys.path.insert(0, str(Path(__file__).parent)) from add_repl import add_repl +from dms_resources import get_resources, records_deployment_history from print_state import get_state_file @@ -26,6 +27,12 @@ def iter_ids_terraform(filename): yield r_name, id +def iter_ids_recorded(target): + for key, value in get_resources(target).items(): + if value["id"]: + yield key.split(".")[1], value["id"] + + def iter_ids_direct(filename): raw = open(filename).read() data = json.loads(raw) @@ -44,11 +51,14 @@ def main(): parser.add_argument("--backup", action="store_true") args = parser.parse_args() - filename = get_state_file(args.target, args.backup) - if filename.endswith(".tfstate"): - it = iter_ids_terraform(filename) + if records_deployment_history(): + it = iter_ids_recorded(args.target) else: - it = iter_ids_direct(filename) + filename = get_state_file(args.target, args.backup) + if filename.endswith(".tfstate"): + it = iter_ids_terraform(filename) + else: + it = iter_ids_direct(filename) for name, id in it: add_repl(id, name.upper() + "_ID") diff --git a/acceptance/bundle/artifacts/whl_dynamic/out.plan_create.direct.json b/acceptance/bundle/artifacts/whl_dynamic/out.plan_create.direct.json index c881a8a76b2..804ba480a90 100644 --- a/acceptance/bundle/artifacts/whl_dynamic/out.plan_create.direct.json +++ b/acceptance/bundle/artifacts/whl_dynamic/out.plan_create.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.jobs.test_job": { diff --git a/acceptance/bundle/artifacts/whl_dynamic/out.plan_update.direct.json b/acceptance/bundle/artifacts/whl_dynamic/out.plan_update.direct.json index b8d5ada22e7..ca1198e6750 100644 --- a/acceptance/bundle/artifacts/whl_dynamic/out.plan_update.direct.json +++ b/acceptance/bundle/artifacts/whl_dynamic/out.plan_update.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/artifacts/whl_dynamic/script b/acceptance/bundle/artifacts/whl_dynamic/script index ddb6fcd55ec..68d328cc67d 100644 --- a/acceptance/bundle/artifacts/whl_dynamic/script +++ b/acceptance/bundle/artifacts/whl_dynamic/script @@ -4,9 +4,9 @@ cp -r $TESTDIR/../whl_explicit/my_test_code . mkdir prebuilt cp -r $TESTDIR/../whl_prebuilt_multiple/dist/lib/other_test_code-0.0.1-py3-none-any.whl prebuilt -trace $CLI bundle validate -o json | nostamp | jq .artifacts +trace $CLI bundle validate -o json | nostamp.py | jq .artifacts -$CLI bundle plan -o json | nostamp > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp.py > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy title "There are 2 original wheels and 2 patched ones" @@ -25,7 +25,7 @@ rm out.requests.txt title "Updating the local wheel and deploying again\n" touch my_test_code/src/new_module.py -$CLI bundle plan -o json | nostamp > out.plan_update.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp.py > out.plan_update.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy title "Verify contents, it should now have new_module.py" diff --git a/acceptance/bundle/artifacts/whl_implicit_custom_path/script b/acceptance/bundle/artifacts/whl_implicit_custom_path/script index 623849d3bb5..21bc2c9addc 100644 --- a/acceptance/bundle/artifacts/whl_implicit_custom_path/script +++ b/acceptance/bundle/artifacts/whl_implicit_custom_path/script @@ -3,7 +3,7 @@ trace $CLI bundle deploy trace find.py --expect 1 whl title "Expecting 1 wheel in libraries section in /jobs/create" -trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body' out.requests.txt | nostamp +trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body' out.requests.txt | nostamp.py title "Expecting 1 wheel to be uploaded" trace jq .path < out.requests.txt | grep import | grep whl | sort diff --git a/acceptance/bundle/artifacts/whl_via_environment_key/script b/acceptance/bundle/artifacts/whl_via_environment_key/script index f82c5d7eccf..d62de02530b 100644 --- a/acceptance/bundle/artifacts/whl_via_environment_key/script +++ b/acceptance/bundle/artifacts/whl_via_environment_key/script @@ -3,7 +3,7 @@ trace $CLI bundle deploy trace find.py --expect 1 whl title "Expecting 1 wheel in environments section in /jobs/create" -trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body' out.requests.txt | nostamp +trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body' out.requests.txt | nostamp.py title "Expecting 1 wheel to be uploaded" trace jq .path < out.requests.txt | grep import | grep whl | sort diff --git a/acceptance/bundle/artifacts/whl_via_environment_key_extras/script b/acceptance/bundle/artifacts/whl_via_environment_key_extras/script index ccc0d9ed7f8..59268fdeb0c 100644 --- a/acceptance/bundle/artifacts/whl_via_environment_key_extras/script +++ b/acceptance/bundle/artifacts/whl_via_environment_key_extras/script @@ -3,7 +3,7 @@ trace $CLI bundle deploy trace find.py --expect 1 whl title "Expecting the environments dependency to keep its [train] extras suffix" -trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body' out.requests.txt | nostamp +trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body' out.requests.txt | nostamp.py title "Expecting 1 wheel to be uploaded under its bare filename (no extras)" trace jq .path < out.requests.txt | grep import | grep whl | sort diff --git a/acceptance/bundle/bundle_tag/id/script b/acceptance/bundle/bundle_tag/id/script index 759cc89eba1..200c5a7d2e6 100644 --- a/acceptance/bundle/bundle_tag/id/script +++ b/acceptance/bundle/bundle_tag/id/script @@ -1,5 +1,5 @@ trace $CLI bundle validate -trace $CLI bundle validate -o json | nostamp | jq .resources +trace $CLI bundle validate -o json | nostamp.py | jq .resources trace $CLI bundle plan trace $CLI bundle deploy trace print_requests.py --nostamp //jobs diff --git a/acceptance/bundle/bundle_tag/url/script b/acceptance/bundle/bundle_tag/url/script index 759cc89eba1..200c5a7d2e6 100644 --- a/acceptance/bundle/bundle_tag/url/script +++ b/acceptance/bundle/bundle_tag/url/script @@ -1,5 +1,5 @@ trace $CLI bundle validate -trace $CLI bundle validate -o json | nostamp | jq .resources +trace $CLI bundle validate -o json | nostamp.py | jq .resources trace $CLI bundle plan trace $CLI bundle deploy trace print_requests.py --nostamp //jobs diff --git a/acceptance/bundle/deploy/experimental-python/script b/acceptance/bundle/deploy/experimental-python/script index 09b90b28e2d..09bf4b14ef6 100644 --- a/acceptance/bundle/deploy/experimental-python/script +++ b/acceptance/bundle/deploy/experimental-python/script @@ -1,3 +1,3 @@ trace uv run --quiet --with $DATABRICKS_BUNDLES_WHEEL -- $CLI bundle deploy -trace $CLI jobs list --output json | nostamp +trace $CLI jobs list --output json | nostamp.py diff --git a/acceptance/bundle/deploy/immutable/script b/acceptance/bundle/deploy/immutable/script index 85e9a908d1d..83136e53447 100644 --- a/acceptance/bundle/deploy/immutable/script +++ b/acceptance/bundle/deploy/immutable/script @@ -5,7 +5,7 @@ cleanup() { trap cleanup EXIT trace $CLI bundle validate -trace $CLI bundle plan -o json | jq '.plan["resources.jobs.my_job"].new_state.value.tasks' +trace $CLI bundle plan -o json | nostamp.py | jq '.plan["resources.jobs.my_job"].new_state.value.tasks' trace $CLI bundle deploy diff --git a/acceptance/bundle/deploy/python-notebook/script b/acceptance/bundle/deploy/python-notebook/script index 21318c58b66..8b12727b59e 100644 --- a/acceptance/bundle/deploy/python-notebook/script +++ b/acceptance/bundle/deploy/python-notebook/script @@ -1,3 +1,3 @@ trace $CLI bundle deploy -trace $CLI jobs list --output json | nostamp +trace $CLI jobs list --output json | nostamp.py diff --git a/acceptance/bundle/deploy/readplan/basic/out.plan_create.json b/acceptance/bundle/deploy/readplan/basic/out.plan_create.json index 4e8f66bafe4..c093dad07ab 100644 --- a/acceptance/bundle/deploy/readplan/basic/out.plan_create.json +++ b/acceptance/bundle/deploy/readplan/basic/out.plan_create.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.jobs.job": { diff --git a/acceptance/bundle/deploy/readplan/basic/out.plan_skip.json b/acceptance/bundle/deploy/readplan/basic/out.plan_skip.json index a6b4e835bb9..542dc55d6ef 100644 --- a/acceptance/bundle/deploy/readplan/basic/out.plan_skip.json +++ b/acceptance/bundle/deploy/readplan/basic/out.plan_skip.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/deploy/readplan/basic/script b/acceptance/bundle/deploy/readplan/basic/script index 89bf2487b86..e01d5c5f3e7 100644 --- a/acceptance/bundle/deploy/readplan/basic/script +++ b/acceptance/bundle/deploy/readplan/basic/script @@ -1,5 +1,5 @@ # Generate initial plan -trace $CLI bundle plan -o json > out.plan_create.json +trace $CLI bundle plan -o json | nostamp.py > out.plan_create.json trace print_requests.py --get //jobs | contains.py "!GET" "!POST" # First deploy to create the resource @@ -7,7 +7,7 @@ trace $CLI bundle deploy --plan out.plan_create.json trace print_requests.py --get //jobs | contains.py "!GET" "POST" # Generate a plan (should show "skip" since nothing changed) -trace $CLI bundle plan -o json > out.plan_skip.json +trace $CLI bundle plan -o json | nostamp.py > out.plan_skip.json trace print_requests.py --get //jobs | contains.py "GET" trace $CLI bundle deploy --plan out.plan_skip.json diff --git a/acceptance/bundle/deploy/readplan/cli-version-mismatch/plan.json b/acceptance/bundle/deploy/readplan/cli-version-mismatch/plan.json index 932edf74f84..bd9561c9ec8 100644 --- a/acceptance/bundle/deploy/readplan/cli-version-mismatch/plan.json +++ b/acceptance/bundle/deploy/readplan/cli-version-mismatch/plan.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "0.0.0-test", "plan": { "resources.jobs.job": { diff --git a/acceptance/bundle/deploy/readplan/lineage-mismatch/plan.json b/acceptance/bundle/deploy/readplan/lineage-mismatch/plan.json index 42cce15cbfc..80310520edb 100644 --- a/acceptance/bundle/deploy/readplan/lineage-mismatch/plan.json +++ b/acceptance/bundle/deploy/readplan/lineage-mismatch/plan.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "0.0.0-test", "lineage": "different-lineage", "serial": 0, diff --git a/acceptance/bundle/deploy/readplan/plan-version-mismatch/output.txt b/acceptance/bundle/deploy/readplan/plan-version-mismatch/output.txt index 387aac2335c..3ec4497a23e 100644 --- a/acceptance/bundle/deploy/readplan/plan-version-mismatch/output.txt +++ b/acceptance/bundle/deploy/readplan/plan-version-mismatch/output.txt @@ -1,4 +1,4 @@ -Error: plan version mismatch: plan has version 999 (generated with CLI "0.0.0"), but current version is 2 +Error: plan version mismatch: plan has version 999 (generated with CLI "0.0.0"), but current version is 3 Exit code: 1 diff --git a/acceptance/bundle/deploy/readplan/test.toml b/acceptance/bundle/deploy/readplan/test.toml index e6ef3fcdcfb..576ff1ab001 100644 --- a/acceptance/bundle/deploy/readplan/test.toml +++ b/acceptance/bundle/deploy/readplan/test.toml @@ -1,3 +1,3 @@ -# Saved plans don't carry the deployment stamp. Applying one plans an update on the next run. -# See dms_no_readplan in acceptance/bundle/test.toml. +# Dumps post-deploy state/plan, which carry the DMS deployment stamp under recording (applied at +# deploy, not saved in the plan). deploy --plan recording is covered by bundle/dms, so opt out here. EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/deploy/readplan/unknown-field/out.plan.json b/acceptance/bundle/deploy/readplan/unknown-field/out.plan.json index fa91f3d0bf9..84df30af6d8 100644 --- a/acceptance/bundle/deploy/readplan/unknown-field/out.plan.json +++ b/acceptance/bundle/deploy/readplan/unknown-field/out.plan.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.jobs.job": { diff --git a/acceptance/bundle/deploy/readplan/unknown-field/script b/acceptance/bundle/deploy/readplan/unknown-field/script index 6032a3ae34e..ea17245ace1 100644 --- a/acceptance/bundle/deploy/readplan/unknown-field/script +++ b/acceptance/bundle/deploy/readplan/unknown-field/script @@ -1,5 +1,5 @@ # Generate a plan -trace $CLI bundle plan -o json > out.plan.json +trace $CLI bundle plan -o json | nostamp.py > out.plan.json # Modify the plan to rename "action" field to "action_renamed" # This will create an unknown field that the strict decoder should reject diff --git a/acceptance/bundle/deploy/readplan/whl-not-uploaded/out.plan.json b/acceptance/bundle/deploy/readplan/whl-not-uploaded/out.plan.json index da6e6cae939..09a1645c786 100644 --- a/acceptance/bundle/deploy/readplan/whl-not-uploaded/out.plan.json +++ b/acceptance/bundle/deploy/readplan/whl-not-uploaded/out.plan.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 2, diff --git a/acceptance/bundle/deploy/wal/chain-3-jobs/out.test.toml b/acceptance/bundle/deploy/wal/chain-3-jobs/out.test.toml index 59b56a2037c..27ec2a7fcd6 100644 --- a/acceptance/bundle/deploy/wal/chain-3-jobs/out.test.toml +++ b/acceptance/bundle/deploy/wal/chain-3-jobs/out.test.toml @@ -1,3 +1,3 @@ Cloud = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -EnvMatrix.DMS = ["", "true"] +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/deploy/wal/chain-3-jobs/script b/acceptance/bundle/deploy/wal/chain-3-jobs/script index c41e8e7f995..5cc77faa786 100644 --- a/acceptance/bundle/deploy/wal/chain-3-jobs/script +++ b/acceptance/bundle/deploy/wal/chain-3-jobs/script @@ -7,7 +7,7 @@ trace errcode $CLI bundle deploy echo "" echo "=== WAL content after crash ===" -jq -S . .databricks/bundle/default/resources.json.wal 2>/dev/null | nostamp || echo "No WAL file" +jq -S . .databricks/bundle/default/resources.json.wal 2>/dev/null | nostamp.py || echo "No WAL file" echo "" echo "=== Number of jobs saved in WAL ===" diff --git a/acceptance/bundle/deploy/wal/crash-after-create/out.test.toml b/acceptance/bundle/deploy/wal/crash-after-create/out.test.toml index b0851b242b8..9a7cc99433c 100644 --- a/acceptance/bundle/deploy/wal/crash-after-create/out.test.toml +++ b/acceptance/bundle/deploy/wal/crash-after-create/out.test.toml @@ -1,4 +1,4 @@ Cloud = false EnvMatrix.COMMAND = ["plan", "deploy --force-lock", "summary"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -EnvMatrix.DMS = ["", "true"] +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/deploy/wal/crash-after-create/script b/acceptance/bundle/deploy/wal/crash-after-create/script index 5647c83e387..22f7587e52e 100644 --- a/acceptance/bundle/deploy/wal/crash-after-create/script +++ b/acceptance/bundle/deploy/wal/crash-after-create/script @@ -8,7 +8,7 @@ trace errcode $CLI bundle deploy trace assert_exists.py .databricks/bundle/default/resources.json.wal trace assert_not_exists.py .databricks/bundle/default/resources.json -trace cat .databricks/bundle/default/resources.json.wal | jq | nostamp +trace cat .databricks/bundle/default/resources.json.wal | jq | nostamp.py title "Any other command recovers state" $CLI bundle $COMMAND &> LOG.COMMAND.txt diff --git a/acceptance/bundle/deploy/wal/empty-wal/out.test.toml b/acceptance/bundle/deploy/wal/empty-wal/out.test.toml index 59b56a2037c..27ec2a7fcd6 100644 --- a/acceptance/bundle/deploy/wal/empty-wal/out.test.toml +++ b/acceptance/bundle/deploy/wal/empty-wal/out.test.toml @@ -1,3 +1,3 @@ Cloud = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -EnvMatrix.DMS = ["", "true"] +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/deploy/wal/failed-plan-no-wal/out.test.toml b/acceptance/bundle/deploy/wal/failed-plan-no-wal/out.test.toml index 59b56a2037c..27ec2a7fcd6 100644 --- a/acceptance/bundle/deploy/wal/failed-plan-no-wal/out.test.toml +++ b/acceptance/bundle/deploy/wal/failed-plan-no-wal/out.test.toml @@ -1,3 +1,3 @@ Cloud = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -EnvMatrix.DMS = ["", "true"] +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/deploy/wal/future-serial-wal/out.test.toml b/acceptance/bundle/deploy/wal/future-serial-wal/out.test.toml index 59b56a2037c..27ec2a7fcd6 100644 --- a/acceptance/bundle/deploy/wal/future-serial-wal/out.test.toml +++ b/acceptance/bundle/deploy/wal/future-serial-wal/out.test.toml @@ -1,3 +1,3 @@ Cloud = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -EnvMatrix.DMS = ["", "true"] +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/deploy/wal/header-only-wal/out.test.toml b/acceptance/bundle/deploy/wal/header-only-wal/out.test.toml index 59b56a2037c..27ec2a7fcd6 100644 --- a/acceptance/bundle/deploy/wal/header-only-wal/out.test.toml +++ b/acceptance/bundle/deploy/wal/header-only-wal/out.test.toml @@ -1,3 +1,3 @@ Cloud = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -EnvMatrix.DMS = ["", "true"] +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/deploy/wal/header-only-wal/output.txt b/acceptance/bundle/deploy/wal/header-only-wal/output.txt index 807fa926044..da8c1bdc1f3 100644 --- a/acceptance/bundle/deploy/wal/header-only-wal/output.txt +++ b/acceptance/bundle/deploy/wal/header-only-wal/output.txt @@ -28,7 +28,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged >>> errcode assert_not_exists.py .databricks/bundle/default/resources.json.wal ->>> errcode cat .databricks/bundle/default/resources.json +>>> errcode print_state.py { "serial": 1, "state_keys": [ diff --git a/acceptance/bundle/deploy/wal/header-only-wal/script b/acceptance/bundle/deploy/wal/header-only-wal/script index 89da7e1a277..557fc7d4f47 100644 --- a/acceptance/bundle/deploy/wal/header-only-wal/script +++ b/acceptance/bundle/deploy/wal/header-only-wal/script @@ -19,4 +19,4 @@ title "Third deploy (must recover and succeed, not blocked by the leftover WAL)" trace errcode $CLI bundle deploy --force-lock trace errcode assert_not_exists.py .databricks/bundle/default/resources.json.wal -trace errcode cat .databricks/bundle/default/resources.json | jq -S '{serial: .serial, state_keys: (.state | keys)}' +trace errcode print_state.py | jq -S '{serial: .serial, state_keys: (.state | keys)}' diff --git a/acceptance/bundle/deploy/wal/lineage-mismatch/out.test.toml b/acceptance/bundle/deploy/wal/lineage-mismatch/out.test.toml index dc89c55cf90..2845c70cfef 100644 --- a/acceptance/bundle/deploy/wal/lineage-mismatch/out.test.toml +++ b/acceptance/bundle/deploy/wal/lineage-mismatch/out.test.toml @@ -1,4 +1,4 @@ Cloud = false EnvMatrix.COMMAND = ["deploy", "plan", "summary"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -EnvMatrix.DMS = ["", "true"] +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/deploy/wal/test.toml b/acceptance/bundle/deploy/wal/test.toml index 5218ccbbd10..e4be659d2e1 100644 --- a/acceptance/bundle/deploy/wal/test.toml +++ b/acceptance/bundle/deploy/wal/test.toml @@ -1,10 +1,16 @@ # WAL (Write-Ahead Log) tests verify crash recovery during bundle deployment. # These tests simulate process crashes using KillCaller and verify state recovery. # Only runs with direct engine since WAL is a direct-engine feature. +# +# Not run under deployment history recording, and not planned to be: WAL recovery is a +# non-recording concept - under recording the service is the source of truth and a leftover WAL is +# discarded, not replayed (see dstate.Open; the DMS side is covered by bundle/dms) - so there is +# nothing here a DMS run would exercise. Env.DATABRICKS_CLI_TEST_PID = "1" EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DMS = [""] [[Repls]] Old = 'script: line \d+:\s+\d+ Killed(: 9)?\s+"\$@"' diff --git a/acceptance/bundle/deploy/wal/wal-with-delete/out.test.toml b/acceptance/bundle/deploy/wal/wal-with-delete/out.test.toml index 59b56a2037c..27ec2a7fcd6 100644 --- a/acceptance/bundle/deploy/wal/wal-with-delete/out.test.toml +++ b/acceptance/bundle/deploy/wal/wal-with-delete/out.test.toml @@ -1,3 +1,3 @@ Cloud = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -EnvMatrix.DMS = ["", "true"] +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/deployment/bind/experiment/script b/acceptance/bundle/deployment/bind/experiment/script index 70cf1933811..e31a17f7eaa 100644 --- a/acceptance/bundle/deployment/bind/experiment/script +++ b/acceptance/bundle/deployment/bind/experiment/script @@ -21,7 +21,7 @@ trace $CLI bundle deploy --force-lock --auto-approve # Use --output json to verify the change details including skip reason. # Write per-engine output since direct includes change details that terraform doesn't. # Filter to only the name change (tags/artifact_location may differ between local and cloud). -$CLI bundle plan --output json | jq '{plan: .plan | map_values({action, name_change: (.changes.name // null)})}' > out.plan1.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan --output json | nostamp.py | jq '{plan: .plan | map_values({action, name_change: (.changes.name // null)})}' > out.plan1.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI experiments get-experiment ${EXPERIMENT_ID} | jq '{name: .experiment.name, lifecycle_stage: .experiment.lifecycle_stage}' @@ -29,7 +29,7 @@ trace $CLI experiments get-experiment ${EXPERIMENT_ID} | jq '{name: .experiment. # Plan should still show no changes because the diff is suppressed. EXPERIMENT_NAME="//Users/${CURRENT_USER_NAME}/test-experiment$UNIQUE_NAME" envsubst < databricks.yml.tmpl > databricks.yml -$CLI bundle plan --output json | jq '{plan: .plan | map_values({action, name_change: (.changes.name // null)})}' > out.plan2.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan --output json | nostamp.py | jq '{plan: .plan | map_values({action, name_change: (.changes.name // null)})}' > out.plan2.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deployment unbind experiment1 diff --git a/acceptance/bundle/destroy/jobs-and-pipeline/script b/acceptance/bundle/destroy/jobs-and-pipeline/script index b02d93bcecb..2391f9d44e0 100644 --- a/acceptance/bundle/destroy/jobs-and-pipeline/script +++ b/acceptance/bundle/destroy/jobs-and-pipeline/script @@ -45,7 +45,7 @@ trace $CLI workspace get-status "${DEPLOYMENT_PATH}" | jq '{path, object_type}' title "Assert the pipeline is created" PIPELINE_ID=$($CLI bundle summary -o json | jq -r '.resources.pipelines.bar.id') -trace $CLI pipelines get "${PIPELINE_ID}" | nostamp | jq "{spec}" +trace $CLI pipelines get "${PIPELINE_ID}" | nostamp.py | jq "{spec}" title "Assert the job is created:\n" JOB_ID=$($CLI bundle summary -o json | jq -r '.resources.jobs.foo.id') diff --git a/acceptance/bundle/dms/declined-deploy/databricks.yml.tmpl b/acceptance/bundle/dms/declined-deploy/databricks.yml.tmpl new file mode 100644 index 00000000000..e909afaab5c --- /dev/null +++ b/acceptance/bundle/dms/declined-deploy/databricks.yml.tmpl @@ -0,0 +1,10 @@ +bundle: + name: dms-declined-deploy-$UNIQUE_NAME +experimental: + record_deployment_history: true + +resources: + schemas: + foo: + name: dms_declined_deploy_schema_$UNIQUE_NAME + catalog_name: main diff --git a/acceptance/bundle/dms/declined-deploy/out.requests.txt b/acceptance/bundle/dms/declined-deploy/out.requests.txt new file mode 100644 index 00000000000..eb169f26a0c --- /dev/null +++ b/acceptance/bundle/dms/declined-deploy/out.requests.txt @@ -0,0 +1,7 @@ +{ + "method": "DELETE", + "path": "/api/2.1/unity-catalog/catalogs/dms_other_[UNIQUE_NAME]", + "q": { + "force": "true" + } +} diff --git a/acceptance/bundle/dms/declined-deploy/out.test.toml b/acceptance/bundle/dms/declined-deploy/out.test.toml new file mode 100644 index 00000000000..23c07f70dca --- /dev/null +++ b/acceptance/bundle/dms/declined-deploy/out.test.toml @@ -0,0 +1,3 @@ +Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DMS = ["true"] diff --git a/acceptance/bundle/dms/declined-deploy/output.txt b/acceptance/bundle/dms/declined-deploy/output.txt new file mode 100644 index 00000000000..d09e8d18743 --- /dev/null +++ b/acceptance/bundle/dms/declined-deploy/output.txt @@ -0,0 +1,88 @@ + +=== Deploy a schema, so the deployment and its first version exist +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-declined-deploy-[UNIQUE_NAME]/default/files... +Created schemas.foo +Files: 5 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py --dms //api/2.0/bundle +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments", + "body": { + "display_name": "dms-declined-deploy-[UNIQUE_NAME]", + "initial_parent_path": "/Workspace/Users/[USERNAME]/.bundle/dms-declined-deploy-[UNIQUE_NAME]/default/state", + "target_name": "default", + "workspace_info": { + "file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-declined-deploy-[UNIQUE_NAME]/default/files", + "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-declined-deploy-[UNIQUE_NAME]/default" + } + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions", + "q": { + "version_id": "1" + }, + "body": { + "cli_version": "[CLI_VERSION]", + "version_type": "VERSION_TYPE_DEPLOY", + "operations": [ + { + "resource_key": "schemas.foo", + "action_type": "OPERATION_ACTION_TYPE_CREATE" + } + ] + } +} +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/schemas.foo", + "q": { + "update_mask": "state,error_message,resource_id,status" + }, + "body": { + "error_message": "", + "resource_id": "main.dms_declined_deploy_schema_[UNIQUE_NAME]", + "sequence_id": "0", + "state": "{\"state\":{\"catalog_name\":\"main\",\"name\":\"dms_declined_deploy_schema_[UNIQUE_NAME]\"}}", + "status": "OPERATION_STATUS_SUCCEEDED" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/complete", + "body": { + "completion_reason": "VERSION_COMPLETE_SUCCESS" + } +} + +=== A destructive change without --auto-approve is declined: this console cannot prompt +>>> update_file.py databricks.yml catalog_name: main catalog_name: dms_other_[UNIQUE_NAME] + +>>> musterr [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-declined-deploy-[UNIQUE_NAME]/default/files... + +This action will result in the deletion or recreation of the following UC schemas. Any underlying data may be lost: + recreate resources.schemas.foo +Error: the deployment requires destructive actions, but the current console does not support prompting. +Deleting data assets such as schemas, pipelines, or volumes may cause permanent data loss and should be carefully reviewed. +To proceed, use --auto-approve after reviewing the plan above. + +Files: 4 uploaded, 0 deleted + +=== Nothing was recorded for the declined deploy - no version, so none to abort +>>> print_requests.py --dms //api/2.0/bundle + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.schemas.foo + +This action will result in the deletion of the following UC schemas. Any underlying data may be lost: + delete resources.schemas.foo + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/dms-declined-deploy-[UNIQUE_NAME]/default + +Destroy: 1 deleted diff --git a/acceptance/bundle/dms/declined-deploy/script b/acceptance/bundle/dms/declined-deploy/script new file mode 100644 index 00000000000..e67ff786b19 --- /dev/null +++ b/acceptance/bundle/dms/declined-deploy/script @@ -0,0 +1,22 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +title "Deploy a schema, so the deployment and its first version exist" +trace $CLI bundle deploy +trace print_requests.py --dms //api/2.0/bundle + +title "A destructive change without --auto-approve is declined: this console cannot prompt" +# Changing the catalog recreates the schema, which needs approval. +# Unity Catalog has no catalog called "other", so make one to move the schema into. +other_catalog="dms_other_${UNIQUE_NAME}" +$CLI catalogs create "$other_catalog" &> LOG.catalog +trap '$CLI catalogs delete "$other_catalog" --force >> LOG.catalog 2>&1' EXIT +trace update_file.py databricks.yml "catalog_name: main" "catalog_name: ${other_catalog}" +trace musterr $CLI bundle deploy + +title "Nothing was recorded for the declined deploy - no version, so none to abort" +# The version number it would have used is left for the next deploy to take, so the +# history has no entry that reads like a deploy which failed or did nothing. +trace print_requests.py --dms //api/2.0/bundle + +trace $CLI bundle destroy --auto-approve +rm -f out.requests.txt diff --git a/acceptance/bundle/dms/depends-on/databricks.yml.tmpl b/acceptance/bundle/dms/depends-on/databricks.yml.tmpl new file mode 100644 index 00000000000..1a60d215ecc --- /dev/null +++ b/acceptance/bundle/dms/depends-on/databricks.yml.tmpl @@ -0,0 +1,12 @@ +bundle: + name: dms-depends-on-$UNIQUE_NAME +experimental: + record_deployment_history: true + +resources: + jobs: + parent: + name: parent + child: + name: child + description: depends on ${resources.jobs.parent.id} diff --git a/acceptance/bundle/dms/depends-on/out.test.toml b/acceptance/bundle/dms/depends-on/out.test.toml new file mode 100644 index 00000000000..23c07f70dca --- /dev/null +++ b/acceptance/bundle/dms/depends-on/out.test.toml @@ -0,0 +1,3 @@ +Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DMS = ["true"] diff --git a/acceptance/bundle/dms/depends-on/output.txt b/acceptance/bundle/dms/depends-on/output.txt new file mode 100644 index 00000000000..4807497d63c --- /dev/null +++ b/acceptance/bundle/dms/depends-on/output.txt @@ -0,0 +1,47 @@ + +=== Deploy a job that references another: depends_on is recorded alongside the config, since the reference is resolved to a literal in the config itself +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-depends-on-[UNIQUE_NAME]/default/files... +Created jobs.child +Created jobs.parent +Files: 5 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py --dms //versions/1/operations +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.parent", + "q": { + "update_mask": "state,error_message,resource_id,status" + }, + "body": { + "error_message": "", + "resource_id": "[NUMID]", + "sequence_id": "0", + "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-depends-on-[UNIQUE_NAME]/default/state/metadata.json\",\"version_id\":\"1\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"parent\",\"queue\":{\"enabled\":true}}}", + "status": "OPERATION_STATUS_SUCCEEDED" + } +} +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.child", + "q": { + "update_mask": "state,error_message,resource_id,status" + }, + "body": { + "error_message": "", + "resource_id": "[NUMID]", + "sequence_id": "0", + "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-depends-on-[UNIQUE_NAME]/default/state/metadata.json\",\"version_id\":\"1\"},\"description\":\"depends on [NUMID]\",\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"child\",\"queue\":{\"enabled\":true}},\"depends_on\":[{\"node\":\"resources.jobs.parent\",\"label\":\"${resources.jobs.parent.id}\"}]}", + "status": "OPERATION_STATUS_SUCCEEDED" + } +} + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.jobs.child + delete resources.jobs.parent + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/dms-depends-on-[UNIQUE_NAME]/default + +Destroy: 2 deleted diff --git a/acceptance/bundle/dms/depends-on/script b/acceptance/bundle/dms/depends-on/script new file mode 100644 index 00000000000..7f5576ce72b --- /dev/null +++ b/acceptance/bundle/dms/depends-on/script @@ -0,0 +1,8 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +title "Deploy a job that references another: depends_on is recorded alongside the config, since the reference is resolved to a literal in the config itself" +trace $CLI bundle deploy +trace print_requests.py --dms //versions/1/operations + +trace $CLI bundle destroy --auto-approve +rm -f out.requests.txt diff --git a/acceptance/bundle/dms/deployment-metadata-change/databricks.yml.tmpl b/acceptance/bundle/dms/deployment-metadata-change/databricks.yml.tmpl new file mode 100644 index 00000000000..07f434b2e7a --- /dev/null +++ b/acceptance/bundle/dms/deployment-metadata-change/databricks.yml.tmpl @@ -0,0 +1,22 @@ +bundle: + name: dms-metadata-before-$UNIQUE_NAME +experimental: + record_deployment_history: true + +workspace: + # Pinned, so renaming the bundle or switching target below keeps the same deployment: the + # deployment is the workspace node under state_path, and a different path would be a + # different deployment. + state_path: /Workspace/Users/${workspace.current_user.userName}/.bundle/dms-metadata-pinned-$UNIQUE_NAME/state + +resources: + schemas: + foo: + name: dms_metadata_schema_$UNIQUE_NAME + catalog_name: main + +targets: + dev: + default: true + prod: + mode: production diff --git a/acceptance/bundle/dms/deployment-metadata-change/out.test.toml b/acceptance/bundle/dms/deployment-metadata-change/out.test.toml new file mode 100644 index 00000000000..23c07f70dca --- /dev/null +++ b/acceptance/bundle/dms/deployment-metadata-change/out.test.toml @@ -0,0 +1,3 @@ +Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DMS = ["true"] diff --git a/acceptance/bundle/dms/deployment-metadata-change/output.txt b/acceptance/bundle/dms/deployment-metadata-change/output.txt new file mode 100644 index 00000000000..0b0798eee7f --- /dev/null +++ b/acceptance/bundle/dms/deployment-metadata-change/output.txt @@ -0,0 +1,144 @@ + +=== Deploy: the deployment is created carrying this bundle's metadata +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-metadata-before-[UNIQUE_NAME]/dev/files... +Created schemas.foo +Files: 5 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py --dms //api/2.0/bundle +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments", + "body": { + "display_name": "dms-metadata-before-[UNIQUE_NAME]", + "initial_parent_path": "/Workspace/Users/[USERNAME]/.bundle/dms-metadata-pinned-[UNIQUE_NAME]/state", + "target_name": "dev", + "workspace_info": { + "file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-metadata-before-[UNIQUE_NAME]/dev/files", + "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-metadata-before-[UNIQUE_NAME]/dev" + } + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions", + "q": { + "version_id": "1" + }, + "body": { + "cli_version": "[CLI_VERSION]", + "version_type": "VERSION_TYPE_DEPLOY", + "operations": [ + { + "resource_key": "schemas.foo", + "action_type": "OPERATION_ACTION_TYPE_CREATE" + } + ] + } +} +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/schemas.foo", + "q": { + "update_mask": "state,error_message,resource_id,status" + }, + "body": { + "error_message": "", + "resource_id": "main.dms_metadata_schema_[UNIQUE_NAME]", + "sequence_id": "0", + "state": "{\"state\":{\"catalog_name\":\"main\",\"name\":\"dms_metadata_schema_[UNIQUE_NAME]\"}}", + "status": "OPERATION_STATUS_SUCCEEDED" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/complete", + "body": { + "completion_reason": "VERSION_COMPLETE_SUCCESS" + } +} + +=== Rename the bundle and switch target+mode. The state path is pinned, so this is the same deployment, updated in place - the mask carries every field that changed +>>> update_file.py databricks.yml dms-metadata-before dms-metadata-after + +>>> [CLI] bundle deploy -t prod +Recommendation: target with 'mode: production' should set 'workspace.root_path' to make sure only one copy is deployed + +A common practice is to use a username or principal name in this path, i.e. use + + root_path: /Workspace/Users/[USERNAME]/.bundle/${bundle.name}/${bundle.target} + +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-metadata-after-[UNIQUE_NAME]/prod/files... +Files: 5 uploaded, 0 deleted +Resources: 0 created, 0 changed, 0 deleted, 1 unchanged + +>>> print_requests.py --dms //api/2.0/bundle +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]", + "q": { + "update_mask": "display_name,target_name,deployment_mode,workspace_info" + }, + "body": { + "deployment_mode": "DEPLOYMENT_MODE_PRODUCTION", + "display_name": "dms-metadata-after-[UNIQUE_NAME]", + "target_name": "prod", + "workspace_info": { + "file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-metadata-after-[UNIQUE_NAME]/prod/files", + "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-metadata-after-[UNIQUE_NAME]/prod" + } + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions", + "q": { + "version_id": "2" + }, + "body": { + "cli_version": "[CLI_VERSION]", + "version_type": "VERSION_TYPE_DEPLOY", + "previous_version_id": "1" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/complete", + "body": { + "completion_reason": "VERSION_COMPLETE_SUCCESS" + } +} + +=== Deploy again without touching anything: the record already says this, so no deployment update +>>> [CLI] bundle deploy -t prod +Recommendation: target with 'mode: production' should set 'workspace.root_path' to make sure only one copy is deployed + +A common practice is to use a username or principal name in this path, i.e. use + + root_path: /Workspace/Users/[USERNAME]/.bundle/${bundle.name}/${bundle.target} + +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-metadata-after-[UNIQUE_NAME]/prod/files... +Files: 2 uploaded, 0 deleted +Resources: 0 created, 0 changed, 0 deleted, 1 unchanged + +>>> print_requests.py --dms //api/2.0/bundle +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions", + "q": { + "version_id": "3" + }, + "body": { + "cli_version": "[CLI_VERSION]", + "version_type": "VERSION_TYPE_DEPLOY", + "previous_version_id": "2" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/3/complete", + "body": { + "completion_reason": "VERSION_COMPLETE_SUCCESS" + } +} diff --git a/acceptance/bundle/dms/deployment-metadata-change/script b/acceptance/bundle/dms/deployment-metadata-change/script new file mode 100644 index 00000000000..03f0c593999 --- /dev/null +++ b/acceptance/bundle/dms/deployment-metadata-change/script @@ -0,0 +1,14 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +title "Deploy: the deployment is created carrying this bundle's metadata" +trace $CLI bundle deploy +trace print_requests.py --dms //api/2.0/bundle + +title "Rename the bundle and switch target+mode. The state path is pinned, so this is the same deployment, updated in place - the mask carries every field that changed" +trace update_file.py databricks.yml dms-metadata-before dms-metadata-after +trace $CLI bundle deploy -t prod +trace print_requests.py --dms //api/2.0/bundle + +title "Deploy again without touching anything: the record already says this, so no deployment update" +trace $CLI bundle deploy -t prod +trace print_requests.py --dms //api/2.0/bundle diff --git a/acceptance/bundle/dms/emptied-resource/databricks.yml.tmpl b/acceptance/bundle/dms/emptied-resource/databricks.yml.tmpl new file mode 100644 index 00000000000..b8c21a3cc96 --- /dev/null +++ b/acceptance/bundle/dms/emptied-resource/databricks.yml.tmpl @@ -0,0 +1,11 @@ +bundle: + name: dms-emptied-resource-$UNIQUE_NAME +experimental: + record_deployment_history: true + +resources: + schemas: + foo: + name: dms_emptied_resource_$UNIQUE_NAME + catalog_name: main + grants: [{principal: deco-test-user@databricks.com, privileges: [USE_SCHEMA]}] diff --git a/acceptance/bundle/dms/emptied-resource/out.test.toml b/acceptance/bundle/dms/emptied-resource/out.test.toml new file mode 100644 index 00000000000..23c07f70dca --- /dev/null +++ b/acceptance/bundle/dms/emptied-resource/out.test.toml @@ -0,0 +1,3 @@ +Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DMS = ["true"] diff --git a/acceptance/bundle/dms/emptied-resource/output.txt b/acceptance/bundle/dms/emptied-resource/output.txt new file mode 100644 index 00000000000..483b3d8249d --- /dev/null +++ b/acceptance/bundle/dms/emptied-resource/output.txt @@ -0,0 +1,142 @@ + +=== Deploy a schema with one grant: the grants node is recorded with its state +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-emptied-resource-[UNIQUE_NAME]/default/files... +Created schemas.foo +Created schemas.foo.grants +Files: 5 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py --dms //api/2.0/bundle +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments", + "body": { + "display_name": "dms-emptied-resource-[UNIQUE_NAME]", + "initial_parent_path": "/Workspace/Users/[USERNAME]/.bundle/dms-emptied-resource-[UNIQUE_NAME]/default/state", + "target_name": "default", + "workspace_info": { + "file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-emptied-resource-[UNIQUE_NAME]/default/files", + "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-emptied-resource-[UNIQUE_NAME]/default" + } + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions", + "q": { + "version_id": "1" + }, + "body": { + "cli_version": "[CLI_VERSION]", + "version_type": "VERSION_TYPE_DEPLOY", + "operations": [ + { + "resource_key": "schemas.foo", + "action_type": "OPERATION_ACTION_TYPE_CREATE" + }, + { + "resource_key": "schemas.foo.grants", + "action_type": "OPERATION_ACTION_TYPE_CREATE" + } + ] + } +} +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/schemas.foo", + "q": { + "update_mask": "state,error_message,resource_id,status" + }, + "body": { + "error_message": "", + "resource_id": "main.dms_emptied_resource_[UNIQUE_NAME]", + "sequence_id": "0", + "state": "{\"state\":{\"catalog_name\":\"main\",\"name\":\"dms_emptied_resource_[UNIQUE_NAME]\"}}", + "status": "OPERATION_STATUS_SUCCEEDED" + } +} +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/schemas.foo.grants", + "q": { + "update_mask": "state,error_message,resource_id,status" + }, + "body": { + "error_message": "", + "resource_id": "schema/main.dms_emptied_resource_[UNIQUE_NAME]", + "sequence_id": "0", + "state": "{\"state\":{\"securable_type\":\"schema\",\"full_name\":\"main.dms_emptied_resource_[UNIQUE_NAME]\",\"__embed__\":[{\"principal\":\"deco-test-user@databricks.com\",\"privileges\":[\"USE_SCHEMA\"]}]},\"depends_on\":[{\"node\":\"resources.schemas.foo\",\"label\":\"${resources.schemas.foo.id}\"}]}", + "status": "OPERATION_STATUS_SUCCEEDED" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/complete", + "body": { + "completion_reason": "VERSION_COMPLETE_SUCCESS" + } +} + +=== Revoke the grant, so the grants node empties out +>>> update_file.py databricks.yml grants: [{principal: deco-test-user@databricks.com, privileges: [USE_SCHEMA]}] grants: [] + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-emptied-resource-[UNIQUE_NAME]/default/files... +Updated schemas.foo.grants +Files: 3 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 1 unchanged + +>>> print_requests.py --dms //api/2.0/bundle +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions", + "q": { + "version_id": "2" + }, + "body": { + "cli_version": "[CLI_VERSION]", + "version_type": "VERSION_TYPE_DEPLOY", + "previous_version_id": "1", + "operations": [ + { + "resource_key": "schemas.foo.grants", + "action_type": "OPERATION_ACTION_TYPE_UPDATE" + } + ] + } +} +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations/schemas.foo.grants", + "q": { + "update_mask": "state,error_message,status" + }, + "body": { + "error_message": "", + "sequence_id": "0", + "status": "OPERATION_STATUS_SUCCEEDED" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/complete", + "body": { + "completion_reason": "VERSION_COMPLETE_SUCCESS" + } +} + +=== Plan again: reading state back from the service works and reports no work +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.schemas.foo + +This action will result in the deletion of the following UC schemas. Any underlying data may be lost: + delete resources.schemas.foo + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/dms-emptied-resource-[UNIQUE_NAME]/default + +Destroy: 1 deleted diff --git a/acceptance/bundle/dms/emptied-resource/script b/acceptance/bundle/dms/emptied-resource/script new file mode 100644 index 00000000000..b38be294ce1 --- /dev/null +++ b/acceptance/bundle/dms/emptied-resource/script @@ -0,0 +1,20 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +title "Deploy a schema with one grant: the grants node is recorded with its state" +trace $CLI bundle deploy +trace print_requests.py --dms //api/2.0/bundle + +title "Revoke the grant, so the grants node empties out" +trace update_file.py databricks.yml 'grants: [{principal: deco-test-user@databricks.com, privileges: [USE_SCHEMA]}]' 'grants: []' +trace $CLI bundle deploy + +# The action type stays update: it is staged from the plan, before apply knows the state +# comes back empty. What drops the resource from the deployment is the update naming state +# with no value. +trace print_requests.py --dms //api/2.0/bundle + +title "Plan again: reading state back from the service works and reports no work" +trace $CLI bundle plan | contains.py "Plan: 0 to add, 0 to change, 0 to delete" "!unexpected end of JSON input" + +trace $CLI bundle destroy --auto-approve +rm -f out.requests.txt diff --git a/acceptance/bundle/dms/existing-state/databricks.yml.tmpl b/acceptance/bundle/dms/existing-state/databricks.yml.tmpl new file mode 100644 index 00000000000..29addd6433e --- /dev/null +++ b/acceptance/bundle/dms/existing-state/databricks.yml.tmpl @@ -0,0 +1,7 @@ +bundle: + name: dms-existing-state-$UNIQUE_NAME + +resources: + jobs: + one: + name: one diff --git a/acceptance/bundle/dms/existing-state/out.test.toml b/acceptance/bundle/dms/existing-state/out.test.toml new file mode 100644 index 00000000000..d73c45e3119 --- /dev/null +++ b/acceptance/bundle/dms/existing-state/out.test.toml @@ -0,0 +1,3 @@ +Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/dms/existing-state/output.txt b/acceptance/bundle/dms/existing-state/output.txt new file mode 100644 index 00000000000..9a7439eef8a --- /dev/null +++ b/acceptance/bundle/dms/existing-state/output.txt @@ -0,0 +1,78 @@ + +=== Deploy without recording: ordinary direct-engine state tracking one job, unknown to DMS +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default/files... +Created jobs.one +Files: 6 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py --dms //api/2.0/bundle --oneline + +>>> jq {state_version, features} .databricks/bundle/default/resources.json +{ + "state_version": 2, + "features": null +} + +=== Enabling recording is refused: this deployment already exists but the service has no record of it, and treating the service as authoritative would create its resources a second time +>>> DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY=true [CLI] bundle deploy +Error: this deployment already exists and is not recorded with the deployment metadata service, so it cannot be recorded without redeploying its resources + +To record this bundle's history, start it over as a new deployment: + 1. remove experimental.record_deployment_history from your bundle configuration + 2. run "databricks bundle destroy" to delete the existing resources + 3. add experimental.record_deployment_history back and deploy again + + +>>> print_requests.py --dms //api/2.0/bundle --oneline + +=== The check is against the remote state, not the local cache: wiping .databricks still refuses, because the deploy pulls the state back from the workspace +>>> DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY=true [CLI] bundle deploy +Error: this deployment already exists and is not recorded with the deployment metadata service, so it cannot be recorded without redeploying its resources + +To record this bundle's history, start it over as a new deployment: + 1. remove experimental.record_deployment_history from your bundle configuration + 2. run "databricks bundle destroy" to delete the existing resources + 3. add experimental.record_deployment_history back and deploy again + + +=== A leftover local WAL is irrelevant too: under recording it is discarded rather than replayed, so the refusal stands +>>> DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY=true [CLI] bundle deploy +Error: this deployment already exists and is not recorded with the deployment metadata service, so it cannot be recorded without redeploying its resources + +To record this bundle's history, start it over as a new deployment: + 1. remove experimental.record_deployment_history from your bundle configuration + 2. run "databricks bundle destroy" to delete the existing resources + 3. add experimental.record_deployment_history back and deploy again + + +=== Destroy clears the deployment. With nothing recorded remotely the bundle is treated as new, so recording can be enabled - and a leftover WAL is still discarded, not replayed +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.jobs.one + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default + +Destroy: 1 deleted + +>>> DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY=true [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default/files... +Created jobs.one +Files: 6 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py --dms //api/2.0/bundle --oneline +{"method": "POST", "path": "/api/2.0/bundle/deployments", "body": {"display_name": "dms-existing-state-[UNIQUE_NAME]", "initial_parent_path": "/Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default/state", "target_name": "default", "workspace_info": {"file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default/files", "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default"}}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": {"version_id": "1"}, "body": {"cli_version": "[CLI_VERSION]", "version_type": "VERSION_TYPE_DEPLOY", "operations": [{"resource_key": "jobs.one", "action_type": "OPERATION_ACTION_TYPE_CREATE"}]}} +{"method": "PATCH", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.one", "q": {"update_mask": "state,error_message,resource_id,status"}, "body": {"error_message": "", "resource_id": "[NUMID]", "sequence_id": "0", "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default/state/metadata.json\",\"version_id\":\"1\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"one\",\"queue\":{\"enabled\":true}}}", "status": "OPERATION_STATUS_SUCCEEDED"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/complete", "body": {"completion_reason": "VERSION_COMPLETE_SUCCESS"}} + +>>> find.py resources[.]json[.]wal --expect 0 + +>>> jq {state_version, features} .databricks/bundle/default/resources.json +{ + "state_version": 2, + "features": { + "record_deployment_history": {} + } +} diff --git a/acceptance/bundle/dms/existing-state/script b/acceptance/bundle/dms/existing-state/script new file mode 100644 index 00000000000..e0784bb406b --- /dev/null +++ b/acceptance/bundle/dms/existing-state/script @@ -0,0 +1,27 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +title "Deploy without recording: ordinary direct-engine state tracking one job, unknown to DMS" +trace $CLI bundle deploy +trace print_requests.py --dms //api/2.0/bundle --oneline +trace jq '{state_version, features}' .databricks/bundle/default/resources.json + +title "Enabling recording is refused: this deployment already exists but the service has no record of it, and treating the service as authoritative would create its resources a second time" +musterr trace DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY=true $CLI bundle deploy +trace print_requests.py --dms //api/2.0/bundle --oneline + +title "The check is against the remote state, not the local cache: wiping .databricks still refuses, because the deploy pulls the state back from the workspace" +rm -rf .databricks +musterr trace DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY=true $CLI bundle deploy + +title "A leftover local WAL is irrelevant too: under recording it is discarded rather than replayed, so the refusal stands" +mkdir -p .databricks/bundle/default +printf '{"lineage":"stale","serial":99,"state_version":2}\n' > .databricks/bundle/default/resources.json.wal +musterr trace DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY=true $CLI bundle deploy + +title "Destroy clears the deployment. With nothing recorded remotely the bundle is treated as new, so recording can be enabled - and a leftover WAL is still discarded, not replayed" +trace $CLI bundle destroy --auto-approve +printf '{"lineage":"stale","serial":99,"state_version":2}\n' > .databricks/bundle/default/resources.json.wal +trace DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY=true $CLI bundle deploy +trace print_requests.py --dms //api/2.0/bundle --oneline +trace find.py 'resources[.]json[.]wal' --expect 0 +trace jq '{state_version, features}' .databricks/bundle/default/resources.json diff --git a/acceptance/bundle/dms/existing-state/test.toml b/acceptance/bundle/dms/existing-state/test.toml new file mode 100644 index 00000000000..bc95f7e86c4 --- /dev/null +++ b/acceptance/bundle/dms/existing-state/test.toml @@ -0,0 +1,3 @@ +# This test deploys without recording first, then turns it on per-command, so it opts out of +# the parent's DMS=true and drives DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY inline instead. +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/dms/failed-delete/databricks.yml.tmpl b/acceptance/bundle/dms/failed-delete/databricks.yml.tmpl new file mode 100644 index 00000000000..abdb86103ff --- /dev/null +++ b/acceptance/bundle/dms/failed-delete/databricks.yml.tmpl @@ -0,0 +1,9 @@ +bundle: + name: dms-failed-delete-$UNIQUE_NAME +experimental: + record_deployment_history: true + +resources: + jobs: + stuck: + name: stuck diff --git a/acceptance/bundle/dms/failed-delete/out.test.toml b/acceptance/bundle/dms/failed-delete/out.test.toml new file mode 100644 index 00000000000..9921e91a794 --- /dev/null +++ b/acceptance/bundle/dms/failed-delete/out.test.toml @@ -0,0 +1,3 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DMS = ["true"] diff --git a/acceptance/bundle/dms/failed-delete/output.txt b/acceptance/bundle/dms/failed-delete/output.txt new file mode 100644 index 00000000000..701b2051c14 --- /dev/null +++ b/acceptance/bundle/dms/failed-delete/output.txt @@ -0,0 +1,149 @@ + +=== Deploy: the job is recorded +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-failed-delete-[UNIQUE_NAME]/default/files... +Created jobs.stuck +Files: 6 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py --dms //api/2.0/bundle +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments", + "body": { + "display_name": "dms-failed-delete-[UNIQUE_NAME]", + "initial_parent_path": "/Workspace/Users/[USERNAME]/.bundle/dms-failed-delete-[UNIQUE_NAME]/default/state", + "target_name": "default", + "workspace_info": { + "file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-failed-delete-[UNIQUE_NAME]/default/files", + "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-failed-delete-[UNIQUE_NAME]/default" + } + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions", + "q": { + "version_id": "1" + }, + "body": { + "cli_version": "[CLI_VERSION]", + "version_type": "VERSION_TYPE_DEPLOY", + "operations": [ + { + "resource_key": "jobs.stuck", + "action_type": "OPERATION_ACTION_TYPE_CREATE" + } + ] + } +} +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.stuck", + "q": { + "update_mask": "state,error_message,resource_id,status" + }, + "body": { + "error_message": "", + "resource_id": "[NUMID]", + "sequence_id": "0", + "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-failed-delete-[UNIQUE_NAME]/default/state/metadata.json\",\"version_id\":\"1\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"stuck\",\"queue\":{\"enabled\":true}}}", + "status": "OPERATION_STATUS_SUCCEEDED" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/complete", + "body": { + "completion_reason": "VERSION_COMPLETE_SUCCESS" + } +} + +=== A resource that fails to delete is recorded as a failed operation carrying the error, so the history says why rather than showing the resource gone +>>> fault.py POST /api/2.2/jobs/delete 400 0 1 INVALID_PARAMETER_VALUE + +>>> musterr [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.jobs.stuck + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/dms-failed-delete-[UNIQUE_NAME]/default + +Error: cannot delete resources.jobs.stuck: deleting id=[NUMID]: Fault injected by test. (400 INVALID_PARAMETER_VALUE) + +Endpoint: POST [DATABRICKS_URL]/api/2.2/jobs/delete +HTTP Status: 400 Bad Request +API error_code: INVALID_PARAMETER_VALUE +API message: Fault injected by test. + + +>>> print_requests.py --dms //api/2.0/bundle +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions", + "q": { + "version_id": "2" + }, + "body": { + "cli_version": "[CLI_VERSION]", + "version_type": "VERSION_TYPE_DESTROY", + "previous_version_id": "1", + "operations": [ + { + "resource_key": "jobs.stuck", + "action_type": "OPERATION_ACTION_TYPE_DELETE" + } + ] + } +} +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations/jobs.stuck", + "q": { + "update_mask": "state,error_message,resource_id,status" + }, + "body": { + "error_message": "deleting id=[NUMID]: Fault injected by test. (400 INVALID_PARAMETER_VALUE)", + "resource_id": "[NUMID]", + "sequence_id": "0", + "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-failed-delete-[UNIQUE_NAME]/default/state/metadata.json\",\"version_id\":\"1\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"stuck\",\"queue\":{\"enabled\":true}}}", + "status": "OPERATION_STATUS_FAILED" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/complete", + "body": { + "completion_reason": "VERSION_COMPLETE_FAILURE" + } +} + +>>> print_state.py +{ + "state_version": 2, + "cli_version": "[CLI_VERSION]", + "lineage": "[UUID]", + "serial": 1, + "features": { + "record_deployment_history": {} + }, + "state": { + "resources.jobs.stuck": { + "__id__": "[NUMID]", + "state": { + "deployment": { + "deployment_id": "[NUMID]", + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-failed-delete-[UNIQUE_NAME]/default/state/metadata.json", + "version_id": "1" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "stuck", + "queue": { + "enabled": true + } + } + } + } +} diff --git a/acceptance/bundle/dms/failed-delete/script b/acceptance/bundle/dms/failed-delete/script new file mode 100644 index 00000000000..fff56460c21 --- /dev/null +++ b/acceptance/bundle/dms/failed-delete/script @@ -0,0 +1,16 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +title "Deploy: the job is recorded" +trace $CLI bundle deploy +trace print_requests.py --dms //api/2.0/bundle + +title "A resource that fails to delete is recorded as a failed operation carrying the error, so the history says why rather than showing the resource gone" +# The state entry survives too, so the next deploy still knows about the resource. +trace fault.py "POST /api/2.2/jobs/delete" 400 0 1 INVALID_PARAMETER_VALUE +trace musterr $CLI bundle destroy --auto-approve +trace print_requests.py --dms //api/2.0/bundle +trace print_state.py + +# print_state.py asks the service for the resources while recording, so drain those reads instead +# of leaving them behind for the harness to diff. +print_requests.py //api/2.0 > /dev/null diff --git a/acceptance/bundle/dms/failed-delete/test.toml b/acceptance/bundle/dms/failed-delete/test.toml new file mode 100644 index 00000000000..1a22ff04ef4 --- /dev/null +++ b/acceptance/bundle/dms/failed-delete/test.toml @@ -0,0 +1,3 @@ +# Local only: this case needs an injected failure - no job delete that the Jobs API refuses - and fault.py registers +# its rule on the fake, which a cloud run does not have. +Cloud = false diff --git a/acceptance/bundle/dms/failed-recreate/databricks.yml.tmpl b/acceptance/bundle/dms/failed-recreate/databricks.yml.tmpl new file mode 100644 index 00000000000..a2937bf9a41 --- /dev/null +++ b/acceptance/bundle/dms/failed-recreate/databricks.yml.tmpl @@ -0,0 +1,11 @@ +bundle: + name: dms-failed-recreate-$UNIQUE_NAME +experimental: + record_deployment_history: true + +resources: + schemas: + foo: + name: dms_failed_recreate_schema_$UNIQUE_NAME + catalog_name: main + comment: v1 diff --git a/acceptance/bundle/dms/failed-recreate/out.requests.txt b/acceptance/bundle/dms/failed-recreate/out.requests.txt new file mode 100644 index 00000000000..eb169f26a0c --- /dev/null +++ b/acceptance/bundle/dms/failed-recreate/out.requests.txt @@ -0,0 +1,7 @@ +{ + "method": "DELETE", + "path": "/api/2.1/unity-catalog/catalogs/dms_other_[UNIQUE_NAME]", + "q": { + "force": "true" + } +} diff --git a/acceptance/bundle/dms/failed-recreate/out.test.toml b/acceptance/bundle/dms/failed-recreate/out.test.toml new file mode 100644 index 00000000000..9921e91a794 --- /dev/null +++ b/acceptance/bundle/dms/failed-recreate/out.test.toml @@ -0,0 +1,3 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DMS = ["true"] diff --git a/acceptance/bundle/dms/failed-recreate/output.txt b/acceptance/bundle/dms/failed-recreate/output.txt new file mode 100644 index 00000000000..718b8108771 --- /dev/null +++ b/acceptance/bundle/dms/failed-recreate/output.txt @@ -0,0 +1,87 @@ + +=== Deploy the schema, so there is a resource to recreate +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-failed-recreate-[UNIQUE_NAME]/default/files... +Created schemas.foo +Files: 6 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +=== A recreate deletes and then fails to create. The delete went through, so the resource really is gone: the failed operation clears its state (self-contained, not assuming the delete's own write reached the service), rather than describing the resource as it was before the deploy +>>> update_file.py databricks.yml catalog_name: main catalog_name: dms_other_[UNIQUE_NAME] + +>>> fault.py POST /api/2.1/unity-catalog/schemas 400 0 1 INVALID_PARAMETER_VALUE + +>>> musterr [CLI] bundle deploy --auto-approve +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-failed-recreate-[UNIQUE_NAME]/default/files... + +This action will result in the deletion or recreation of the following UC schemas. Any underlying data may be lost: + recreate resources.schemas.foo +Error: cannot recreate resources.schemas.foo: Fault injected by test. (400 INVALID_PARAMETER_VALUE) + +Endpoint: POST [DATABRICKS_URL]/api/2.1/unity-catalog/schemas +HTTP Status: 400 Bad Request +API error_code: INVALID_PARAMETER_VALUE +API message: Fault injected by test. + +Files: 4 uploaded, 0 deleted + +>>> print_requests.py --dms //api/2.0/bundle +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions", + "q": { + "version_id": "2" + }, + "body": { + "cli_version": "[CLI_VERSION]", + "version_type": "VERSION_TYPE_DEPLOY", + "previous_version_id": "1", + "operations": [ + { + "resource_key": "schemas.foo", + "action_type": "OPERATION_ACTION_TYPE_RECREATE" + } + ] + } +} +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations/schemas.foo", + "q": { + "update_mask": "state,error_message,status" + }, + "body": { + "error_message": "", + "sequence_id": "0", + "status": "OPERATION_STATUS_PENDING" + } +} +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations/schemas.foo", + "q": { + "update_mask": "state,error_message,status" + }, + "body": { + "error_message": "Fault injected by test. (400 INVALID_PARAMETER_VALUE)", + "sequence_id": "1", + "status": "OPERATION_STATUS_FAILED" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/complete", + "body": { + "completion_reason": "VERSION_COMPLETE_FAILURE" + } +} + +=== The resource is not listed: state is what projects a resource, and the failed recreate left none +>>> MSYS_NO_PATHCONV=1 [CLI] api get /api/2.0/bundle/deployments/[NUMID]/resources +{} + +=== Planning from DMS state alone creates the schema, which is what has to happen: it no longer exists +>>> [CLI] bundle plan +create schemas.foo + +Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged diff --git a/acceptance/bundle/dms/failed-recreate/script b/acceptance/bundle/dms/failed-recreate/script new file mode 100644 index 00000000000..e2d4a68f508 --- /dev/null +++ b/acceptance/bundle/dms/failed-recreate/script @@ -0,0 +1,32 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +title "Deploy the schema, so there is a resource to recreate" +trace $CLI bundle deploy +rm -f out.requests.txt + +title "A recreate deletes and then fails to create. The delete went through, so the resource really is gone: the failed operation clears its state (self-contained, not assuming the delete's own write reached the service), rather than describing the resource as it was before the deploy" +# Unity Catalog has no catalog called "other", so make one to move the schema into. +other_catalog="dms_other_${UNIQUE_NAME}" +$CLI catalogs create "$other_catalog" &> LOG.catalog +trap '$CLI catalogs delete "$other_catalog" --force >> LOG.catalog 2>&1' EXIT +trace update_file.py databricks.yml "catalog_name: main" "catalog_name: ${other_catalog}" +# Fail only the create that follows the delete; the delete itself still goes through. +trace fault.py "POST /api/2.1/unity-catalog/schemas" 400 0 1 INVALID_PARAMETER_VALUE +trace musterr $CLI bundle deploy --auto-approve +# Not sorted: the point of this test is the order the operations are recorded in. +trace print_requests.py --dms //api/2.0/bundle + +title "The resource is not listed: state is what projects a resource, and the failed recreate left none" +# The deployment ID is the workspace node's ID; read it back the way the CLI does. +# Extracted with python, not jq: the ID exceeds 2^53 and jq 1.6 rounds it. +deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-failed-recreate-${UNIQUE_NAME}/default/state/resources.deployment.json" -o json | python3 -c 'import sys,json; print(json.load(sys.stdin)["object_id"])') +# MSYS_NO_PATHCONV as above: Git Bash on Windows would rewrite the leading-'/' API +# path into C:/Program Files/Git/api/2.0/... before the CLI saw it. +trace MSYS_NO_PATHCONV=1 $CLI api get "/api/2.0/bundle/deployments/${deployment_id}/resources" + +title "Planning from DMS state alone creates the schema, which is what has to happen: it no longer exists" +rm -rf .databricks +trace $CLI bundle plan +# plan probes the state files concurrently, so the recorded order varies. Only the +# operations above are asserted; drop the rest rather than diff a racy order. +rm -f out.requests.txt diff --git a/acceptance/bundle/dms/failed-recreate/test.toml b/acceptance/bundle/dms/failed-recreate/test.toml new file mode 100644 index 00000000000..8a850ba07c5 --- /dev/null +++ b/acceptance/bundle/dms/failed-recreate/test.toml @@ -0,0 +1,5 @@ +# Local only: the failure is injected, and fault.py registers its rule on the fake, which +# a cloud run does not have. Simulating it for real instead - a catalog or principal that +# does not exist - does not work either: the fake accepts both, so the case would stop +# failing locally. Making it portable needs the fake to model that existence. +Cloud = false diff --git a/acceptance/bundle/dms/failed-update-permissions/databricks.yml.tmpl b/acceptance/bundle/dms/failed-update-permissions/databricks.yml.tmpl new file mode 100644 index 00000000000..622598450cd --- /dev/null +++ b/acceptance/bundle/dms/failed-update-permissions/databricks.yml.tmpl @@ -0,0 +1,12 @@ +bundle: + name: dms-failed-update-permissions-$UNIQUE_NAME +experimental: + record_deployment_history: true + +resources: + jobs: + foo: + name: foo + permissions: + - level: CAN_VIEW + user_name: viewer@example.com diff --git a/acceptance/bundle/dms/failed-update-permissions/out.test.toml b/acceptance/bundle/dms/failed-update-permissions/out.test.toml new file mode 100644 index 00000000000..9921e91a794 --- /dev/null +++ b/acceptance/bundle/dms/failed-update-permissions/out.test.toml @@ -0,0 +1,3 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DMS = ["true"] diff --git a/acceptance/bundle/dms/failed-update-permissions/output.txt b/acceptance/bundle/dms/failed-update-permissions/output.txt new file mode 100644 index 00000000000..080f4b6c8be --- /dev/null +++ b/acceptance/bundle/dms/failed-update-permissions/output.txt @@ -0,0 +1,29 @@ + +=== Deploy the job and its permissions +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-failed-update-permissions-[UNIQUE_NAME]/default/files... +Created jobs.foo +Created jobs.foo.permissions +Files: 6 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged + +=== A permissions update that fails records the failure with the permissions it still has. The resource ID here is a path rather than a number, and the state is the embedded permission list, so this covers a shape the resource tests do not +>>> update_file.py databricks.yml CAN_VIEW CAN_MANAGE + +>>> fault.py PUT /api/2.0/permissions/jobs/* 403 0 1 PERMISSION_DENIED + +>>> musterr [CLI] bundle deploy --auto-approve +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-failed-update-permissions-[UNIQUE_NAME]/default/files... +Error: cannot update resources.jobs.foo.permissions: updating id=/jobs/[NUMID]: Fault injected by test. (403 PERMISSION_DENIED) + +Endpoint: PUT [DATABRICKS_URL]/api/2.0/permissions/jobs/[NUMID] +HTTP Status: 403 Forbidden +API error_code: PERMISSION_DENIED +API message: Fault injected by test. + +Files: 3 uploaded, 0 deleted + +>>> print_requests.py --dms //api/2.0/bundle --oneline +{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": {"version_id": "2"}, "body": {"cli_version": "[CLI_VERSION]", "version_type": "VERSION_TYPE_DEPLOY", "previous_version_id": "1", "operations": [{"resource_key": "jobs.foo.permissions", "action_type": "OPERATION_ACTION_TYPE_UPDATE"}]}} +{"method": "PATCH", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations/jobs.foo.permissions", "q": {"update_mask": "state,error_message,resource_id,status"}, "body": {"error_message": "updating id=/jobs/[NUMID]: Fault injected by test. (403 PERMISSION_DENIED)", "resource_id": "/jobs/[NUMID]", "sequence_id": "0", "state": "{\"state\":{\"object_id\":\"/jobs/[NUMID]\",\"__embed__\":[{\"level\":\"CAN_VIEW\",\"user_name\":\"viewer@example.com\"},{\"level\":\"IS_OWNER\",\"user_name\":\"[USERNAME]\"}]},\"depends_on\":[{\"node\":\"resources.jobs.foo\",\"label\":\"${resources.jobs.foo.id}\"}]}", "status": "OPERATION_STATUS_FAILED"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/complete", "body": {"completion_reason": "VERSION_COMPLETE_FAILURE"}} diff --git a/acceptance/bundle/dms/failed-update-permissions/script b/acceptance/bundle/dms/failed-update-permissions/script new file mode 100644 index 00000000000..56141a30f4c --- /dev/null +++ b/acceptance/bundle/dms/failed-update-permissions/script @@ -0,0 +1,12 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +title "Deploy the job and its permissions" +trace $CLI bundle deploy +rm -f out.requests.txt + +title "A permissions update that fails records the failure with the permissions it still has. The resource ID here is a path rather than a number, and the state is the embedded permission list, so this covers a shape the resource tests do not" +trace update_file.py databricks.yml "CAN_VIEW" "CAN_MANAGE" +# A 403 is what the API returns when the caller cannot change a job's owner. +trace fault.py "PUT /api/2.0/permissions/jobs/*" 403 0 1 PERMISSION_DENIED +trace musterr $CLI bundle deploy --auto-approve +trace print_requests.py --dms //api/2.0/bundle --oneline diff --git a/acceptance/bundle/dms/failed-update-permissions/test.toml b/acceptance/bundle/dms/failed-update-permissions/test.toml new file mode 100644 index 00000000000..8a850ba07c5 --- /dev/null +++ b/acceptance/bundle/dms/failed-update-permissions/test.toml @@ -0,0 +1,5 @@ +# Local only: the failure is injected, and fault.py registers its rule on the fake, which +# a cloud run does not have. Simulating it for real instead - a catalog or principal that +# does not exist - does not work either: the fake accepts both, so the case would stop +# failing locally. Making it portable needs the fake to model that existence. +Cloud = false diff --git a/acceptance/bundle/dms/failed-update/databricks.yml.tmpl b/acceptance/bundle/dms/failed-update/databricks.yml.tmpl new file mode 100644 index 00000000000..87cfcff9f36 --- /dev/null +++ b/acceptance/bundle/dms/failed-update/databricks.yml.tmpl @@ -0,0 +1,11 @@ +bundle: + name: dms-failed-update-$UNIQUE_NAME +experimental: + record_deployment_history: true + +resources: + schemas: + foo: + name: dms_failed_update_schema_$UNIQUE_NAME + catalog_name: main + comment: v1 diff --git a/acceptance/bundle/dms/failed-update/out.test.toml b/acceptance/bundle/dms/failed-update/out.test.toml new file mode 100644 index 00000000000..9921e91a794 --- /dev/null +++ b/acceptance/bundle/dms/failed-update/out.test.toml @@ -0,0 +1,3 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DMS = ["true"] diff --git a/acceptance/bundle/dms/failed-update/output.txt b/acceptance/bundle/dms/failed-update/output.txt new file mode 100644 index 00000000000..125c16608b0 --- /dev/null +++ b/acceptance/bundle/dms/failed-update/output.txt @@ -0,0 +1,86 @@ + +=== Deploy the schema, so there is an existing resource to update +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-failed-update-[UNIQUE_NAME]/default/files... +Created schemas.foo +Files: 6 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +=== An update that fails before writing any state only marks its operation failed. It names no state, so the deployment keeps describing the schema the last successful version recorded - which is right, because nothing touched it +>>> update_file.py databricks.yml comment: v1 comment: v2 + +>>> fault.py PATCH /api/2.1/unity-catalog/schemas/* 400 0 1 INVALID_PARAMETER_VALUE + +>>> musterr [CLI] bundle deploy --auto-approve +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-failed-update-[UNIQUE_NAME]/default/files... +Error: cannot update resources.schemas.foo: updating id=main.dms_failed_update_schema_[UNIQUE_NAME]: Fault injected by test. (400 INVALID_PARAMETER_VALUE) + +Endpoint: PATCH [DATABRICKS_URL]/api/2.1/unity-catalog/schemas/main.dms_failed_update_schema_[UNIQUE_NAME] +HTTP Status: 400 Bad Request +API error_code: INVALID_PARAMETER_VALUE +API message: Fault injected by test. + +Files: 3 uploaded, 0 deleted + +>>> print_requests.py --dms //api/2.0/bundle +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions", + "q": { + "version_id": "2" + }, + "body": { + "cli_version": "[CLI_VERSION]", + "version_type": "VERSION_TYPE_DEPLOY", + "previous_version_id": "1", + "operations": [ + { + "resource_key": "schemas.foo", + "action_type": "OPERATION_ACTION_TYPE_UPDATE" + } + ] + } +} +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations/schemas.foo", + "q": { + "update_mask": "state,error_message,resource_id,status" + }, + "body": { + "error_message": "updating id=main.dms_failed_update_schema_[UNIQUE_NAME]: Fault injected by test. (400 INVALID_PARAMETER_VALUE)", + "resource_id": "main.dms_failed_update_schema_[UNIQUE_NAME]", + "sequence_id": "0", + "state": "{\"state\":{\"catalog_name\":\"main\",\"comment\":\"v1\",\"name\":\"dms_failed_update_schema_[UNIQUE_NAME]\"}}", + "status": "OPERATION_STATUS_FAILED" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/complete", + "body": { + "completion_reason": "VERSION_COMPLETE_FAILURE" + } +} + +=== The schema is still listed as the previous version left it, so the next plan updates it rather than creating a second one +>>> MSYS_NO_PATHCONV=1 [CLI] api get /api/2.0/bundle/deployments/[NUMID]/resources +{ + "resources": [ + { + "last_action_type": "OPERATION_ACTION_TYPE_UPDATE", + "last_version_id": "2", + "name": "deployments/[NUMID]/resources/schemas.foo", + "resource_id": "main.dms_failed_update_schema_[UNIQUE_NAME]", + "resource_key": "schemas.foo", + "resource_type": "", + "state": "{\"state\":{\"catalog_name\":\"main\",\"comment\":\"v1\",\"name\":\"dms_failed_update_schema_[UNIQUE_NAME]\"}}" + } + ] +} + +=== Planning from DMS state alone updates the schema rather than creating a second one, which is what recording the prior state buys +>>> [CLI] bundle plan +update schemas.foo + +Plan: 0 to add, 1 to change, 0 to delete, 0 unchanged diff --git a/acceptance/bundle/dms/failed-update/script b/acceptance/bundle/dms/failed-update/script new file mode 100644 index 00000000000..91640bd7631 --- /dev/null +++ b/acceptance/bundle/dms/failed-update/script @@ -0,0 +1,28 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +title "Deploy the schema, so there is an existing resource to update" +trace $CLI bundle deploy +rm -f out.requests.txt + +title "An update that fails before writing any state only marks its operation failed. It names no state, so the deployment keeps describing the schema the last successful version recorded - which is right, because nothing touched it" +trace update_file.py databricks.yml "comment: v1" "comment: v2" +# Fail the update call itself, so the deploy never writes state for the schema. +trace fault.py "PATCH /api/2.1/unity-catalog/schemas/*" 400 0 1 INVALID_PARAMETER_VALUE +trace musterr $CLI bundle deploy --auto-approve +# Not sorted: the point of this test is the order the operations are recorded in. +trace print_requests.py --dms //api/2.0/bundle + +title "The schema is still listed as the previous version left it, so the next plan updates it rather than creating a second one" +# The deployment ID is the workspace node's ID; read it back the way the CLI does. +# Extracted with python, not jq: the ID exceeds 2^53 and jq 1.6 rounds it. +deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-failed-update-${UNIQUE_NAME}/default/state/resources.deployment.json" -o json | python3 -c 'import sys,json; print(json.load(sys.stdin)["object_id"])') +# MSYS_NO_PATHCONV as above: Git Bash on Windows would rewrite the leading-'/' API +# path into C:/Program Files/Git/api/2.0/... before the CLI saw it. +trace MSYS_NO_PATHCONV=1 $CLI api get "/api/2.0/bundle/deployments/${deployment_id}/resources" + +title "Planning from DMS state alone updates the schema rather than creating a second one, which is what recording the prior state buys" +rm -rf .databricks +trace $CLI bundle plan +# plan probes the state files concurrently, so the recorded order varies. Only the +# operations above are asserted; drop the rest rather than diff a racy order. +rm -f out.requests.txt diff --git a/acceptance/bundle/dms/failed-update/test.toml b/acceptance/bundle/dms/failed-update/test.toml new file mode 100644 index 00000000000..2b4eeb1bb04 --- /dev/null +++ b/acceptance/bundle/dms/failed-update/test.toml @@ -0,0 +1,3 @@ +# Local only: this case needs an injected failure - no schema update that Unity Catalog rejects outright - and fault.py registers +# its rule on the fake, which a cloud run does not have. +Cloud = false diff --git a/acceptance/bundle/dms/multiple-resources/databricks.yml.tmpl b/acceptance/bundle/dms/multiple-resources/databricks.yml.tmpl new file mode 100644 index 00000000000..faa6ef5e63c --- /dev/null +++ b/acceptance/bundle/dms/multiple-resources/databricks.yml.tmpl @@ -0,0 +1,17 @@ +bundle: + name: dms-multiple-resources-$UNIQUE_NAME +experimental: + record_deployment_history: true + +resources: + jobs: + one: + name: one + two: + name: two + three: + name: three + four: + name: four + five: + name: five diff --git a/acceptance/bundle/dms/multiple-resources/out.test.toml b/acceptance/bundle/dms/multiple-resources/out.test.toml new file mode 100644 index 00000000000..23c07f70dca --- /dev/null +++ b/acceptance/bundle/dms/multiple-resources/out.test.toml @@ -0,0 +1,3 @@ +Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DMS = ["true"] diff --git a/acceptance/bundle/dms/multiple-resources/output.txt b/acceptance/bundle/dms/multiple-resources/output.txt new file mode 100644 index 00000000000..ecc2092225a --- /dev/null +++ b/acceptance/bundle/dms/multiple-resources/output.txt @@ -0,0 +1,28 @@ + +=== Deploy several resources: operations are uploaded from background workers, so exactly one is recorded per resource no matter what order the uploads finish in. The serialized state is dropped from the output here; bundle/dms/record covers it. +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/files... +Created jobs.five +Created jobs.four +Created jobs.one +Created jobs.three +Created jobs.two +Files: 5 uploaded, 0 deleted +Resources: 5 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py --dms //versions/1/operations --sort --del-body state --oneline +{"method": "PATCH", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.five", "q": {"update_mask": "state,error_message,resource_id,status"}, "body": {"error_message": "", "resource_id": "[NUMID]", "sequence_id": "0", "status": "OPERATION_STATUS_SUCCEEDED"}} +{"method": "PATCH", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.four", "q": {"update_mask": "state,error_message,resource_id,status"}, "body": {"error_message": "", "resource_id": "[NUMID]", "sequence_id": "0", "status": "OPERATION_STATUS_SUCCEEDED"}} +{"method": "PATCH", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.one", "q": {"update_mask": "state,error_message,resource_id,status"}, "body": {"error_message": "", "resource_id": "[NUMID]", "sequence_id": "0", "status": "OPERATION_STATUS_SUCCEEDED"}} +{"method": "PATCH", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.three", "q": {"update_mask": "state,error_message,resource_id,status"}, "body": {"error_message": "", "resource_id": "[NUMID]", "sequence_id": "0", "status": "OPERATION_STATUS_SUCCEEDED"}} +{"method": "PATCH", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.two", "q": {"update_mask": "state,error_message,resource_id,status"}, "body": {"error_message": "", "resource_id": "[NUMID]", "sequence_id": "0", "status": "OPERATION_STATUS_SUCCEEDED"}} + +=== Redeploy with no changes: nothing is applied, so no operations are recorded and only the version is opened and completed +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/files... +Files: 2 uploaded, 0 deleted +Resources: 0 created, 0 changed, 0 deleted, 5 unchanged + +>>> print_requests.py --dms //api/2.0/bundle --sort --oneline +{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": {"version_id": "2"}, "body": {"cli_version": "[CLI_VERSION]", "version_type": "VERSION_TYPE_DEPLOY", "previous_version_id": "1"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/complete", "body": {"completion_reason": "VERSION_COMPLETE_SUCCESS"}} diff --git a/acceptance/bundle/dms/multiple-resources/script b/acceptance/bundle/dms/multiple-resources/script new file mode 100644 index 00000000000..584423a4d5e --- /dev/null +++ b/acceptance/bundle/dms/multiple-resources/script @@ -0,0 +1,9 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +title "Deploy several resources: operations are uploaded from background workers, so exactly one is recorded per resource no matter what order the uploads finish in. The serialized state is dropped from the output here; bundle/dms/record covers it." +trace $CLI bundle deploy +trace print_requests.py --dms //versions/1/operations --sort --del-body state --oneline + +title "Redeploy with no changes: nothing is applied, so no operations are recorded and only the version is opened and completed" +trace $CLI bundle deploy +trace print_requests.py --dms //api/2.0/bundle --sort --oneline diff --git a/acceptance/bundle/dms/no-drift/databricks.yml.tmpl b/acceptance/bundle/dms/no-drift/databricks.yml.tmpl new file mode 100644 index 00000000000..e9892fa79a5 --- /dev/null +++ b/acceptance/bundle/dms/no-drift/databricks.yml.tmpl @@ -0,0 +1,20 @@ +bundle: + name: dms-no-drift-$UNIQUE_NAME +experimental: + record_deployment_history: true + +resources: + jobs: + foo: + name: foo + pipelines: + bar: + # Unique: the Pipelines API refuses a duplicate name, so two overlapping runs on the + # same workspace would collide. Jobs allow duplicates, which is why foo above does not. + name: bar-$UNIQUE_NAME + catalog: main + schema: default + serverless: true + libraries: + - file: + path: ./transform.py diff --git a/acceptance/bundle/dms/no-drift/out.test.toml b/acceptance/bundle/dms/no-drift/out.test.toml new file mode 100644 index 00000000000..23c07f70dca --- /dev/null +++ b/acceptance/bundle/dms/no-drift/out.test.toml @@ -0,0 +1,3 @@ +Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DMS = ["true"] diff --git a/acceptance/bundle/dms/no-drift/output.txt b/acceptance/bundle/dms/no-drift/output.txt new file mode 100644 index 00000000000..7b23792edcd --- /dev/null +++ b/acceptance/bundle/dms/no-drift/output.txt @@ -0,0 +1,126 @@ + +=== Deploy job foo and pipeline bar; both are recorded at version 1 +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-no-drift-[UNIQUE_NAME]/default/files... +Created jobs.foo +Created pipelines.bar +Files: 6 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged + +>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[NUMID] -o json +[ + { + "last_action_type": "OPERATION_ACTION_TYPE_CREATE", + "last_version_id": "1", + "resource_key": "jobs.foo" + }, + { + "last_action_type": "OPERATION_ACTION_TYPE_CREATE", + "last_version_id": "1", + "resource_key": "pipelines.bar" + } +] + +=== Plan without touching anything: the deployment stamp must not show as drift +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged + +=== Update only the pipeline (not the job), then redeploy: only the pipeline's recorded version bumps to 2 +>>> update_file.py databricks.yml serverless: true serverless: true + development: true + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-no-drift-[UNIQUE_NAME]/default/files... +Updated pipelines.bar +Files: 3 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 1 unchanged + +>>> print_requests.py --dms //api/2.2/jobs //api/2.0/pipelines --sort +{ + "method": "POST", + "path": "/api/2.0/pipelines", + "body": { + "catalog": "main", + "channel": "CURRENT", + "deployment": { + "deployment_id": "[NUMID]", + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-no-drift-[UNIQUE_NAME]/default/state/metadata.json", + "version_id": "1" + }, + "edition": "ADVANCED", + "libraries": [ + { + "file": { + "path": "/Workspace/Users/[USERNAME]/.bundle/dms-no-drift-[UNIQUE_NAME]/default/files/transform.py" + } + } + ], + "name": "bar-[UNIQUE_NAME]", + "schema": "default", + "serverless": true + } +} +{ + "method": "POST", + "path": "/api/2.2/jobs/create", + "body": { + "deployment": { + "deployment_id": "[NUMID]", + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-no-drift-[UNIQUE_NAME]/default/state/metadata.json", + "version_id": "1" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "foo", + "queue": { + "enabled": true + } + } +} +{ + "method": "PUT", + "path": "/api/2.0/pipelines/[UUID]", + "body": { + "catalog": "main", + "channel": "CURRENT", + "deployment": { + "deployment_id": "[NUMID]", + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-no-drift-[UNIQUE_NAME]/default/state/metadata.json", + "version_id": "2" + }, + "development": true, + "edition": "ADVANCED", + "libraries": [ + { + "file": { + "path": "/Workspace/Users/[USERNAME]/.bundle/dms-no-drift-[UNIQUE_NAME]/default/files/transform.py" + } + } + ], + "name": "bar-[UNIQUE_NAME]", + "schema": "default", + "serverless": true + } +} + +>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[NUMID] -o json +[ + { + "last_action_type": "OPERATION_ACTION_TYPE_CREATE", + "last_version_id": "1", + "resource_key": "jobs.foo" + }, + { + "last_action_type": "OPERATION_ACTION_TYPE_UPDATE", + "last_version_id": "2", + "resource_key": "pipelines.bar" + } +] + +=== Plan again: no residual drift, and the version stamps are ignored +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged diff --git a/acceptance/bundle/dms/no-drift/script b/acceptance/bundle/dms/no-drift/script new file mode 100644 index 00000000000..8a18d13f258 --- /dev/null +++ b/acceptance/bundle/dms/no-drift/script @@ -0,0 +1,26 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +title "Deploy job foo and pipeline bar; both are recorded at version 1" +trace $CLI bundle deploy + +deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-no-drift-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') +trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' + +# Only deploy stamps deployment.deployment_id. Plan sees it in state but not in config, +# so it must ignore it rather than report spurious changes. +title "Plan without touching anything: the deployment stamp must not show as drift" +trace $CLI bundle plan | contains.py "Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged" + +title "Update only the pipeline (not the job), then redeploy: only the pipeline's recorded version bumps to 2" +trace update_file.py databricks.yml 'serverless: true' 'serverless: true + development: true' +trace $CLI bundle deploy +trace print_requests.py --dms //api/2.2/jobs //api/2.0/pipelines --sort +trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' + +title "Plan again: no residual drift, and the version stamps are ignored" +trace $CLI bundle plan | contains.py "Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged" + +# The list-resources and final plan above record requests we do not assert on; drop them so +# out.requests.txt does not become a nondeterministic golden. +rm -f out.requests.txt diff --git a/acceptance/bundle/dms/no-drift/transform.py b/acceptance/bundle/dms/no-drift/transform.py new file mode 100644 index 00000000000..c07007592a6 --- /dev/null +++ b/acceptance/bundle/dms/no-drift/transform.py @@ -0,0 +1 @@ +# Minimal source so the pipeline has a library, which the API requires. diff --git a/acceptance/bundle/dms/no-resources/databricks.yml.tmpl b/acceptance/bundle/dms/no-resources/databricks.yml.tmpl new file mode 100644 index 00000000000..973189b1603 --- /dev/null +++ b/acceptance/bundle/dms/no-resources/databricks.yml.tmpl @@ -0,0 +1,4 @@ +bundle: + name: dms-no-resources-$UNIQUE_NAME +experimental: + record_deployment_history: true diff --git a/acceptance/bundle/dms/no-resources/out.test.toml b/acceptance/bundle/dms/no-resources/out.test.toml new file mode 100644 index 00000000000..23c07f70dca --- /dev/null +++ b/acceptance/bundle/dms/no-resources/out.test.toml @@ -0,0 +1,3 @@ +Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DMS = ["true"] diff --git a/acceptance/bundle/dms/no-resources/output.txt b/acceptance/bundle/dms/no-resources/output.txt new file mode 100644 index 00000000000..a4f993a04e5 --- /dev/null +++ b/acceptance/bundle/dms/no-resources/output.txt @@ -0,0 +1,80 @@ + +=== First deploy of a bundle with no resources: the deployment is created, and its workspace node identifies it even though no resource state was written +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-no-resources-[UNIQUE_NAME]/default/files... +Files: 5 uploaded, 0 deleted +Resources: 0 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py --dms //api/2.0/bundle --get +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments", + "body": { + "display_name": "dms-no-resources-[UNIQUE_NAME]", + "initial_parent_path": "/Workspace/Users/[USERNAME]/.bundle/dms-no-resources-[UNIQUE_NAME]/default/state", + "target_name": "default", + "workspace_info": { + "file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-no-resources-[UNIQUE_NAME]/default/files", + "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-no-resources-[UNIQUE_NAME]/default" + } + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions", + "q": { + "version_id": "1" + }, + "body": { + "cli_version": "[CLI_VERSION]", + "version_type": "VERSION_TYPE_DEPLOY" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/complete", + "body": { + "completion_reason": "VERSION_COMPLETE_SUCCESS" + } +} + +>>> MSYS_NO_PATHCONV=1 [CLI] workspace get-status /Workspace/Users/[USERNAME]/.bundle/dms-no-resources-[UNIQUE_NAME]/default/state/resources.deployment.json +{ + "object_type": "BUNDLE_DEPLOYMENT", + "path": "/Workspace/Users/[USERNAME]/.bundle/dms-no-resources-[UNIQUE_NAME]/default/state/resources.deployment.json" +} + +=== Redeploy: the deployment is resolved from that node, so no second deployment is created +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-no-resources-[UNIQUE_NAME]/default/files... +Files: 2 uploaded, 0 deleted +Resources: 0 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py --dms //api/2.0/bundle --get +{ + "method": "GET", + "path": "/api/2.0/bundle/deployments/[NUMID]" +} +{ + "method": "GET", + "path": "/api/2.0/bundle/deployments/[NUMID]/resources" +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions", + "q": { + "version_id": "2" + }, + "body": { + "cli_version": "[CLI_VERSION]", + "version_type": "VERSION_TYPE_DEPLOY", + "previous_version_id": "1" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/complete", + "body": { + "completion_reason": "VERSION_COMPLETE_SUCCESS" + } +} diff --git a/acceptance/bundle/dms/no-resources/script b/acceptance/bundle/dms/no-resources/script new file mode 100644 index 00000000000..13924f3664f --- /dev/null +++ b/acceptance/bundle/dms/no-resources/script @@ -0,0 +1,10 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +title "First deploy of a bundle with no resources: the deployment is created, and its workspace node identifies it even though no resource state was written" +trace $CLI bundle deploy +trace print_requests.py --dms //api/2.0/bundle --get +trace MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-no-resources-${UNIQUE_NAME}/default/state/resources.deployment.json" | jq '{object_type,path}' + +title "Redeploy: the deployment is resolved from that node, so no second deployment is created" +trace $CLI bundle deploy +trace print_requests.py --dms //api/2.0/bundle --get diff --git a/acceptance/bundle/dms/operation-upload-fails-once/databricks.yml.tmpl b/acceptance/bundle/dms/operation-upload-fails-once/databricks.yml.tmpl new file mode 100644 index 00000000000..c46cfb03d73 --- /dev/null +++ b/acceptance/bundle/dms/operation-upload-fails-once/databricks.yml.tmpl @@ -0,0 +1,11 @@ +bundle: + name: dms-upload-fails-once-$UNIQUE_NAME +experimental: + record_deployment_history: true + +resources: + schemas: + foo: + name: dms_upload_fails_once_schema_$UNIQUE_NAME + catalog_name: main + comment: v1 diff --git a/acceptance/bundle/dms/operation-upload-fails-once/out.requests.txt b/acceptance/bundle/dms/operation-upload-fails-once/out.requests.txt new file mode 100644 index 00000000000..eb169f26a0c --- /dev/null +++ b/acceptance/bundle/dms/operation-upload-fails-once/out.requests.txt @@ -0,0 +1,7 @@ +{ + "method": "DELETE", + "path": "/api/2.1/unity-catalog/catalogs/dms_other_[UNIQUE_NAME]", + "q": { + "force": "true" + } +} diff --git a/acceptance/bundle/dms/operation-upload-fails-once/out.test.toml b/acceptance/bundle/dms/operation-upload-fails-once/out.test.toml new file mode 100644 index 00000000000..9921e91a794 --- /dev/null +++ b/acceptance/bundle/dms/operation-upload-fails-once/out.test.toml @@ -0,0 +1,3 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DMS = ["true"] diff --git a/acceptance/bundle/dms/operation-upload-fails-once/output.txt b/acceptance/bundle/dms/operation-upload-fails-once/output.txt new file mode 100644 index 00000000000..abdeedeb959 --- /dev/null +++ b/acceptance/bundle/dms/operation-upload-fails-once/output.txt @@ -0,0 +1,79 @@ + +=== Deploy so there is a recorded resource to recreate +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-upload-fails-once-[UNIQUE_NAME]/default/files... +Created schemas.foo +Files: 6 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +=== A recreate records twice, and the first one fails: the second must still carry the sequence id staging left, because a rejected update earned no new one +>>> fault.py PATCH /api/2.0/bundle/* 500 0 1 + +>>> update_file.py databricks.yml catalog_name: main catalog_name: dms_other_[UNIQUE_NAME] + +>>> musterr [CLI] bundle deploy --auto-approve +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-upload-fails-once-[UNIQUE_NAME]/default/files... + +This action will result in the deletion or recreation of the following UC schemas. Any underlying data may be lost: + recreate resources.schemas.foo +Error: recording operation for resources.schemas.foo: Fault injected by test. (500 INJECTED) + +Endpoint: PATCH [DATABRICKS_URL]/api/2.0/bundle/deployments/[NUMID]/versions/2/operations/schemas.foo?update_mask=state%2Cerror_message%2Cstatus +HTTP Status: 500 Internal Server Error +API error_code: INJECTED +API message: Fault injected by test. + +Files: 4 uploaded, 0 deleted + +>>> print_requests.py --dms //api/2.0/bundle +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions", + "q": { + "version_id": "2" + }, + "body": { + "cli_version": "[CLI_VERSION]", + "version_type": "VERSION_TYPE_DEPLOY", + "previous_version_id": "1", + "operations": [ + { + "resource_key": "schemas.foo", + "action_type": "OPERATION_ACTION_TYPE_RECREATE" + } + ] + } +} +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations/schemas.foo", + "q": { + "update_mask": "state,error_message,status" + }, + "body": { + "error_message": "", + "sequence_id": "0", + "status": "OPERATION_STATUS_PENDING" + } +} +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations/schemas.foo", + "q": { + "update_mask": "state,error_message,resource_id,status" + }, + "body": { + "error_message": "", + "resource_id": "dms_other_[UNIQUE_NAME].dms_upload_fails_once_schema_[UNIQUE_NAME]", + "sequence_id": "0", + "state": "{\"state\":{\"catalog_name\":\"dms_other_[UNIQUE_NAME]\",\"comment\":\"v1\",\"name\":\"dms_upload_fails_once_schema_[UNIQUE_NAME]\"}}", + "status": "OPERATION_STATUS_SUCCEEDED" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/complete", + "body": { + "completion_reason": "VERSION_COMPLETE_FAILURE" + } +} diff --git a/acceptance/bundle/dms/operation-upload-fails-once/script b/acceptance/bundle/dms/operation-upload-fails-once/script new file mode 100644 index 00000000000..5b609b1b8ee --- /dev/null +++ b/acceptance/bundle/dms/operation-upload-fails-once/script @@ -0,0 +1,15 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +title "Deploy so there is a recorded resource to recreate" +trace $CLI bundle deploy +rm -f out.requests.txt + +title "A recreate records twice, and the first one fails: the second must still carry the sequence id staging left, because a rejected update earned no new one" +trace fault.py "PATCH /api/2.0/bundle/*" 500 0 1 +# Unity Catalog has no catalog called "other", so make one to move the schema into. +other_catalog="dms_other_${UNIQUE_NAME}" +$CLI catalogs create "$other_catalog" &> LOG.catalog +trap '$CLI catalogs delete "$other_catalog" --force >> LOG.catalog 2>&1' EXIT +trace update_file.py databricks.yml "catalog_name: main" "catalog_name: ${other_catalog}" +trace musterr $CLI bundle deploy --auto-approve +trace print_requests.py --dms //api/2.0/bundle diff --git a/acceptance/bundle/dms/operation-upload-fails-once/test.toml b/acceptance/bundle/dms/operation-upload-fails-once/test.toml new file mode 100644 index 00000000000..5baa9c9d923 --- /dev/null +++ b/acceptance/bundle/dms/operation-upload-fails-once/test.toml @@ -0,0 +1,3 @@ +# Local only: this case needs an injected failure - DMS itself must reject the operation upload, once - and fault.py registers +# its rule on the fake, which a cloud run does not have. +Cloud = false diff --git a/acceptance/bundle/dms/operation-upload-fails/databricks.yml.tmpl b/acceptance/bundle/dms/operation-upload-fails/databricks.yml.tmpl new file mode 100644 index 00000000000..b6aa93265df --- /dev/null +++ b/acceptance/bundle/dms/operation-upload-fails/databricks.yml.tmpl @@ -0,0 +1,23 @@ +bundle: + name: dms-operation-upload-fails-$UNIQUE_NAME +experimental: + record_deployment_history: true + +resources: + jobs: + one: + name: one + two: + name: two + three: + name: three + four: + name: four + five: + name: five + six: + name: six + seven: + name: seven + eight: + name: eight diff --git a/acceptance/bundle/dms/operation-upload-fails/out.test.toml b/acceptance/bundle/dms/operation-upload-fails/out.test.toml new file mode 100644 index 00000000000..9921e91a794 --- /dev/null +++ b/acceptance/bundle/dms/operation-upload-fails/out.test.toml @@ -0,0 +1,3 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DMS = ["true"] diff --git a/acceptance/bundle/dms/operation-upload-fails/output.txt b/acceptance/bundle/dms/operation-upload-fails/output.txt new file mode 100644 index 00000000000..2b273972417 --- /dev/null +++ b/acceptance/bundle/dms/operation-upload-fails/output.txt @@ -0,0 +1,4 @@ + +=== An operation upload failure fails the deploy instead of reporting only at the end +>>> grep -c ^Error: LOG.deploy +deploy reported errors diff --git a/acceptance/bundle/dms/operation-upload-fails/script b/acceptance/bundle/dms/operation-upload-fails/script new file mode 100644 index 00000000000..cc2ead92c6f --- /dev/null +++ b/acceptance/bundle/dms/operation-upload-fails/script @@ -0,0 +1,9 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +title "An operation upload failure fails the deploy instead of reporting only at the end" +# Which resources get refused depends on how far apply got before a background +# upload failed, so the per-resource errors go to a LOG file rather than the diff. +fault.py "PATCH /api/2.0/bundle/*" 500 0 99 INTERNAL_ERROR +errcode $CLI bundle deploy &> LOG.deploy +contains.py 'recording operation for' '!panic' < LOG.deploy > /dev/null +trace grep -c "^Error:" LOG.deploy > /dev/null && echo "deploy reported errors" diff --git a/acceptance/bundle/dms/operation-upload-fails/test.toml b/acceptance/bundle/dms/operation-upload-fails/test.toml new file mode 100644 index 00000000000..b5502a2516c --- /dev/null +++ b/acceptance/bundle/dms/operation-upload-fails/test.toml @@ -0,0 +1,10 @@ +# Local only: this case needs an injected failure - DMS itself must reject the operation upload - and fault.py registers +# its rule on the fake, which a cloud run does not have. +Cloud = false + +# Which requests are made depends on how far apply got before a background upload +# failed, so recording them would make the output nondeterministic. +RecordRequests = false + +# Completed versions make DMS the source of truth; unrecorded resources get recreated. +# Deploy must stop rather than continue creating. diff --git a/acceptance/bundle/dms/operation-upload-message/databricks.yml.tmpl b/acceptance/bundle/dms/operation-upload-message/databricks.yml.tmpl new file mode 100644 index 00000000000..bc5b4ec97ad --- /dev/null +++ b/acceptance/bundle/dms/operation-upload-message/databricks.yml.tmpl @@ -0,0 +1,9 @@ +bundle: + name: dms-operation-upload-message-$UNIQUE_NAME +experimental: + record_deployment_history: true + +resources: + jobs: + foo: + name: foo diff --git a/acceptance/bundle/dms/operation-upload-message/out.test.toml b/acceptance/bundle/dms/operation-upload-message/out.test.toml new file mode 100644 index 00000000000..9921e91a794 --- /dev/null +++ b/acceptance/bundle/dms/operation-upload-message/out.test.toml @@ -0,0 +1,3 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DMS = ["true"] diff --git a/acceptance/bundle/dms/operation-upload-message/output.txt b/acceptance/bundle/dms/operation-upload-message/output.txt new file mode 100644 index 00000000000..0791010b1f1 --- /dev/null +++ b/acceptance/bundle/dms/operation-upload-message/output.txt @@ -0,0 +1,16 @@ + +=== What a user sees when an operation cannot be recorded +>>> fault.py PATCH /api/2.0/bundle/* 500 0 1 INTERNAL_ERROR + +>>> errcode [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-operation-upload-message-[UNIQUE_NAME]/default/files... +Error: recording operation for resources.jobs.foo: Fault injected by test. (500 INTERNAL_ERROR) + +Endpoint: PATCH [DATABRICKS_URL]/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.foo?update_mask=state%2Cerror_message%2Cresource_id%2Cstatus +HTTP Status: 500 Internal Server Error +API error_code: INTERNAL_ERROR +API message: Fault injected by test. + +Files: 5 uploaded, 0 deleted + +Exit code: 1 diff --git a/acceptance/bundle/dms/operation-upload-message/script b/acceptance/bundle/dms/operation-upload-message/script new file mode 100644 index 00000000000..c6b796e4149 --- /dev/null +++ b/acceptance/bundle/dms/operation-upload-message/script @@ -0,0 +1,5 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +title "What a user sees when an operation cannot be recorded" +trace fault.py "PATCH /api/2.0/bundle/*" 500 0 1 INTERNAL_ERROR +trace errcode $CLI bundle deploy diff --git a/acceptance/bundle/dms/operation-upload-message/test.toml b/acceptance/bundle/dms/operation-upload-message/test.toml new file mode 100644 index 00000000000..1dbd55da1f0 --- /dev/null +++ b/acceptance/bundle/dms/operation-upload-message/test.toml @@ -0,0 +1,8 @@ +# Local only: this case needs an injected failure - DMS itself must reject the operation upload - and fault.py registers +# its rule on the fake, which a cloud run does not have. +Cloud = false + +# One resource, so which operation is refused is fixed and the error a user sees can be +# asserted. operation-upload-fails covers a deploy of several resources, where which ones +# were reached before the failure landed is not. +RecordRequests = false diff --git a/acceptance/bundle/dms/provenance/databricks.yml.tmpl b/acceptance/bundle/dms/provenance/databricks.yml.tmpl new file mode 100644 index 00000000000..0a3fa1656ae --- /dev/null +++ b/acceptance/bundle/dms/provenance/databricks.yml.tmpl @@ -0,0 +1,14 @@ +bundle: + name: dms-provenance-$UNIQUE_NAME +experimental: + record_deployment_history: true + +targets: + dev: + default: true + mode: development + +resources: + jobs: + foo: + name: foo diff --git a/acceptance/bundle/dms/provenance/out.test.toml b/acceptance/bundle/dms/provenance/out.test.toml new file mode 100644 index 00000000000..23c07f70dca --- /dev/null +++ b/acceptance/bundle/dms/provenance/out.test.toml @@ -0,0 +1,3 @@ +Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DMS = ["true"] diff --git a/acceptance/bundle/dms/provenance/output.txt b/acceptance/bundle/dms/provenance/output.txt new file mode 100644 index 00000000000..2142dabb700 --- /dev/null +++ b/acceptance/bundle/dms/provenance/output.txt @@ -0,0 +1,68 @@ + +=== Deploying from a git repo records where the source came from: the version carries git_info, workspace_info and the target's deployment_mode +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-provenance-[UNIQUE_NAME]/dev/files... +Created jobs.foo +Files: 6 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py --dms //versions +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions", + "q": { + "version_id": "1" + }, + "body": { + "cli_version": "[CLI_VERSION]", + "version_type": "VERSION_TYPE_DEPLOY", + "git_info": { + "branch": "main", + "commit": "[COMMIT]", + "origin_url": "https://github.com/databricks/bundle-examples.git" + }, + "operations": [ + { + "resource_key": "jobs.foo", + "action_type": "OPERATION_ACTION_TYPE_CREATE" + } + ] + } +} +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.foo", + "q": { + "update_mask": "state,error_message,resource_id,status" + }, + "body": { + "error_message": "", + "resource_id": "[NUMID]", + "sequence_id": "0", + "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-provenance-[UNIQUE_NAME]/dev/state/metadata.json\",\"version_id\":\"1\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":4,\"name\":\"[dev [USERNAME]] foo\",\"queue\":{\"enabled\":true},\"tags\":{\"dev\":\"[USERNAME]\"}}}", + "status": "OPERATION_STATUS_SUCCEEDED" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/complete", + "body": { + "completion_reason": "VERSION_COMPLETE_SUCCESS" + } +} + +=== The service denormalizes them onto the deployment, so a reader gets the provenance of the latest version +>>> MSYS_NO_PATHCONV=1 [CLI] api get /api/2.0/bundle/deployments/[NUMID] +{ + "target_name": "dev", + "deployment_mode": "DEPLOYMENT_MODE_DEVELOPMENT", + "git_info": { + "branch": "main", + "commit": "[COMMIT]", + "origin_url": "https://github.com/databricks/bundle-examples.git" + }, + "workspace_info": { + "file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-provenance-[UNIQUE_NAME]/dev/files", + "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-provenance-[UNIQUE_NAME]/dev" + } +} diff --git a/acceptance/bundle/dms/provenance/script b/acceptance/bundle/dms/provenance/script new file mode 100644 index 00000000000..8b15822fa4b --- /dev/null +++ b/acceptance/bundle/dms/provenance/script @@ -0,0 +1,19 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +title "Deploying from a git repo records where the source came from: the version carries git_info, workspace_info and the target's deployment_mode" +git-repo-init +git remote add origin https://github.com/databricks/bundle-examples.git +trace $CLI bundle deploy +# The commit SHA changes every run, so assert it is a 40-char hex string and drop it. +add_repl.py "$(git rev-parse HEAD)" COMMIT +trace print_requests.py --dms //versions + +title "The service denormalizes them onto the deployment, so a reader gets the provenance of the latest version" +deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-provenance-${UNIQUE_NAME}/dev/state/resources.deployment.json" -o json | python3 -c 'import sys,json; print(json.load(sys.stdin)["object_id"])') +# MSYS_NO_PATHCONV as above: Git Bash on Windows would rewrite the leading-'/' API +# path into C:/Program Files/Git/api/2.0/... before the CLI saw it. +trace MSYS_NO_PATHCONV=1 $CLI api get "/api/2.0/bundle/deployments/${deployment_id}" | jq '{target_name, deployment_mode, git_info, workspace_info}' + +# The deploy uploads files in a nondeterministic order; only the requests above are +# asserted. +rm -f out.requests.txt diff --git a/acceptance/bundle/dms/provenance/test.toml b/acceptance/bundle/dms/provenance/test.toml new file mode 100644 index 00000000000..0a47bfb1b91 --- /dev/null +++ b/acceptance/bundle/dms/provenance/test.toml @@ -0,0 +1,4 @@ +# git-repo-init creates a repo in the test directory so the deploy has git provenance. +Ignore = [ + '.git', +] diff --git a/acceptance/bundle/dms/readplan/databricks.yml.tmpl b/acceptance/bundle/dms/readplan/databricks.yml.tmpl new file mode 100644 index 00000000000..d11621ab1e5 --- /dev/null +++ b/acceptance/bundle/dms/readplan/databricks.yml.tmpl @@ -0,0 +1,19 @@ +bundle: + name: dms-readplan-$UNIQUE_NAME +experimental: + record_deployment_history: true + +resources: + jobs: + foo: + name: foo + pipelines: + bar: + # Unique: the Pipelines API refuses a duplicate name across overlapping runs. + name: bar-$UNIQUE_NAME + catalog: main + schema: default + serverless: true + libraries: + - file: + path: ./transform.py diff --git a/acceptance/bundle/dms/readplan/out.test.toml b/acceptance/bundle/dms/readplan/out.test.toml new file mode 100644 index 00000000000..23c07f70dca --- /dev/null +++ b/acceptance/bundle/dms/readplan/out.test.toml @@ -0,0 +1,3 @@ +Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DMS = ["true"] diff --git a/acceptance/bundle/dms/readplan/output.txt b/acceptance/bundle/dms/readplan/output.txt new file mode 100644 index 00000000000..371855dbcc0 --- /dev/null +++ b/acceptance/bundle/dms/readplan/output.txt @@ -0,0 +1,22 @@ + +=== Save a plan, then deploy from it: the deploy stamps the id and version onto the job and pipeline the plan predates +>>> [CLI] bundle plan -o json + +>>> [CLI] bundle deploy --plan tmp.plan.json +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-readplan-[UNIQUE_NAME]/default/files... +Created jobs.foo +Created pipelines.bar +Files: 7 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged + +=== Re-plan: no drift, the deployed job and pipeline carry the deployment stamp +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged + +=== Both recorded operations carry deployment_id and version_id (lineage), not just kind +>>> print_requests.py --dms //api/2.0/bundle --sort --oneline +{"method": "PATCH", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.foo", "q": {"update_mask": "state,error_message,resource_id,status"}, "body": {"error_message": "", "resource_id": "[NUMID]", "sequence_id": "0", "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-readplan-[UNIQUE_NAME]/default/state/metadata.json\",\"version_id\":\"1\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"foo\",\"queue\":{\"enabled\":true}}}", "status": "OPERATION_STATUS_SUCCEEDED"}} +{"method": "PATCH", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/pipelines.bar", "q": {"update_mask": "state,error_message,resource_id,status"}, "body": {"error_message": "", "resource_id": "[UUID]", "sequence_id": "0", "state": "{\"state\":{\"catalog\":\"main\",\"channel\":\"CURRENT\",\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-readplan-[UNIQUE_NAME]/default/state/metadata.json\",\"version_id\":\"1\"},\"edition\":\"ADVANCED\",\"libraries\":[{\"file\":{\"path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-readplan-[UNIQUE_NAME]/default/files/transform.py\"}}],\"name\":\"bar-[UNIQUE_NAME]\",\"schema\":\"default\",\"serverless\":true}}", "status": "OPERATION_STATUS_SUCCEEDED"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments", "body": {"display_name": "dms-readplan-[UNIQUE_NAME]", "initial_parent_path": "/Workspace/Users/[USERNAME]/.bundle/dms-readplan-[UNIQUE_NAME]/default/state", "target_name": "default", "workspace_info": {"file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-readplan-[UNIQUE_NAME]/default/files", "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-readplan-[UNIQUE_NAME]/default"}}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": {"version_id": "1"}, "body": {"cli_version": "[CLI_VERSION]", "version_type": "VERSION_TYPE_DEPLOY", "operations": [{"resource_key": "jobs.foo", "action_type": "OPERATION_ACTION_TYPE_CREATE"}, {"resource_key": "pipelines.bar", "action_type": "OPERATION_ACTION_TYPE_CREATE"}]}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/complete", "body": {"completion_reason": "VERSION_COMPLETE_SUCCESS"}} diff --git a/acceptance/bundle/dms/readplan/script b/acceptance/bundle/dms/readplan/script new file mode 100644 index 00000000000..40bd110bf00 --- /dev/null +++ b/acceptance/bundle/dms/readplan/script @@ -0,0 +1,13 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +title "Save a plan, then deploy from it: the deploy stamps the id and version onto the job and pipeline the plan predates" +trace $CLI bundle plan -o json > tmp.plan.json +trace $CLI bundle deploy --plan tmp.plan.json + +title "Re-plan: no drift, the deployed job and pipeline carry the deployment stamp" +trace $CLI bundle plan + +title "Both recorded operations carry deployment_id and version_id (lineage), not just kind" +trace print_requests.py --dms //api/2.0/bundle --sort --oneline | contains.py deployment_id version_id + +rm -f tmp.plan.json diff --git a/acceptance/bundle/dms/readplan/transform.py b/acceptance/bundle/dms/readplan/transform.py new file mode 100644 index 00000000000..c07007592a6 --- /dev/null +++ b/acceptance/bundle/dms/readplan/transform.py @@ -0,0 +1 @@ +# Minimal source so the pipeline has a library, which the API requires. diff --git a/acceptance/bundle/dms/record-failure/databricks.yml.tmpl b/acceptance/bundle/dms/record-failure/databricks.yml.tmpl new file mode 100644 index 00000000000..87f0eb4d1a9 --- /dev/null +++ b/acceptance/bundle/dms/record-failure/databricks.yml.tmpl @@ -0,0 +1,9 @@ +bundle: + name: dms-record-failure-$UNIQUE_NAME +experimental: + record_deployment_history: true + +resources: + jobs: + doomed: + name: doomed diff --git a/acceptance/bundle/dms/record-failure/out.test.toml b/acceptance/bundle/dms/record-failure/out.test.toml new file mode 100644 index 00000000000..9921e91a794 --- /dev/null +++ b/acceptance/bundle/dms/record-failure/out.test.toml @@ -0,0 +1,3 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DMS = ["true"] diff --git a/acceptance/bundle/dms/record-failure/output.txt b/acceptance/bundle/dms/record-failure/output.txt new file mode 100644 index 00000000000..479acca7a84 --- /dev/null +++ b/acceptance/bundle/dms/record-failure/output.txt @@ -0,0 +1,75 @@ + +=== A resource that fails to apply is recorded as a failed operation carrying the error, so the deployment history says why rather than omitting the resource +>>> fault.py POST /api/2.2/jobs/create 400 0 1 INVALID_PARAMETER_VALUE + +>>> musterr [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-record-failure-[UNIQUE_NAME]/default/files... +Error: cannot create resources.jobs.doomed: Fault injected by test. (400 INVALID_PARAMETER_VALUE) + +Endpoint: POST [DATABRICKS_URL]/api/2.2/jobs/create +HTTP Status: 400 Bad Request +API error_code: INVALID_PARAMETER_VALUE +API message: Fault injected by test. + +Files: 6 uploaded, 0 deleted + +>>> print_requests.py --dms //api/2.0/bundle +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments", + "body": { + "display_name": "dms-record-failure-[UNIQUE_NAME]", + "initial_parent_path": "/Workspace/Users/[USERNAME]/.bundle/dms-record-failure-[UNIQUE_NAME]/default/state", + "target_name": "default", + "workspace_info": { + "file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-record-failure-[UNIQUE_NAME]/default/files", + "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-record-failure-[UNIQUE_NAME]/default" + } + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions", + "q": { + "version_id": "1" + }, + "body": { + "cli_version": "[CLI_VERSION]", + "version_type": "VERSION_TYPE_DEPLOY", + "operations": [ + { + "resource_key": "jobs.doomed", + "action_type": "OPERATION_ACTION_TYPE_CREATE" + } + ] + } +} +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.doomed", + "q": { + "update_mask": "error_message,status" + }, + "body": { + "error_message": "Fault injected by test. (400 INVALID_PARAMETER_VALUE)", + "sequence_id": "0", + "status": "OPERATION_STATUS_FAILED" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/complete", + "body": { + "completion_reason": "VERSION_COMPLETE_FAILURE" + } +} + +=== The failed resource is not listed at all: state is what projects a resource, and a failed create records none, so a later deploy plans to create it rather than treating it as already deployed +>>> MSYS_NO_PATHCONV=1 [CLI] api get /api/2.0/bundle/deployments/[NUMID]/resources +{} + +=== Redeploying from DMS state alone still creates the failed resource: it was never applied, so it must not be skipped as unchanged +>>> [CLI] bundle plan +create jobs.doomed + +Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged diff --git a/acceptance/bundle/dms/record-failure/script b/acceptance/bundle/dms/record-failure/script new file mode 100644 index 00000000000..bc0947a1d91 --- /dev/null +++ b/acceptance/bundle/dms/record-failure/script @@ -0,0 +1,21 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +title "A resource that fails to apply is recorded as a failed operation carrying the error, so the deployment history says why rather than omitting the resource" +trace fault.py "POST /api/2.2/jobs/create" 400 0 1 INVALID_PARAMETER_VALUE +trace musterr $CLI bundle deploy +trace print_requests.py --dms //api/2.0/bundle + +title "The failed resource is not listed at all: state is what projects a resource, and a failed create records none, so a later deploy plans to create it rather than treating it as already deployed" +# The deployment ID is the workspace node's ID; read it back the way the CLI does. +# Extracted with python, not jq: the ID exceeds 2^53 and jq 1.6 rounds it. +deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-record-failure-${UNIQUE_NAME}/default/state/resources.deployment.json" -o json | python3 -c 'import sys,json; print(json.load(sys.stdin)["object_id"])') +# MSYS_NO_PATHCONV as above: Git Bash on Windows would rewrite the leading-'/' API +# path into C:/Program Files/Git/api/2.0/... before the CLI saw it. +trace MSYS_NO_PATHCONV=1 $CLI api get "/api/2.0/bundle/deployments/${deployment_id}/resources" + +title "Redeploying from DMS state alone still creates the failed resource: it was never applied, so it must not be skipped as unchanged" +rm -rf .databricks +trace $CLI bundle plan +# plan probes the state files concurrently, so the recorded order varies. Only the +# operations above are asserted; drop the rest rather than diff a racy order. +rm -f out.requests.txt diff --git a/acceptance/bundle/dms/record-failure/test.toml b/acceptance/bundle/dms/record-failure/test.toml new file mode 100644 index 00000000000..8a850ba07c5 --- /dev/null +++ b/acceptance/bundle/dms/record-failure/test.toml @@ -0,0 +1,5 @@ +# Local only: the failure is injected, and fault.py registers its rule on the fake, which +# a cloud run does not have. Simulating it for real instead - a catalog or principal that +# does not exist - does not work either: the fake accepts both, so the case would stop +# failing locally. Making it portable needs the fake to model that existence. +Cloud = false diff --git a/acceptance/bundle/dms/record/databricks.yml.tmpl b/acceptance/bundle/dms/record/databricks.yml.tmpl new file mode 100644 index 00000000000..214793eca38 --- /dev/null +++ b/acceptance/bundle/dms/record/databricks.yml.tmpl @@ -0,0 +1,9 @@ +bundle: + name: dms-record-$UNIQUE_NAME +experimental: + record_deployment_history: true + +resources: + jobs: + foo: + name: foo diff --git a/acceptance/bundle/dms/record/out.test.toml b/acceptance/bundle/dms/record/out.test.toml new file mode 100644 index 00000000000..23c07f70dca --- /dev/null +++ b/acceptance/bundle/dms/record/out.test.toml @@ -0,0 +1,3 @@ +Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DMS = ["true"] diff --git a/acceptance/bundle/dms/record/output.txt b/acceptance/bundle/dms/record/output.txt new file mode 100644 index 00000000000..7e63506bf79 --- /dev/null +++ b/acceptance/bundle/dms/record/output.txt @@ -0,0 +1,84 @@ + +=== Deploy, then destroy: set up a deployment and delete it, so the deploy below starts from nothing +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-record-[UNIQUE_NAME]/default/files... +Created jobs.foo +Files: 5 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.jobs.foo + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/dms-record-[UNIQUE_NAME]/default + +Destroy: 1 deleted + +=== Deploy again: the destroy took the workspace node with it, so there is nothing to resolve and a fresh deployment starts at version 1, not a continuation of the deleted one +>>> MSYS_NO_PATHCONV=1 musterr [CLI] workspace get-status /Workspace/Users/[USERNAME]/.bundle/dms-record-[UNIQUE_NAME]/default/state/resources.deployment.json +Error: Path (/Workspace/Users/[USERNAME]/.bundle/dms-record-[UNIQUE_NAME]/default/state/resources.deployment.json) doesn't exist. + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-record-[UNIQUE_NAME]/default/files... +Created jobs.foo +Files: 5 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +>>> MSYS_NO_PATHCONV=1 [CLI] workspace get-status /Workspace/Users/[USERNAME]/.bundle/dms-record-[UNIQUE_NAME]/default/state/resources.deployment.json +{ + "object_type": "BUNDLE_DEPLOYMENT", + "path": "/Workspace/Users/[USERNAME]/.bundle/dms-record-[UNIQUE_NAME]/default/state/resources.deployment.json" +} + +>>> print_requests.py --get --dms //api/2.0/bundle +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments", + "body": { + "display_name": "dms-record-[UNIQUE_NAME]", + "initial_parent_path": "/Workspace/Users/[USERNAME]/.bundle/dms-record-[UNIQUE_NAME]/default/state", + "target_name": "default", + "workspace_info": { + "file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-record-[UNIQUE_NAME]/default/files", + "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-record-[UNIQUE_NAME]/default" + } + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions", + "q": { + "version_id": "1" + }, + "body": { + "cli_version": "[CLI_VERSION]", + "version_type": "VERSION_TYPE_DEPLOY", + "operations": [ + { + "resource_key": "jobs.foo", + "action_type": "OPERATION_ACTION_TYPE_CREATE" + } + ] + } +} +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.foo", + "q": { + "update_mask": "state,error_message,resource_id,status" + }, + "body": { + "error_message": "", + "resource_id": "[NUMID]", + "sequence_id": "0", + "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-record-[UNIQUE_NAME]/default/state/metadata.json\",\"version_id\":\"1\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"foo\",\"queue\":{\"enabled\":true}}}", + "status": "OPERATION_STATUS_SUCCEEDED" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/complete", + "body": { + "completion_reason": "VERSION_COMPLETE_SUCCESS" + } +} diff --git a/acceptance/bundle/dms/record/script b/acceptance/bundle/dms/record/script new file mode 100644 index 00000000000..c521416b158 --- /dev/null +++ b/acceptance/bundle/dms/record/script @@ -0,0 +1,16 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +title "Deploy, then destroy: set up a deployment and delete it, so the deploy below starts from nothing" +trace $CLI bundle deploy +trace $CLI bundle destroy --auto-approve + +# Discard the deploy+destroy requests above; this test is only about the deploy that FOLLOWS a +# destroy. The individual deploy and destroy request sequences are covered by no-resources, +# successful-recreate, and multiple-resources. +rm -f out.requests.txt + +title "Deploy again: the destroy took the workspace node with it, so there is nothing to resolve and a fresh deployment starts at version 1, not a continuation of the deleted one" +trace MSYS_NO_PATHCONV=1 musterr $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-record-${UNIQUE_NAME}/default/state/resources.deployment.json" +trace $CLI bundle deploy +trace MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-record-${UNIQUE_NAME}/default/state/resources.deployment.json" | jq '{object_type,path}' +trace print_requests.py --get --dms //api/2.0/bundle diff --git a/acceptance/bundle/dms/serialized-plan/databricks.yml.tmpl b/acceptance/bundle/dms/serialized-plan/databricks.yml.tmpl new file mode 100644 index 00000000000..01d891c3880 --- /dev/null +++ b/acceptance/bundle/dms/serialized-plan/databricks.yml.tmpl @@ -0,0 +1,9 @@ +bundle: + name: dms-serialized-plan-$UNIQUE_NAME +experimental: + record_deployment_history: true + +resources: + jobs: + foo: + name: foo diff --git a/acceptance/bundle/dms/serialized-plan/out.test.toml b/acceptance/bundle/dms/serialized-plan/out.test.toml new file mode 100644 index 00000000000..23c07f70dca --- /dev/null +++ b/acceptance/bundle/dms/serialized-plan/out.test.toml @@ -0,0 +1,3 @@ +Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DMS = ["true"] diff --git a/acceptance/bundle/dms/serialized-plan/output.txt b/acceptance/bundle/dms/serialized-plan/output.txt new file mode 100644 index 00000000000..fc84deb9f4c --- /dev/null +++ b/acceptance/bundle/dms/serialized-plan/output.txt @@ -0,0 +1,41 @@ + +=== Deploy once to create the deployment (records version 1) +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-serialized-plan-[UNIQUE_NAME]/default/files... +Created jobs.foo +Files: 5 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +=== A saved plan carries the deployment id and the versions it targets (last 1, next 2) +>>> [CLI] bundle plan -o json + +>>> jq {deployment_id, last_version_id, next_version_id} tmp.plan.json +{ + "deployment_id": "[NUMID]", + "last_version_id": "1", + "next_version_id": "2" +} + +=== Change the job and deploy again: advances the deployment to version 2 +>>> update_file.py databricks.yml name: foo name: foo-v2 + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-serialized-plan-[UNIQUE_NAME]/default/files... +Updated jobs.foo +Files: 4 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 0 unchanged + +=== Replaying the earlier plan is rejected: it predates the deployment's current version +>>> musterr [CLI] bundle deploy --plan tmp.plan.json +Error: this plan predates the deployment's current version 2; run 'bundle plan' again + + +=== Only versions 1 and 2 were recorded; the rejected replay created none +>>> print_requests.py --dms //api/2.0/bundle --oneline +{"method": "POST", "path": "/api/2.0/bundle/deployments", "body": {"display_name": "dms-serialized-plan-[UNIQUE_NAME]", "initial_parent_path": "/Workspace/Users/[USERNAME]/.bundle/dms-serialized-plan-[UNIQUE_NAME]/default/state", "target_name": "default", "workspace_info": {"file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-serialized-plan-[UNIQUE_NAME]/default/files", "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-serialized-plan-[UNIQUE_NAME]/default"}}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": {"version_id": "1"}, "body": {"cli_version": "[CLI_VERSION]", "version_type": "VERSION_TYPE_DEPLOY", "operations": [{"resource_key": "jobs.foo", "action_type": "OPERATION_ACTION_TYPE_CREATE"}]}} +{"method": "PATCH", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.foo", "q": {"update_mask": "state,error_message,resource_id,status"}, "body": {"error_message": "", "resource_id": "[NUMID]", "sequence_id": "0", "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-serialized-plan-[UNIQUE_NAME]/default/state/metadata.json\",\"version_id\":\"1\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"foo\",\"queue\":{\"enabled\":true}}}", "status": "OPERATION_STATUS_SUCCEEDED"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/complete", "body": {"completion_reason": "VERSION_COMPLETE_SUCCESS"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": {"version_id": "2"}, "body": {"cli_version": "[CLI_VERSION]", "version_type": "VERSION_TYPE_DEPLOY", "previous_version_id": "1", "operations": [{"resource_key": "jobs.foo", "action_type": "OPERATION_ACTION_TYPE_UPDATE"}]}} +{"method": "PATCH", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations/jobs.foo", "q": {"update_mask": "state,error_message,resource_id,status"}, "body": {"error_message": "", "resource_id": "[NUMID]", "sequence_id": "0", "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-serialized-plan-[UNIQUE_NAME]/default/state/metadata.json\",\"version_id\":\"2\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"foo-v2\",\"queue\":{\"enabled\":true}}}", "status": "OPERATION_STATUS_SUCCEEDED"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/complete", "body": {"completion_reason": "VERSION_COMPLETE_SUCCESS"}} diff --git a/acceptance/bundle/dms/serialized-plan/script b/acceptance/bundle/dms/serialized-plan/script new file mode 100644 index 00000000000..0800e1a8265 --- /dev/null +++ b/acceptance/bundle/dms/serialized-plan/script @@ -0,0 +1,20 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +title "Deploy once to create the deployment (records version 1)" +trace $CLI bundle deploy + +title "A saved plan carries the deployment id and the versions it targets (last 1, next 2)" +trace $CLI bundle plan -o json > tmp.plan.json +trace jq '{deployment_id, last_version_id, next_version_id}' tmp.plan.json + +title "Change the job and deploy again: advances the deployment to version 2" +trace update_file.py databricks.yml "name: foo" "name: foo-v2" +trace $CLI bundle deploy + +title "Replaying the earlier plan is rejected: it predates the deployment's current version" +trace musterr $CLI bundle deploy --plan tmp.plan.json + +title "Only versions 1 and 2 were recorded; the rejected replay created none" +trace print_requests.py --dms //api/2.0/bundle --oneline + +rm -f tmp.plan.json diff --git a/acceptance/bundle/dms/stale-plan/databricks.yml.tmpl b/acceptance/bundle/dms/stale-plan/databricks.yml.tmpl new file mode 100644 index 00000000000..dea8fbd9a8d --- /dev/null +++ b/acceptance/bundle/dms/stale-plan/databricks.yml.tmpl @@ -0,0 +1,9 @@ +bundle: + name: dms-stale-plan-$UNIQUE_NAME +experimental: + record_deployment_history: true + +resources: + jobs: + foo: + name: foo diff --git a/acceptance/bundle/dms/stale-plan/out.test.toml b/acceptance/bundle/dms/stale-plan/out.test.toml new file mode 100644 index 00000000000..23c07f70dca --- /dev/null +++ b/acceptance/bundle/dms/stale-plan/out.test.toml @@ -0,0 +1,3 @@ +Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DMS = ["true"] diff --git a/acceptance/bundle/dms/stale-plan/output.txt b/acceptance/bundle/dms/stale-plan/output.txt new file mode 100644 index 00000000000..2ce3b598d1c --- /dev/null +++ b/acceptance/bundle/dms/stale-plan/output.txt @@ -0,0 +1,21 @@ + +=== Save a plan, then deploy it: records version 1 +>>> [CLI] bundle plan -o json + +>>> [CLI] bundle deploy --plan tmp.plan.json +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-stale-plan-[UNIQUE_NAME]/default/files... +Created jobs.foo +Files: 6 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +=== Replay: re-deploying the same plan is rejected - it predates the recorded version +>>> musterr [CLI] bundle deploy --plan tmp.plan.json +Error: this plan predates the deployment's current version 1; run 'bundle plan' again + + +=== Only the first deploy recorded a version; the rejected replay created none +>>> print_requests.py --dms //api/2.0/bundle --oneline +{"method": "POST", "path": "/api/2.0/bundle/deployments", "body": {"display_name": "dms-stale-plan-[UNIQUE_NAME]", "initial_parent_path": "/Workspace/Users/[USERNAME]/.bundle/dms-stale-plan-[UNIQUE_NAME]/default/state", "target_name": "default", "workspace_info": {"file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-stale-plan-[UNIQUE_NAME]/default/files", "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-stale-plan-[UNIQUE_NAME]/default"}}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": {"version_id": "1"}, "body": {"cli_version": "[CLI_VERSION]", "version_type": "VERSION_TYPE_DEPLOY", "operations": [{"resource_key": "jobs.foo", "action_type": "OPERATION_ACTION_TYPE_CREATE"}]}} +{"method": "PATCH", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.foo", "q": {"update_mask": "state,error_message,resource_id,status"}, "body": {"error_message": "", "resource_id": "[NUMID]", "sequence_id": "0", "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-stale-plan-[UNIQUE_NAME]/default/state/metadata.json\",\"version_id\":\"1\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"foo\",\"queue\":{\"enabled\":true}}}", "status": "OPERATION_STATUS_SUCCEEDED"}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/complete", "body": {"completion_reason": "VERSION_COMPLETE_SUCCESS"}} diff --git a/acceptance/bundle/dms/stale-plan/script b/acceptance/bundle/dms/stale-plan/script new file mode 100644 index 00000000000..1f9e792f30f --- /dev/null +++ b/acceptance/bundle/dms/stale-plan/script @@ -0,0 +1,13 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +title "Save a plan, then deploy it: records version 1" +trace $CLI bundle plan -o json > tmp.plan.json +trace $CLI bundle deploy --plan tmp.plan.json + +title "Replay: re-deploying the same plan is rejected - it predates the recorded version" +trace musterr $CLI bundle deploy --plan tmp.plan.json + +title "Only the first deploy recorded a version; the rejected replay created none" +trace print_requests.py --dms //api/2.0/bundle --oneline + +rm -f tmp.plan.json diff --git a/acceptance/bundle/dms/successful-recreate/databricks.yml.tmpl b/acceptance/bundle/dms/successful-recreate/databricks.yml.tmpl new file mode 100644 index 00000000000..8a40cabeda6 --- /dev/null +++ b/acceptance/bundle/dms/successful-recreate/databricks.yml.tmpl @@ -0,0 +1,11 @@ +bundle: + name: dms-successful-recreate-$UNIQUE_NAME +experimental: + record_deployment_history: true + +resources: + schemas: + foo: + name: dms_successful_recreate_schema_$UNIQUE_NAME + catalog_name: main + comment: v1 diff --git a/acceptance/bundle/dms/successful-recreate/out.requests.txt b/acceptance/bundle/dms/successful-recreate/out.requests.txt new file mode 100644 index 00000000000..eb169f26a0c --- /dev/null +++ b/acceptance/bundle/dms/successful-recreate/out.requests.txt @@ -0,0 +1,7 @@ +{ + "method": "DELETE", + "path": "/api/2.1/unity-catalog/catalogs/dms_other_[UNIQUE_NAME]", + "q": { + "force": "true" + } +} diff --git a/acceptance/bundle/dms/successful-recreate/out.test.toml b/acceptance/bundle/dms/successful-recreate/out.test.toml new file mode 100644 index 00000000000..23c07f70dca --- /dev/null +++ b/acceptance/bundle/dms/successful-recreate/out.test.toml @@ -0,0 +1,3 @@ +Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DMS = ["true"] diff --git a/acceptance/bundle/dms/successful-recreate/output.txt b/acceptance/bundle/dms/successful-recreate/output.txt new file mode 100644 index 00000000000..4d497d81f2b --- /dev/null +++ b/acceptance/bundle/dms/successful-recreate/output.txt @@ -0,0 +1,180 @@ + +=== Deploy: the state write records the resource +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-successful-recreate-[UNIQUE_NAME]/default/files... +Created schemas.foo +Files: 5 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py --dms //api/2.0/bundle +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments", + "body": { + "display_name": "dms-successful-recreate-[UNIQUE_NAME]", + "initial_parent_path": "/Workspace/Users/[USERNAME]/.bundle/dms-successful-recreate-[UNIQUE_NAME]/default/state", + "target_name": "default", + "workspace_info": { + "file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-successful-recreate-[UNIQUE_NAME]/default/files", + "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-successful-recreate-[UNIQUE_NAME]/default" + } + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions", + "q": { + "version_id": "1" + }, + "body": { + "cli_version": "[CLI_VERSION]", + "version_type": "VERSION_TYPE_DEPLOY", + "operations": [ + { + "resource_key": "schemas.foo", + "action_type": "OPERATION_ACTION_TYPE_CREATE" + } + ] + } +} +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/schemas.foo", + "q": { + "update_mask": "state,error_message,resource_id,status" + }, + "body": { + "error_message": "", + "resource_id": "main.dms_successful_recreate_schema_[UNIQUE_NAME]", + "sequence_id": "0", + "state": "{\"state\":{\"catalog_name\":\"main\",\"comment\":\"v1\",\"name\":\"dms_successful_recreate_schema_[UNIQUE_NAME]\"}}", + "status": "OPERATION_STATUS_SUCCEEDED" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/complete", + "body": { + "completion_reason": "VERSION_COMPLETE_SUCCESS" + } +} + +=== Recreate writes state twice - the entry is dropped, then the new resource is saved +>>> update_file.py databricks.yml catalog_name: main catalog_name: dms_other_[UNIQUE_NAME] + +>>> [CLI] bundle deploy --auto-approve +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-successful-recreate-[UNIQUE_NAME]/default/files... + +This action will result in the deletion or recreation of the following UC schemas. Any underlying data may be lost: + recreate resources.schemas.foo +Recreated schemas.foo +Files: 4 uploaded, 0 deleted +Resources: 1 created, 0 changed, 1 deleted, 0 unchanged + +>>> print_requests.py --dms //api/2.0/bundle +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions", + "q": { + "version_id": "2" + }, + "body": { + "cli_version": "[CLI_VERSION]", + "version_type": "VERSION_TYPE_DEPLOY", + "previous_version_id": "1", + "operations": [ + { + "resource_key": "schemas.foo", + "action_type": "OPERATION_ACTION_TYPE_RECREATE" + } + ] + } +} +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations/schemas.foo", + "q": { + "update_mask": "state,error_message,status" + }, + "body": { + "error_message": "", + "sequence_id": "0", + "status": "OPERATION_STATUS_PENDING" + } +} +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations/schemas.foo", + "q": { + "update_mask": "state,error_message,resource_id,status" + }, + "body": { + "error_message": "", + "resource_id": "dms_other_[UNIQUE_NAME].dms_successful_recreate_schema_[UNIQUE_NAME]", + "sequence_id": "1", + "state": "{\"state\":{\"catalog_name\":\"dms_other_[UNIQUE_NAME]\",\"comment\":\"v1\",\"name\":\"dms_successful_recreate_schema_[UNIQUE_NAME]\"}}", + "status": "OPERATION_STATUS_SUCCEEDED" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/complete", + "body": { + "completion_reason": "VERSION_COMPLETE_SUCCESS" + } +} + +=== Destroy: the delete is recorded with the id and no state +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.schemas.foo + +This action will result in the deletion of the following UC schemas. Any underlying data may be lost: + delete resources.schemas.foo + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/dms-successful-recreate-[UNIQUE_NAME]/default + +Destroy: 1 deleted + +>>> print_requests.py --dms //api/2.0/bundle +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions", + "q": { + "version_id": "3" + }, + "body": { + "cli_version": "[CLI_VERSION]", + "version_type": "VERSION_TYPE_DESTROY", + "previous_version_id": "2", + "operations": [ + { + "resource_key": "schemas.foo", + "action_type": "OPERATION_ACTION_TYPE_DELETE" + } + ] + } +} +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/3/operations/schemas.foo", + "q": { + "update_mask": "state,error_message,status" + }, + "body": { + "error_message": "", + "sequence_id": "0", + "status": "OPERATION_STATUS_SUCCEEDED" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/3/complete", + "body": { + "completion_reason": "VERSION_COMPLETE_SUCCESS" + } +} +{ + "method": "DELETE", + "path": "/api/2.0/bundle/deployments/[NUMID]" +} diff --git a/acceptance/bundle/dms/successful-recreate/script b/acceptance/bundle/dms/successful-recreate/script new file mode 100644 index 00000000000..43050cc7c8f --- /dev/null +++ b/acceptance/bundle/dms/successful-recreate/script @@ -0,0 +1,20 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +title "Deploy: the state write records the resource" +trace $CLI bundle deploy +trace print_requests.py --dms //api/2.0/bundle + +title "Recreate writes state twice - the entry is dropped, then the new resource is saved" +# One operation per resource per version; both writes land on the same one. +# The drop opens it IN_PROGRESS, the save patches it to SUCCEEDED. +# Unity Catalog has no catalog called "other", so make one to move the schema into. +other_catalog="dms_other_${UNIQUE_NAME}" +$CLI catalogs create "$other_catalog" &> LOG.catalog +trap '$CLI catalogs delete "$other_catalog" --force >> LOG.catalog 2>&1' EXIT +trace update_file.py databricks.yml "catalog_name: main" "catalog_name: ${other_catalog}" +trace $CLI bundle deploy --auto-approve +trace print_requests.py --dms //api/2.0/bundle + +title "Destroy: the delete is recorded with the id and no state" +trace $CLI bundle destroy --auto-approve +trace print_requests.py --dms //api/2.0/bundle diff --git a/acceptance/bundle/dms/summary/databricks.yml.tmpl b/acceptance/bundle/dms/summary/databricks.yml.tmpl new file mode 100644 index 00000000000..1c2fa3c7f20 --- /dev/null +++ b/acceptance/bundle/dms/summary/databricks.yml.tmpl @@ -0,0 +1,9 @@ +bundle: + name: dms-summary-$UNIQUE_NAME +experimental: + record_deployment_history: true + +resources: + jobs: + foo: + name: foo diff --git a/acceptance/bundle/dms/summary/out.test.toml b/acceptance/bundle/dms/summary/out.test.toml new file mode 100644 index 00000000000..23c07f70dca --- /dev/null +++ b/acceptance/bundle/dms/summary/out.test.toml @@ -0,0 +1,3 @@ +Cloud = true +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DMS = ["true"] diff --git a/acceptance/bundle/dms/summary/output.txt b/acceptance/bundle/dms/summary/output.txt new file mode 100644 index 00000000000..233f2416dc3 --- /dev/null +++ b/acceptance/bundle/dms/summary/output.txt @@ -0,0 +1,37 @@ + +=== Summary reports the deployment recorded with the metadata service, so a caller can find the deployment and the version it is on +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-summary-[UNIQUE_NAME]/default/files... +Created jobs.foo +Files: 5 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +>>> [CLI] bundle summary -o json +{ + "deployment_id": "[NUMID]", + "latest_version_id": "1" +} + +=== Redeploying advances the version the summary reports +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-summary-[UNIQUE_NAME]/default/files... +Files: 2 uploaded, 0 deleted +Resources: 0 created, 0 changed, 0 deleted, 1 unchanged + +>>> [CLI] bundle summary -o json +{ + "deployment_id": "[NUMID]", + "latest_version_id": "2" +} + +=== After a destroy the deployment is gone, so the summary reports no history rather than a dangling ID +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.jobs.foo + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/dms-summary-[UNIQUE_NAME]/default + +Destroy: 1 deleted + +>>> [CLI] bundle summary -o json +false diff --git a/acceptance/bundle/dms/summary/script b/acceptance/bundle/dms/summary/script new file mode 100644 index 00000000000..75ec8612ffe --- /dev/null +++ b/acceptance/bundle/dms/summary/script @@ -0,0 +1,17 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +title "Summary reports the deployment recorded with the metadata service, so a caller can find the deployment and the version it is on" +trace $CLI bundle deploy +trace $CLI bundle summary -o json | jq .bundle.deployment.history + +title "Redeploying advances the version the summary reports" +trace $CLI bundle deploy +trace $CLI bundle summary -o json | jq .bundle.deployment.history + +title "After a destroy the deployment is gone, so the summary reports no history rather than a dangling ID" +trace $CLI bundle destroy --auto-approve +trace $CLI bundle summary -o json | jq '.bundle.deployment | has("history")' + +# This test asserts the summary output, not the requests behind it; the file uploads +# recorded here are ordered nondeterministically. +rm -f out.requests.txt diff --git a/acceptance/bundle/dms/test.toml b/acceptance/bundle/dms/test.toml new file mode 100644 index 00000000000..32a52823ff7 --- /dev/null +++ b/acceptance/bundle/dms/test.toml @@ -0,0 +1,19 @@ +Cloud = true + +# Deployment Metadata Service (DMS) recording is only supported by the direct +# engine; it is a no-op on terraform. +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] + + +RecordRequests = true + +Ignore = [ + '.databricks', + 'databricks.yml', +] + +# These tests set experimental.record_deployment_history in databricks.yml; the DMS env var +# both records and permits that otherwise-gated field. bundle/dms/not-supported covers the +# rejection path. Pinned to "true" (not the suite's ["", "true"]) since the field already +# records - the empty variant would just duplicate it. +EnvMatrix.DMS = ["true"] diff --git a/acceptance/bundle/dms/version-never-created/databricks.yml b/acceptance/bundle/dms/version-never-created/databricks.yml new file mode 100644 index 00000000000..a7077f18b1f --- /dev/null +++ b/acceptance/bundle/dms/version-never-created/databricks.yml @@ -0,0 +1,10 @@ +bundle: + name: dms-version-never-created + +experimental: + record_deployment_history: true + +resources: + jobs: + foo: + name: foo diff --git a/acceptance/bundle/dms/version-never-created/out.test.toml b/acceptance/bundle/dms/version-never-created/out.test.toml new file mode 100644 index 00000000000..9921e91a794 --- /dev/null +++ b/acceptance/bundle/dms/version-never-created/out.test.toml @@ -0,0 +1,3 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DMS = ["true"] diff --git a/acceptance/bundle/dms/version-never-created/output.txt b/acceptance/bundle/dms/version-never-created/output.txt new file mode 100644 index 00000000000..8a7e5fe8506 --- /dev/null +++ b/acceptance/bundle/dms/version-never-created/output.txt @@ -0,0 +1,38 @@ + +=== The first version fails, so no deployment record exists - only the node naming its ID +>>> musterr [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-version-never-created/default/files... +Error: failed to create deployment version: Internal error (500 INTERNAL_ERROR) + +Endpoint: POST [DATABRICKS_URL]/api/2.0/bundle/deployments/[NUMID]/versions?version_id=1 +HTTP Status: 500 Internal Server Error +API error_code: INTERNAL_ERROR +API message: Internal error + +Files: 5 uploaded, 0 deleted + +>>> MSYS_NO_PATHCONV=1 [CLI] workspace get-status /Workspace/Users/[USERNAME]/.bundle/dms-version-never-created/default/state/resources.deployment.json +{ + "object_type": "BUNDLE_DEPLOYMENT" +} + +>>> print_requests.py --dms //api/2.0/bundle --get --oneline +{"method": "POST", "path": "/api/2.0/bundle/deployments", "body": {"display_name": "dms-version-never-created", "initial_parent_path": "/Workspace/Users/[USERNAME]/.bundle/dms-version-never-created/default/state", "target_name": "default", "workspace_info": {"file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-version-never-created/default/files", "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-version-never-created/default"}}} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": {"version_id": "1"}, "body": {"cli_version": "[CLI_VERSION]", "version_type": "VERSION_TYPE_DEPLOY", "operations": [{"resource_key": "jobs.foo", "action_type": "OPERATION_ACTION_TYPE_CREATE"}]}} + +=== The next deploy reuses the ID that node names and retries version 1, rather than creating a second deployment +>>> musterr [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-version-never-created/default/files... +Error: failed to create deployment version: Internal error (500 INTERNAL_ERROR) + +Endpoint: POST [DATABRICKS_URL]/api/2.0/bundle/deployments/[NUMID]/versions?version_id=1 +HTTP Status: 500 Internal Server Error +API error_code: INTERNAL_ERROR +API message: Internal error + +Files: 2 uploaded, 0 deleted + +>>> print_requests.py --dms //api/2.0/bundle --get --oneline +{"method": "GET", "path": "/api/2.0/bundle/deployments/[NUMID]"} +{"method": "GET", "path": "/api/2.0/bundle/deployments/[NUMID]/resources"} +{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": {"version_id": "1"}, "body": {"cli_version": "[CLI_VERSION]", "version_type": "VERSION_TYPE_DEPLOY", "operations": [{"resource_key": "jobs.foo", "action_type": "OPERATION_ACTION_TYPE_CREATE"}]}} diff --git a/acceptance/bundle/dms/version-never-created/script b/acceptance/bundle/dms/version-never-created/script new file mode 100644 index 00000000000..58178feaf53 --- /dev/null +++ b/acceptance/bundle/dms/version-never-created/script @@ -0,0 +1,8 @@ +title "The first version fails, so no deployment record exists - only the node naming its ID" +trace musterr $CLI bundle deploy +trace MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-version-never-created/default/state/resources.deployment.json" | jq '{object_type}' +trace print_requests.py --dms //api/2.0/bundle --get --oneline + +title "The next deploy reuses the ID that node names and retries version 1, rather than creating a second deployment" +trace musterr $CLI bundle deploy +trace print_requests.py --dms //api/2.0/bundle --get --oneline diff --git a/acceptance/bundle/dms/version-never-created/test.toml b/acceptance/bundle/dms/version-never-created/test.toml new file mode 100644 index 00000000000..6553e7c09bb --- /dev/null +++ b/acceptance/bundle/dms/version-never-created/test.toml @@ -0,0 +1,12 @@ +# Local only: the failure has to land on CreateVersion and not on the CreateDeployment +# that precedes it, which fault.py's trailing-* pattern cannot express, so this case +# keeps a [[Server]] stub - and the cloud harness ignores stubs. +Cloud = false + +# The first version fails, so the deployment record is never created - only the +# workspace node CreateDeployment registered. The next deploy resolves the ID from +# that node and has to cope with a deployment that has no record yet. +[[Server]] +Pattern = "POST /api/2.0/bundle/deployments/{deployment_id}/versions" +Response.StatusCode = 500 +Response.Body = '''{"error_code": "INTERNAL_ERROR", "message": "Internal error"}''' diff --git a/acceptance/bundle/empty_string_dropped/script b/acceptance/bundle/empty_string_dropped/script index a03c479c071..01ed00ce42f 100644 --- a/acceptance/bundle/empty_string_dropped/script +++ b/acceptance/bundle/empty_string_dropped/script @@ -8,7 +8,7 @@ # Resolved config the engines see. Today every "" field is still present here; a # fix in the initialize phase would drop them, and this golden would show that. -$CLI bundle validate -o json -t direct | nostamp | jq .resources > out.validate.json +$CLI bundle validate -o json -t direct | nostamp.py | jq .resources > out.validate.json # Exclude non-create traffic: workspace file ops, telemetry (nondeterministic), and the # deployment history calls the DMS run adds. diff --git a/acceptance/bundle/environments/dependencies/script b/acceptance/bundle/environments/dependencies/script index 00e5fc846cc..77c215da892 100644 --- a/acceptance/bundle/environments/dependencies/script +++ b/acceptance/bundle/environments/dependencies/script @@ -4,11 +4,11 @@ set -euo pipefail trace $CLI bundle validate trace $CLI bundle deploy -trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body.environments' out.requests.txt | nostamp +trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body.environments' out.requests.txt | nostamp.py -trace jq -s '.[] | select(.path=="/api/2.0/pipelines")' out.requests.txt | nostamp +trace jq -s '.[] | select(.path=="/api/2.0/pipelines")' out.requests.txt | nostamp.py -trace $CLI bundle validate -o json | nostamp | jq '.resources.jobs.test_job.environments' -trace $CLI bundle validate -o json | nostamp | jq '.resources.pipelines.test_pipeline.environment' +trace $CLI bundle validate -o json | nostamp.py | jq '.resources.jobs.test_job.environments' +trace $CLI bundle validate -o json | nostamp.py | jq '.resources.pipelines.test_pipeline.environment' rm out.requests.txt diff --git a/acceptance/bundle/escaped_refs/out.plan.direct.json b/acceptance/bundle/escaped_refs/out.plan.direct.json index ae4fd743d26..a64f09b477b 100644 --- a/acceptance/bundle/escaped_refs/out.plan.direct.json +++ b/acceptance/bundle/escaped_refs/out.plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.jobs.example_ingestion_job": { diff --git a/acceptance/bundle/escaped_refs/out.test.toml b/acceptance/bundle/escaped_refs/out.test.toml index bf4f0e7d3c6..d5b9a6f4a16 100644 --- a/acceptance/bundle/escaped_refs/out.test.toml +++ b/acceptance/bundle/escaped_refs/out.test.toml @@ -1,4 +1,4 @@ Cloud = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] -EnvMatrix.DMS = ["", "true"] +EnvMatrix.DMS = [""] EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/escaped_refs/output.txt b/acceptance/bundle/escaped_refs/output.txt index 6e12dbfd8fc..342535d0037 100644 --- a/acceptance/bundle/escaped_refs/output.txt +++ b/acceptance/bundle/escaped_refs/output.txt @@ -15,7 +15,7 @@ Created jobs.example_ingestion_job Files: 6 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged ->>> print_requests.py //jobs +>>> print_requests.py //jobs --nostamp { "method": "POST", "path": "/api/2.2/jobs/create", diff --git a/acceptance/bundle/escaped_refs/script b/acceptance/bundle/escaped_refs/script index 34640492656..1b8986539c6 100644 --- a/acceptance/bundle/escaped_refs/script +++ b/acceptance/bundle/escaped_refs/script @@ -12,10 +12,12 @@ title "Validate: escapes are still escaped in the config" trace $CLI bundle validate -o json | jq '.resources.jobs.example_ingestion_job.tasks[0].notebook_task.base_parameters' title "Plan (json)" -trace $CLI bundle plan -o json > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.plan.$DATABRICKS_BUNDLE_ENGINE.json # Not traced: the READPLAN variant adds "--plan ...", and the point of that variant is # that the payload below is identical either way. title "Deploy: payload sent to the jobs API\n" $CLI bundle deploy $(readplanarg out.plan.$DATABRICKS_BUNDLE_ENGINE.json) -trace print_requests.py //jobs +# --nostamp: under deployment-history recording the job payload carries the DMS deployment +# stamp; drop it so the assertion is the same whether or not recording is on. +trace print_requests.py //jobs --nostamp diff --git a/acceptance/bundle/escaped_refs/test.toml b/acceptance/bundle/escaped_refs/test.toml index e7b9cac3bf1..94b36638c0e 100644 --- a/acceptance/bundle/escaped_refs/test.toml +++ b/acceptance/bundle/escaped_refs/test.toml @@ -8,3 +8,7 @@ Ignore = [ ".databricks", "databricks.yml", ] + +# This test dumps a plan and replays it via deploy --plan/readplanarg; a nostamp.py'd dump can't +# carry the DMS fields the replay needs, so opt out of recording (covered by bundle/dms). +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/generate/dashboard-inplace/out.test.toml b/acceptance/bundle/generate/dashboard-inplace/out.test.toml index e1af1a235ad..ae800809893 100644 --- a/acceptance/bundle/generate/dashboard-inplace/out.test.toml +++ b/acceptance/bundle/generate/dashboard-inplace/out.test.toml @@ -1,3 +1,3 @@ Cloud = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] -EnvMatrix.DMS = ["", "true"] +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/generate/dashboard-inplace/test.toml b/acceptance/bundle/generate/dashboard-inplace/test.toml index d5511c15564..10eef9f96c4 100644 --- a/acceptance/bundle/generate/dashboard-inplace/test.toml +++ b/acceptance/bundle/generate/dashboard-inplace/test.toml @@ -1,3 +1,7 @@ +# bundle generate reading recorded state from DMS is a fast followup; until then this +# deploy-then-generate flow can't run under recording (local state is a tombstone). +EnvMatrix.DMS = [""] + [[Repls]] Old = "[0-9a-f]{32}" New = "[DASHBOARD_ID]" diff --git a/acceptance/bundle/generate/genie_space_inplace/out.test.toml b/acceptance/bundle/generate/genie_space_inplace/out.test.toml index 59b56a2037c..27ec2a7fcd6 100644 --- a/acceptance/bundle/generate/genie_space_inplace/out.test.toml +++ b/acceptance/bundle/generate/genie_space_inplace/out.test.toml @@ -1,3 +1,3 @@ Cloud = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -EnvMatrix.DMS = ["", "true"] +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/generate/genie_space_inplace/test.toml b/acceptance/bundle/generate/genie_space_inplace/test.toml index 723c9907976..91944fc581e 100644 --- a/acceptance/bundle/generate/genie_space_inplace/test.toml +++ b/acceptance/bundle/generate/genie_space_inplace/test.toml @@ -1,5 +1,8 @@ # Genie spaces are only deployed via the direct deployment engine. EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +# bundle generate reading recorded state from DMS is a fast followup; until then this +# deploy-then-generate flow can't run under recording (local state is a tombstone). +EnvMatrix.DMS = [""] [[Repls]] Old = "[0-9a-f]{32}" diff --git a/acceptance/bundle/migrate/auto-migrate-clean/out.plan_update.direct.json b/acceptance/bundle/migrate/auto-migrate-clean/out.plan_update.direct.json index cbaa60b7ace..e564ae92773 100644 --- a/acceptance/bundle/migrate/auto-migrate-clean/out.plan_update.direct.json +++ b/acceptance/bundle/migrate/auto-migrate-clean/out.plan_update.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 3, diff --git a/acceptance/bundle/migrate/auto-migrate-clean/script b/acceptance/bundle/migrate/auto-migrate-clean/script index 0f2eb7c1ce1..5afad5a39c5 100644 --- a/acceptance/bundle/migrate/auto-migrate-clean/script +++ b/acceptance/bundle/migrate/auto-migrate-clean/script @@ -31,7 +31,7 @@ rm -f out.requests.txt title "Modify the config — plan shows the pending update" trace update_file.py databricks.yml "Test Auto-Migrate Job" "Test Auto-Migrate Job (renamed)" trace $CLI bundle plan -$CLI bundle plan -o json > out.plan_update.direct.json +$CLI bundle plan -o json | nostamp.py > out.plan_update.direct.json title "Deploy picks up the update (with and without --plan)\n" $CLI bundle deploy $(readplanarg out.plan_update.direct.json) diff --git a/acceptance/bundle/migrate/auto-migrate-clean/test.toml b/acceptance/bundle/migrate/auto-migrate-clean/test.toml index ff8a66c196e..32db4804fbf 100644 --- a/acceptance/bundle/migrate/auto-migrate-clean/test.toml +++ b/acceptance/bundle/migrate/auto-migrate-clean/test.toml @@ -1 +1,5 @@ EnvMatrix.READPLAN = ["", "1"] + +# This test dumps a plan and replays it via deploy --plan/readplanarg; a nostamp.py'd dump can't +# carry the DMS fields the replay needs, so opt out of recording (covered by bundle/dms). +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/migrate/auto-migrate-direct-only/script b/acceptance/bundle/migrate/auto-migrate-direct-only/script index 3cb22cb053a..7f9c18cac46 100644 --- a/acceptance/bundle/migrate/auto-migrate-direct-only/script +++ b/acceptance/bundle/migrate/auto-migrate-direct-only/script @@ -10,7 +10,7 @@ trace update_file.py databricks.yml "name: test-bundle" $'name: test-bundle\n e title "Plan skips the direct-only resource, since this run still uses terraform" trace $CLI bundle plan -trace $CLI bundle plan -o json | gron.py | grep instance_pools +trace $CLI bundle plan -o json | nostamp.py | gron.py | grep instance_pools title "Deploy migrates the state, skipping the direct-only resource" trace $CLI bundle deploy diff --git a/acceptance/bundle/migrate/basic/out.plan_update.json b/acceptance/bundle/migrate/basic/out.plan_update.json index 7b9967a085d..fec804b95b3 100644 --- a/acceptance/bundle/migrate/basic/out.plan_update.json +++ b/acceptance/bundle/migrate/basic/out.plan_update.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 6, diff --git a/acceptance/bundle/migrate/basic/script b/acceptance/bundle/migrate/basic/script index 3247e9fad8d..64c42e6d641 100755 --- a/acceptance/bundle/migrate/basic/script +++ b/acceptance/bundle/migrate/basic/script @@ -43,7 +43,7 @@ rm out.requests.txt title "Update databricks.yml, run plan & update" trace update_file.py databricks.yml Migration Migrated trace DATABRICKS_BUNDLE_ENGINE= $CLI bundle plan -trace DATABRICKS_BUNDLE_ENGINE= $CLI bundle plan -o json > out.plan_update.json +trace DATABRICKS_BUNDLE_ENGINE= $CLI bundle plan -o json | nostamp.py > out.plan_update.json trace DATABRICKS_BUNDLE_ENGINE= $CLI bundle deploy title "Different target, still on terraform" diff --git a/acceptance/bundle/migrate/dashboards/out.plan_after_migrate.json b/acceptance/bundle/migrate/dashboards/out.plan_after_migrate.json index cbdc8b8203b..66f20559a9a 100644 --- a/acceptance/bundle/migrate/dashboards/out.plan_after_migrate.json +++ b/acceptance/bundle/migrate/dashboards/out.plan_after_migrate.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 3, diff --git a/acceptance/bundle/migrate/dashboards/script b/acceptance/bundle/migrate/dashboards/script index bd599144f94..b2c9639a0fd 100755 --- a/acceptance/bundle/migrate/dashboards/script +++ b/acceptance/bundle/migrate/dashboards/script @@ -15,7 +15,7 @@ rm out.requests.txt # Badness: known issue with dashboards, will show up as 'recreate' see https://github.com/databricks/cli/pull/3966 trace DATABRICKS_BUNDLE_ENGINE=direct $CLI bundle plan | contains.py "1 unchanged" trace DATABRICKS_BUNDLE_ENGINE= $CLI bundle plan | contains.py "1 unchanged" -trace DATABRICKS_BUNDLE_ENGINE= $CLI bundle plan -o json > out.plan_after_migrate.json +trace DATABRICKS_BUNDLE_ENGINE= $CLI bundle plan -o json | nostamp.py > out.plan_after_migrate.json trace print_requests.py --get //dashboards | contains.py 'engine/direct' > /dev/null trace DATABRICKS_BUNDLE_ENGINE="" $CLI bundle deploy --auto-approve @@ -32,7 +32,7 @@ rm out.requests.txt #title "Update databricks.yml, run plan & update" #trace update_file.py databricks.yml dashboard DASHBOARD #trace DATABRICKS_BUNDLE_ENGINE= $CLI bundle plan -#trace DATABRICKS_BUNDLE_ENGINE= $CLI bundle plan -o json > out.plan_update.json +#trace DATABRICKS_BUNDLE_ENGINE= $CLI bundle plan -o json | nostamp.py > out.plan_update.json #trace DATABRICKS_BUNDLE_ENGINE= $CLI bundle deploy #rm out.requests.txt diff --git a/acceptance/bundle/migrate/default-python/out.plan_after_deploy.json b/acceptance/bundle/migrate/default-python/out.plan_after_deploy.json index 3e01413959f..84de0f4ffdc 100644 --- a/acceptance/bundle/migrate/default-python/out.plan_after_deploy.json +++ b/acceptance/bundle/migrate/default-python/out.plan_after_deploy.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 6, diff --git a/acceptance/bundle/migrate/default-python/out.plan_after_migration.json b/acceptance/bundle/migrate/default-python/out.plan_after_migration.json index e21c3b33069..a54d3357cb1 100644 --- a/acceptance/bundle/migrate/default-python/out.plan_after_migration.json +++ b/acceptance/bundle/migrate/default-python/out.plan_after_migration.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 5, diff --git a/acceptance/bundle/migrate/default-python/script b/acceptance/bundle/migrate/default-python/script index e3a0b9b3e05..6e05ab85f7f 100755 --- a/acceptance/bundle/migrate/default-python/script +++ b/acceptance/bundle/migrate/default-python/script @@ -11,7 +11,7 @@ trace print_state.py > ../out.state_after_migration.json trace jq '.. | .libraries? | select(.)' ../out.state_after_migration.json trace DATABRICKS_BUNDLE_ENGINE=direct $CLI bundle plan -trace DATABRICKS_BUNDLE_ENGINE=direct $CLI bundle plan -o json > ../out.plan_after_migration.json +trace DATABRICKS_BUNDLE_ENGINE=direct $CLI bundle plan -o json | nostamp.py > ../out.plan_after_migration.json trace jq '.plan[] | .changes' ../out.plan_after_migration.json # Badness: contains changes for whl and num_workers trace DATABRICKS_BUNDLE_ENGINE="" $CLI bundle deploy @@ -21,5 +21,5 @@ rm ../out.state_after_deploy.json title "Extra plan: should have no drift" trace DATABRICKS_BUNDLE_ENGINE= $CLI bundle plan -trace DATABRICKS_BUNDLE_ENGINE=direct $CLI bundle plan -o json > ../out.plan_after_deploy.json +trace DATABRICKS_BUNDLE_ENGINE=direct $CLI bundle plan -o json | nostamp.py > ../out.plan_after_deploy.json trace jq '.plan[] | .changes' ../out.plan_after_migration.json # Badness: contains changes for whl diff --git a/acceptance/bundle/migrate/runas/out.plan.json b/acceptance/bundle/migrate/runas/out.plan.json index 0888cdc22ef..56992a01bc0 100644 --- a/acceptance/bundle/migrate/runas/out.plan.json +++ b/acceptance/bundle/migrate/runas/out.plan.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 5, diff --git a/acceptance/bundle/migrate/runas/script b/acceptance/bundle/migrate/runas/script index f3675f4d9bb..4affba0a373 100644 --- a/acceptance/bundle/migrate/runas/script +++ b/acceptance/bundle/migrate/runas/script @@ -23,4 +23,4 @@ trace DATABRICKS_BUNDLE_ENGINE=terraform $CLI bundle plan trace DATABRICKS_BUNDLE_ENGINE= $CLI bundle deployment migrate trace print_state.py > out.new_state.json trace DATABRICKS_BUNDLE_ENGINE= $CLI bundle plan -trace DATABRICKS_BUNDLE_ENGINE= $CLI bundle plan -o json > out.plan.json +trace DATABRICKS_BUNDLE_ENGINE= $CLI bundle plan -o json | nostamp.py > out.plan.json diff --git a/acceptance/bundle/resource_deps/escaped_ref/output.txt b/acceptance/bundle/resource_deps/escaped_ref/output.txt index 2d7df920e29..5370251a14d 100644 --- a/acceptance/bundle/resource_deps/escaped_ref/output.txt +++ b/acceptance/bundle/resource_deps/escaped_ref/output.txt @@ -18,7 +18,7 @@ Created jobs.foo Files: 6 uploaded, 0 deleted Resources: 2 created, 0 changed, 0 deleted, 0 unchanged ->>> print_requests.py //jobs --sort +>>> print_requests.py //jobs --sort --nostamp { "method": "POST", "path": "/api/2.2/jobs/create", diff --git a/acceptance/bundle/resource_deps/escaped_ref/script b/acceptance/bundle/resource_deps/escaped_ref/script index 413394d8b7d..c53b9b85671 100644 --- a/acceptance/bundle/resource_deps/escaped_ref/script +++ b/acceptance/bundle/resource_deps/escaped_ref/script @@ -12,4 +12,6 @@ trace jq '.plan | map_values(.depends_on)' plan.json # that the payload below is identical either way. title "Payload: escaped stays literal, real is resolved\n" $CLI bundle deploy $(readplanarg plan.json) -trace print_requests.py //jobs --sort +# --nostamp: under deployment-history recording the job payload carries the DMS deployment +# stamp; drop it so the assertion is the same whether or not recording is on. +trace print_requests.py //jobs --sort --nostamp diff --git a/acceptance/bundle/resource_deps/grant_ref/out.plan_create.direct.json b/acceptance/bundle/resource_deps/grant_ref/out.plan_create.direct.json index 3bb4c8310c8..424b73dc67b 100644 --- a/acceptance/bundle/resource_deps/grant_ref/out.plan_create.direct.json +++ b/acceptance/bundle/resource_deps/grant_ref/out.plan_create.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.schemas.schema_a": { diff --git a/acceptance/bundle/resource_deps/grant_ref/script b/acceptance/bundle/resource_deps/grant_ref/script index 82800b1654e..ad897f76987 100644 --- a/acceptance/bundle/resource_deps/grant_ref/script +++ b/acceptance/bundle/resource_deps/grant_ref/script @@ -1,4 +1,4 @@ trace $CLI bundle plan -trace $CLI bundle plan -o json > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy diff --git a/acceptance/bundle/resource_deps/id_chain/out.plan_create.direct.json b/acceptance/bundle/resource_deps/id_chain/out.plan_create.direct.json index f004dcdfcde..1c0e5115a32 100644 --- a/acceptance/bundle/resource_deps/id_chain/out.plan_create.direct.json +++ b/acceptance/bundle/resource_deps/id_chain/out.plan_create.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.jobs.a": { diff --git a/acceptance/bundle/resource_deps/id_chain/out.plan_skip.direct.json b/acceptance/bundle/resource_deps/id_chain/out.plan_skip.direct.json index fa72eb2700b..343ca061a3b 100644 --- a/acceptance/bundle/resource_deps/id_chain/out.plan_skip.direct.json +++ b/acceptance/bundle/resource_deps/id_chain/out.plan_skip.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 2, diff --git a/acceptance/bundle/resource_deps/id_chain/out.plan_update.direct.json b/acceptance/bundle/resource_deps/id_chain/out.plan_update.direct.json index 0c9904aec79..192e2dd8c2b 100644 --- a/acceptance/bundle/resource_deps/id_chain/out.plan_update.direct.json +++ b/acceptance/bundle/resource_deps/id_chain/out.plan_update.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resource_deps/id_chain/script b/acceptance/bundle/resource_deps/id_chain/script index bcd103ac2a5..eada43ba1c9 100644 --- a/acceptance/bundle/resource_deps/id_chain/script +++ b/acceptance/bundle/resource_deps/id_chain/script @@ -16,7 +16,7 @@ print_requests_short() { trace $CLI bundle plan trace print_requests -$CLI bundle plan -o json | nostamp > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp.py > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json trace print_requests trace $CLI bundle deploy @@ -25,7 +25,7 @@ trace print_requests_short trace update_file.py databricks.yml aa_desc aa_new_desc trace update_file.py databricks.yml prefix new_prefix -$CLI bundle plan -o json | nostamp > out.plan_update.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp.py > out.plan_update.$DATABRICKS_BUNDLE_ENGINE.json trace print_requests trace $CLI bundle plan @@ -34,5 +34,5 @@ trace print_requests trace $CLI bundle deploy trace print_requests_short -$CLI bundle plan -o json | nostamp > out.plan_skip.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp.py > out.plan_skip.$DATABRICKS_BUNDLE_ENGINE.json trace print_requests diff --git a/acceptance/bundle/resource_deps/id_star/out.plan_create.direct.json b/acceptance/bundle/resource_deps/id_star/out.plan_create.direct.json index 7c32ae3fe4c..b955e17cf46 100644 --- a/acceptance/bundle/resource_deps/id_star/out.plan_create.direct.json +++ b/acceptance/bundle/resource_deps/id_star/out.plan_create.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.jobs.a": { diff --git a/acceptance/bundle/resource_deps/id_star/script b/acceptance/bundle/resource_deps/id_star/script index 6b67f59b7b8..46366e7706a 100644 --- a/acceptance/bundle/resource_deps/id_star/script +++ b/acceptance/bundle/resource_deps/id_star/script @@ -1 +1 @@ -$CLI bundle plan -o json > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp.py > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resource_deps/job_id/out.plan_create.direct.json b/acceptance/bundle/resource_deps/job_id/out.plan_create.direct.json index 43c1928ebb8..4cdbdd7e630 100644 --- a/acceptance/bundle/resource_deps/job_id/out.plan_create.direct.json +++ b/acceptance/bundle/resource_deps/job_id/out.plan_create.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.jobs.bar": { diff --git a/acceptance/bundle/resource_deps/job_id/out.plan_delete.direct.json b/acceptance/bundle/resource_deps/job_id/out.plan_delete.direct.json index 1bfbc8fe053..c6a1b111aca 100644 --- a/acceptance/bundle/resource_deps/job_id/out.plan_delete.direct.json +++ b/acceptance/bundle/resource_deps/job_id/out.plan_delete.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resource_deps/job_id/script b/acceptance/bundle/resource_deps/job_id/script index f8d30917d1c..96d8aed011f 100644 --- a/acceptance/bundle/resource_deps/job_id/script +++ b/acceptance/bundle/resource_deps/job_id/script @@ -1,6 +1,6 @@ trace $CLI bundle plan -trace $CLI bundle plan -o json | nostamp > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py --nostamp //jobs @@ -8,7 +8,7 @@ foo_id=`read_id.py foo` bar_id=`read_id.py bar` cp empty.yml databricks.yml -trace $CLI bundle plan -o json | nostamp > out.plan_delete.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.plan_delete.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py --nostamp //jobs diff --git a/acceptance/bundle/resource_deps/job_id_delete_bar/out.plan_delete.direct.json b/acceptance/bundle/resource_deps/job_id_delete_bar/out.plan_delete.direct.json index 64087a8c63c..8aa9e5e2544 100644 --- a/acceptance/bundle/resource_deps/job_id_delete_bar/out.plan_delete.direct.json +++ b/acceptance/bundle/resource_deps/job_id_delete_bar/out.plan_delete.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resource_deps/job_id_delete_bar/script b/acceptance/bundle/resource_deps/job_id_delete_bar/script index 03b9a087810..8fd0af69437 100644 --- a/acceptance/bundle/resource_deps/job_id_delete_bar/script +++ b/acceptance/bundle/resource_deps/job_id_delete_bar/script @@ -8,7 +8,7 @@ bar_id=`read_id.py bar` title "Delete bar, keep foo (foo config updated to not depend on bar)" cp only_foo.yml databricks.yml -trace $CLI bundle plan -o json | nostamp > out.plan_delete.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.plan_delete.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy # Sort requests because foo's update and bar's delete have no dependency edge # (foo's dependency on bar was removed in only_foo.yml), so execution order is non-deterministic. diff --git a/acceptance/bundle/resource_deps/job_id_delete_foo/out.plan_delete.direct.json b/acceptance/bundle/resource_deps/job_id_delete_foo/out.plan_delete.direct.json index 6c6c9577e8f..83132ceb0b4 100644 --- a/acceptance/bundle/resource_deps/job_id_delete_foo/out.plan_delete.direct.json +++ b/acceptance/bundle/resource_deps/job_id_delete_foo/out.plan_delete.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resource_deps/job_id_delete_foo/script b/acceptance/bundle/resource_deps/job_id_delete_foo/script index f5efbd22609..30e1194c4cd 100644 --- a/acceptance/bundle/resource_deps/job_id_delete_foo/script +++ b/acceptance/bundle/resource_deps/job_id_delete_foo/script @@ -8,7 +8,7 @@ bar_id=`read_id.py bar` title "Delete foo, keep bar" cp only_bar.yml databricks.yml -trace $CLI bundle plan -o json | nostamp > out.plan_delete.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.plan_delete.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py --nostamp //jobs diff --git a/acceptance/bundle/resource_deps/jobs_update/script b/acceptance/bundle/resource_deps/jobs_update/script index f806f2db846..b1fc7332355 100644 --- a/acceptance/bundle/resource_deps/jobs_update/script +++ b/acceptance/bundle/resource_deps/jobs_update/script @@ -21,8 +21,8 @@ trace $CLI bundle plan title "Fetch job ID and verify remote state" # Badness: output should not be different per engine; investigate if it's the same on cloud -trace $CLI jobs get $foo_id | nostamp > out.get_foo.$DATABRICKS_BUNDLE_ENGINE.json -trace $CLI jobs get $bar_id | nostamp +trace $CLI jobs get $foo_id | nostamp.py > out.get_foo.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI jobs get $bar_id | nostamp.py rm out.requests.txt trace $CLI bundle destroy --auto-approve diff --git a/acceptance/bundle/resource_deps/jobs_update_remote/out.plan_create.direct.json b/acceptance/bundle/resource_deps/jobs_update_remote/out.plan_create.direct.json index d768b766d36..be2fb1b17e7 100644 --- a/acceptance/bundle/resource_deps/jobs_update_remote/out.plan_create.direct.json +++ b/acceptance/bundle/resource_deps/jobs_update_remote/out.plan_create.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.jobs.bar": { diff --git a/acceptance/bundle/resource_deps/jobs_update_remote/out.plan_update.direct.json b/acceptance/bundle/resource_deps/jobs_update_remote/out.plan_update.direct.json index 77e77c19eb1..deb96f123e6 100644 --- a/acceptance/bundle/resource_deps/jobs_update_remote/out.plan_update.direct.json +++ b/acceptance/bundle/resource_deps/jobs_update_remote/out.plan_update.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resource_deps/jobs_update_remote/script b/acceptance/bundle/resource_deps/jobs_update_remote/script index d456ed8b023..89a3fc4c36f 100644 --- a/acceptance/bundle/resource_deps/jobs_update_remote/script +++ b/acceptance/bundle/resource_deps/jobs_update_remote/script @@ -1,5 +1,5 @@ echo "*" > .gitignore -$CLI bundle plan -o json | nostamp > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp.py > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py --nostamp //jobs @@ -13,7 +13,7 @@ bar_id=`read_id.py bar` title "Update trigger.periodic.unit remotely and re-deploy; jobs.bar is unchanged" trace envsubst < job_update.json > tmp.json && mv tmp.json job_update.json trace $CLI jobs reset --json @job_update.json -$CLI bundle plan -o json | nostamp > out.plan_update.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp.py > out.plan_update.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle destroy --auto-approve trace print_requests.py --nostamp //jobs diff --git a/acceptance/bundle/resource_deps/missing_string_field/script b/acceptance/bundle/resource_deps/missing_string_field/script index c526e980444..d96e2f30b0a 100644 --- a/acceptance/bundle/resource_deps/missing_string_field/script +++ b/acceptance/bundle/resource_deps/missing_string_field/script @@ -1,4 +1,4 @@ -trace $CLI bundle validate -o json | nostamp | jq .resources +trace $CLI bundle validate -o json | nostamp.py | jq .resources errcode $CLI bundle plan trace print_requests.py --nostamp //pipeline diff --git a/acceptance/bundle/resource_deps/permission_ref/out.plan_create.direct.json b/acceptance/bundle/resource_deps/permission_ref/out.plan_create.direct.json index d7c0cc7b12f..8dea931a43e 100644 --- a/acceptance/bundle/resource_deps/permission_ref/out.plan_create.direct.json +++ b/acceptance/bundle/resource_deps/permission_ref/out.plan_create.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.jobs.job_a": { diff --git a/acceptance/bundle/resource_deps/permission_ref/script b/acceptance/bundle/resource_deps/permission_ref/script index 82800b1654e..ad897f76987 100644 --- a/acceptance/bundle/resource_deps/permission_ref/script +++ b/acceptance/bundle/resource_deps/permission_ref/script @@ -1,4 +1,4 @@ trace $CLI bundle plan -trace $CLI bundle plan -o json > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy diff --git a/acceptance/bundle/resource_deps/pipelines_recreate/out.plan_create.direct.json b/acceptance/bundle/resource_deps/pipelines_recreate/out.plan_create.direct.json index 84d1505045e..60da159159b 100644 --- a/acceptance/bundle/resource_deps/pipelines_recreate/out.plan_create.direct.json +++ b/acceptance/bundle/resource_deps/pipelines_recreate/out.plan_create.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.jobs.bar": { diff --git a/acceptance/bundle/resource_deps/pipelines_recreate/out.plan_noop.direct.json b/acceptance/bundle/resource_deps/pipelines_recreate/out.plan_noop.direct.json index 7b3f986f02c..34c1baf3067 100644 --- a/acceptance/bundle/resource_deps/pipelines_recreate/out.plan_noop.direct.json +++ b/acceptance/bundle/resource_deps/pipelines_recreate/out.plan_noop.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 2, diff --git a/acceptance/bundle/resource_deps/pipelines_recreate/out.plan_update.direct.json b/acceptance/bundle/resource_deps/pipelines_recreate/out.plan_update.direct.json index 4f7329005fb..222f87d5afb 100644 --- a/acceptance/bundle/resource_deps/pipelines_recreate/out.plan_update.direct.json +++ b/acceptance/bundle/resource_deps/pipelines_recreate/out.plan_update.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resource_deps/pipelines_recreate/script b/acceptance/bundle/resource_deps/pipelines_recreate/script index 691e24279b2..b1a6741f081 100644 --- a/acceptance/bundle/resource_deps/pipelines_recreate/script +++ b/acceptance/bundle/resource_deps/pipelines_recreate/script @@ -1,7 +1,7 @@ echo "*" > .gitignore trace $CLI bundle plan -$CLI bundle plan -o json | nostamp > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp.py > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py --nostamp //jobs //pipelines '^//api/2.0/bundle' > out.create.requests.json @@ -14,7 +14,7 @@ trace $CLI bundle plan # empty title "Update storage, triggering recreate for pipeline; this means updating downstream deps" trace update_file.py databricks.yml "storage: dbfs:/my-storage" "storage: dbfs:/my-new-storage" trace $CLI bundle plan -$CLI bundle plan -o json | nostamp > out.plan_update.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp.py > out.plan_update.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy --auto-approve trace print_requests.py --nostamp //jobs //pipelines '^//api/2.0/bundle' > out.update.requests.$DATABRICKS_BUNDLE_ENGINE.json @@ -23,13 +23,13 @@ foo_id_2=`read_id.py foo` title "Fetch resource IDs and verify remote state" trace musterr $CLI pipelines get $foo_id -trace $CLI pipelines get $foo_id_2 | nostamp -trace $CLI jobs get $bar_id | nostamp| jq 'del(.settings.run_as)' +trace $CLI pipelines get $foo_id_2 | nostamp.py +trace $CLI jobs get $bar_id | nostamp.py| jq 'del(.settings.run_as)' rm out.requests.txt title "Follow up plan & deploy do nothing" trace $CLI bundle plan -$CLI bundle plan -o json | nostamp > out.plan_noop.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp.py > out.plan_noop.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py --nostamp //jobs //pipelines '^//api/2.0/bundle' diff --git a/acceptance/bundle/resource_deps/remote_pipeline/out.plan_create.direct.json b/acceptance/bundle/resource_deps/remote_pipeline/out.plan_create.direct.json index ef8c827e130..869b15ad6f7 100644 --- a/acceptance/bundle/resource_deps/remote_pipeline/out.plan_create.direct.json +++ b/acceptance/bundle/resource_deps/remote_pipeline/out.plan_create.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.pipelines.foo1": { diff --git a/acceptance/bundle/resource_deps/remote_pipeline/out.plan_skip.direct.json b/acceptance/bundle/resource_deps/remote_pipeline/out.plan_skip.direct.json index 0bb3be878f6..ac8af43c069 100644 --- a/acceptance/bundle/resource_deps/remote_pipeline/out.plan_skip.direct.json +++ b/acceptance/bundle/resource_deps/remote_pipeline/out.plan_skip.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resource_deps/remote_pipeline/script b/acceptance/bundle/resource_deps/remote_pipeline/script index b8683429aac..4df5a844e73 100644 --- a/acceptance/bundle/resource_deps/remote_pipeline/script +++ b/acceptance/bundle/resource_deps/remote_pipeline/script @@ -3,8 +3,8 @@ print_requests() { rm out.requests.txt } -$CLI bundle plan -o json | nostamp > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp.py > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests -$CLI bundle plan -o json | nostamp > out.plan_skip.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp.py > out.plan_skip.$DATABRICKS_BUNDLE_ENGINE.json trace print_requests diff --git a/acceptance/bundle/resource_deps/resources_var/out.plan.direct.json b/acceptance/bundle/resource_deps/resources_var/out.plan.direct.json index bfa6735a725..6f4ff29a254 100644 --- a/acceptance/bundle/resource_deps/resources_var/out.plan.direct.json +++ b/acceptance/bundle/resource_deps/resources_var/out.plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.pipelines.mypipeline": { diff --git a/acceptance/bundle/resource_deps/resources_var/script b/acceptance/bundle/resource_deps/resources_var/script index cc0db790b5c..c04b243240b 100644 --- a/acceptance/bundle/resource_deps/resources_var/script +++ b/acceptance/bundle/resource_deps/resources_var/script @@ -1,5 +1,5 @@ trace $CLI bundle validate -t dev -o json | jq .resources -$CLI bundle plan -o json > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp.py > out.plan.$DATABRICKS_BUNDLE_ENGINE.json trace errcode $CLI bundle deploy -t dev &> out.deploy.txt trace jq -s '.[] | select(.path=="/api/2.0/pipelines") | .body.name' out.requests.txt trace print_telemetry_bool_values | grep -v direct_drymigrate diff --git a/acceptance/bundle/resource_deps/resources_var_presets_implicit_deps/out.plan.direct.json b/acceptance/bundle/resource_deps/resources_var_presets_implicit_deps/out.plan.direct.json index 748830efbe4..cb7acf0c2d8 100644 --- a/acceptance/bundle/resource_deps/resources_var_presets_implicit_deps/out.plan.direct.json +++ b/acceptance/bundle/resource_deps/resources_var_presets_implicit_deps/out.plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.pipelines.mypipeline": { diff --git a/acceptance/bundle/resource_deps/resources_var_presets_implicit_deps/script b/acceptance/bundle/resource_deps/resources_var_presets_implicit_deps/script index 707ca1f347f..2fb906b4945 100644 --- a/acceptance/bundle/resource_deps/resources_var_presets_implicit_deps/script +++ b/acceptance/bundle/resource_deps/resources_var_presets_implicit_deps/script @@ -1,5 +1,5 @@ trace $CLI bundle validate -t dev -o json | jq .resources -$CLI bundle plan -o json -t dev > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json -t dev | nostamp.py > out.plan.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy -t dev trace jq -s '.[] | select(.path=="/api/2.0/pipelines") | .body.name' out.requests.txt rm out.requests.txt diff --git a/acceptance/bundle/resource_deps/tf_path_renames/out.plan.direct.json b/acceptance/bundle/resource_deps/tf_path_renames/out.plan.direct.json index 37b90504345..fed77573273 100644 --- a/acceptance/bundle/resource_deps/tf_path_renames/out.plan.direct.json +++ b/acceptance/bundle/resource_deps/tf_path_renames/out.plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.jobs.dst_git": { diff --git a/acceptance/bundle/resource_deps/tf_path_renames/script b/acceptance/bundle/resource_deps/tf_path_renames/script index 568894d3ba9..dc491ff0f5e 100644 --- a/acceptance/bundle/resource_deps/tf_path_renames/script +++ b/acceptance/bundle/resource_deps/tf_path_renames/script @@ -1,3 +1,3 @@ -trace $CLI bundle plan -o json > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.plan.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy print_requests.py //jobs | jq -rs '[.[].body.name] | sort[]' diff --git a/acceptance/bundle/resource_deps/volume_path_contains_id/out.plan.direct.json b/acceptance/bundle/resource_deps/volume_path_contains_id/out.plan.direct.json index 1fecd0a7b5a..081c3abd05d 100644 --- a/acceptance/bundle/resource_deps/volume_path_contains_id/out.plan.direct.json +++ b/acceptance/bundle/resource_deps/volume_path_contains_id/out.plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.volumes.bar": { diff --git a/acceptance/bundle/resource_deps/volume_path_contains_id/script b/acceptance/bundle/resource_deps/volume_path_contains_id/script index 914f75443d2..21d91ccd863 100644 --- a/acceptance/bundle/resource_deps/volume_path_contains_id/script +++ b/acceptance/bundle/resource_deps/volume_path_contains_id/script @@ -5,7 +5,7 @@ cleanup() { trace $CLI bundle validate -o json | jq .resources trace $CLI bundle plan -$CLI bundle plan -o json > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp.py > out.plan.$DATABRICKS_BUNDLE_ENGINE.json trap cleanup EXIT trace errcode $CLI bundle deploy &> out.deploy.txt diff --git a/acceptance/bundle/resource_deps/volume_path_job_ref/out.plan.direct.json b/acceptance/bundle/resource_deps/volume_path_job_ref/out.plan.direct.json index a6cab2ee51e..9bca8261612 100644 --- a/acceptance/bundle/resource_deps/volume_path_job_ref/out.plan.direct.json +++ b/acceptance/bundle/resource_deps/volume_path_job_ref/out.plan.direct.json @@ -1,7 +1,7 @@ >>> errcode [CLI] bundle plan -o json { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.jobs.foo": { diff --git a/acceptance/bundle/resource_deps/volume_path_job_ref/out.test.toml b/acceptance/bundle/resource_deps/volume_path_job_ref/out.test.toml index 59b56a2037c..27ec2a7fcd6 100644 --- a/acceptance/bundle/resource_deps/volume_path_job_ref/out.test.toml +++ b/acceptance/bundle/resource_deps/volume_path_job_ref/out.test.toml @@ -1,3 +1,3 @@ Cloud = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -EnvMatrix.DMS = ["", "true"] +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/resource_deps/volume_path_job_ref/script b/acceptance/bundle/resource_deps/volume_path_job_ref/script index 6c7250cc564..21a045fa49c 100644 --- a/acceptance/bundle/resource_deps/volume_path_job_ref/script +++ b/acceptance/bundle/resource_deps/volume_path_job_ref/script @@ -1,5 +1,5 @@ envsubst < databricks.yml.tmpl > databricks.yml -trace $CLI bundle validate -o json | nostamp | jq '.resources.jobs, .resources.volumes' +trace $CLI bundle validate -o json | nostamp.py | jq '.resources.jobs, .resources.volumes' # The job's data_path default references the volume's computed volume_path. # Record the JSON plan per engine to confirm the reference is resolved into the diff --git a/acceptance/bundle/resource_deps/volume_path_job_ref/test.toml b/acceptance/bundle/resource_deps/volume_path_job_ref/test.toml index b53e5e7fbe8..10ce9d961a6 100644 --- a/acceptance/bundle/resource_deps/volume_path_job_ref/test.toml +++ b/acceptance/bundle/resource_deps/volume_path_job_ref/test.toml @@ -6,3 +6,7 @@ RecordRequests = true # so the case is unsupported there (see CHANGELOG.md). Direct is the # default engine, so the motivating use case works out of the box. EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] + +# The plan is captured with `errcode ... &> out` (trace line + JSON together), which nostamp.py +# cannot filter; opt out of the recording variant (deploy --plan recording is covered by bundle/dms). +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/resources/apps/config-drift-stopped/script b/acceptance/bundle/resources/apps/config-drift-stopped/script index c449a3099a3..50e046d9442 100644 --- a/acceptance/bundle/resources/apps/config-drift-stopped/script +++ b/acceptance/bundle/resources/apps/config-drift-stopped/script @@ -10,10 +10,10 @@ trace $CLI bundle deploy title "Change config while running: drift is detected (active deployment present)" trace update_file.py databricks.yml original_value changed_value -trace $CLI bundle plan -o json | jq '.plan[].changes | with_entries(select(.key | startswith("config")))' +trace $CLI bundle plan -o json | nostamp.py | jq '.plan[].changes | with_entries(select(.key | startswith("config")))' title "Stop the app: the backend clears the active deployment" trace $CLI apps stop $UNIQUE_NAME | jq '.compute_status.state' title "Same config change is now skipped: no active deployment to compare against" -trace $CLI bundle plan -o json | jq '.plan[].changes | with_entries(select(.key | startswith("config")))' +trace $CLI bundle plan -o json | nostamp.py | jq '.plan[].changes | with_entries(select(.key | startswith("config")))' diff --git a/acceptance/bundle/resources/apps/config-drift/script b/acceptance/bundle/resources/apps/config-drift/script index 4728255dc8f..23154a3fdf9 100644 --- a/acceptance/bundle/resources/apps/config-drift/script +++ b/acceptance/bundle/resources/apps/config-drift/script @@ -9,7 +9,7 @@ trap cleanup EXIT trace $CLI bundle deploy title "Verify no drift after deploy" -trace $CLI bundle plan -o json | jq '.plan."resources.apps.myapp".changes.config // .plan."resources.apps.myapp".changes' | jq 'del(.[] | select(.action == "skip"))' > out.plan.direct.json +trace $CLI bundle plan -o json | nostamp.py | jq '.plan."resources.apps.myapp".changes.config // .plan."resources.apps.myapp".changes' | jq 'del(.[] | select(.action == "skip"))' > out.plan.direct.json SOURCE_CODE_PATH=$(trace $CLI apps get $UNIQUE_NAME --output json | jq -r '.active_deployment.source_code_path') @@ -27,13 +27,13 @@ $CLI apps deploy $UNIQUE_NAME --no-wait --json '{ title "Plan should detect config drift" trace $CLI bundle plan # Skip entries with action "skip" -$CLI bundle plan -o json | jq '.plan."resources.apps.myapp".changes.config // .plan."resources.apps.myapp".changes' | jq 'del(.[] | select(.action == "skip"))' >> out.plan.direct.json +$CLI bundle plan -o json | nostamp.py | jq '.plan."resources.apps.myapp".changes.config // .plan."resources.apps.myapp".changes' | jq 'del(.[] | select(.action == "skip"))' >> out.plan.direct.json title "Redeploy to fix drift" trace $CLI bundle deploy title "Verify no drift after fix" -trace $CLI bundle plan -o json | jq '.plan."resources.apps.myapp".changes.config // .plan."resources.apps.myapp".changes' | jq 'del(.[] | select(.action == "skip"))' >> out.plan.direct.json +trace $CLI bundle plan -o json | nostamp.py | jq '.plan."resources.apps.myapp".changes.config // .plan."resources.apps.myapp".changes' | jq 'del(.[] | select(.action == "skip"))' >> out.plan.direct.json # TODO: add test for git_source drift when git_source is supported in the Deploy API # Currently it fails with the error: Git source reference is required diff --git a/acceptance/bundle/resources/apps/config-no-deployment/script b/acceptance/bundle/resources/apps/config-no-deployment/script index f1970999645..3b7e38e3493 100644 --- a/acceptance/bundle/resources/apps/config-no-deployment/script +++ b/acceptance/bundle/resources/apps/config-no-deployment/script @@ -7,4 +7,4 @@ title "Plan is a no-op: config/source_code_path drift is skipped while the app h trace $CLI bundle plan | contains.py "Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged" title "Both deploy-only fields are skipped with reason \"no active deployment\"" -trace $CLI bundle plan -o json | jq '.plan[].changes | {config, source_code_path}' +trace $CLI bundle plan -o json | nostamp.py | jq '.plan[].changes | {config, source_code_path}' diff --git a/acceptance/bundle/resources/apps/git-source-no-deployment/script b/acceptance/bundle/resources/apps/git-source-no-deployment/script index 63cce6312c3..28e12809b2e 100644 --- a/acceptance/bundle/resources/apps/git-source-no-deployment/script +++ b/acceptance/bundle/resources/apps/git-source-no-deployment/script @@ -7,4 +7,4 @@ title "Plan is a no-op: git_source drift is skipped while the app has no active trace $CLI bundle plan | contains.py "Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged" title "git_source is skipped with reason \"no active deployment\"" -trace $CLI bundle plan -o json | jq '.plan[].changes | {git_source}' +trace $CLI bundle plan -o json | nostamp.py | jq '.plan[].changes | {git_source}' diff --git a/acceptance/bundle/resources/apps/lifecycle-started-omitted/script b/acceptance/bundle/resources/apps/lifecycle-started-omitted/script index dd08f33fff5..8bdbb4dfd9c 100644 --- a/acceptance/bundle/resources/apps/lifecycle-started-omitted/script +++ b/acceptance/bundle/resources/apps/lifecycle-started-omitted/script @@ -79,6 +79,6 @@ trace $CLI bundle deploy trace print_app_requests title "(started omitted, app running) -> bundle plan shows no drift" -$CLI bundle plan -o json | nostamp > LOG.planjson +$CLI bundle plan -o json | nostamp.py > LOG.planjson verify_no_drift.py LOG.planjson echo "Plan: no drift detected" diff --git a/acceptance/bundle/resources/apps/uc-securable-drift/script b/acceptance/bundle/resources/apps/uc-securable-drift/script index 04438951475..7f3022a2bb1 100644 --- a/acceptance/bundle/resources/apps/uc-securable-drift/script +++ b/acceptance/bundle/resources/apps/uc-securable-drift/script @@ -10,4 +10,4 @@ title "Re-plan is clean: the TABLE securable's server-computed securable_kind is trace $CLI bundle plan | contains.py "Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged" title "securable_kind is classified output-only (skip); the VOLUME securable has none" -trace $CLI bundle plan -o json | jq '.plan."resources.apps.myapp".changes // {} | with_entries(select(.key | test("uc_securable")))' +trace $CLI bundle plan -o json | nostamp.py | jq '.plan."resources.apps.myapp".changes // {} | with_entries(select(.key | test("uc_securable")))' diff --git a/acceptance/bundle/resources/catalogs/comment_out_of_band/script b/acceptance/bundle/resources/catalogs/comment_out_of_band/script index fe406882746..c0f21c30082 100644 --- a/acceptance/bundle/resources/catalogs/comment_out_of_band/script +++ b/acceptance/bundle/resources/catalogs/comment_out_of_band/script @@ -18,7 +18,7 @@ title "Set the comment out of band, the way Catalog Explorer does" MSYS_NO_PATHCONV=1 $CLI api patch "/api/2.1/unity-catalog/catalogs/$CATALOG" --json '{"comment":"set outside the bundle"}' > /dev/null title "The remote comment is drift, so the plan updates the catalog" -trace $CLI bundle plan --output json | jq '.plan[].changes' +trace $CLI bundle plan --output json | nostamp.py | jq '.plan[].changes' title "Redeploy clears the comment" trace $CLI bundle deploy diff --git a/acceptance/bundle/resources/catalogs/drift/managed_properties/script b/acceptance/bundle/resources/catalogs/drift/managed_properties/script index 9413632bb9c..25664d4eefd 100644 --- a/acceptance/bundle/resources/catalogs/drift/managed_properties/script +++ b/acceptance/bundle/resources/catalogs/drift/managed_properties/script @@ -7,7 +7,7 @@ title "Plan is a no-op despite UC auto-populating managed properties" trace $CLI bundle plan | contains.py "Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged" title "The remote-only properties map is skipped as a backend default (confirms the matched rule)" -trace $CLI bundle plan --output json | jq '.plan[].changes' +trace $CLI bundle plan --output json | nostamp.py | jq '.plan[].changes' title "Redeploy is a no-op (no UpdateCatalog call)" trace $CLI bundle deploy diff --git a/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/out.plan_.direct.json b/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/out.plan_.direct.json index c7af4b9a333..cdf7731c99b 100644 --- a/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/out.plan_.direct.json +++ b/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/out.plan_.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.clusters.test_cluster": { @@ -17,7 +17,7 @@ } } { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, @@ -97,7 +97,7 @@ } } { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 2, @@ -180,7 +180,7 @@ } } { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 3, @@ -265,7 +265,7 @@ } } { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 4, diff --git a/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/script b/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/script index 71d7b6860af..00614a0ec10 100755 --- a/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/script +++ b/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/script @@ -10,11 +10,11 @@ trap cleanup EXIT # resize, terraform update), so the per-resource line would diverge. The Resources: # summary agrees, so it is kept rather than using -qq. $CLI bundle plan > out.plan_.$DATABRICKS_BUNDLE_ENGINE.txt -$CLI bundle plan -o json | nostamp > out.plan_.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp.py > out.plan_.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy -q title "Cluster should exist with num_workers after bundle deployment:\n" -CLUSTER_ID=$($CLI bundle summary -o json | nostamp | jq -r '.resources.clusters.test_cluster.id') +CLUSTER_ID=$($CLI bundle summary -o json | nostamp.py | jq -r '.resources.clusters.test_cluster.id') add_repl "$CLUSTER_ID" CLUSTER_ID $CLI clusters get "${CLUSTER_ID}" | jq '{cluster_name,num_workers,autoscale}' @@ -26,7 +26,7 @@ update_file.py databricks.yml " num_workers: 2" " autoscale: min_workers: 2 max_workers: 4" $CLI bundle plan >> out.plan_.$DATABRICKS_BUNDLE_ENGINE.txt -$CLI bundle plan -o json >> out.plan_.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp.py >> out.plan_.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy -q trace jq 'select(.method == "POST" and (.path | contains("/clusters/edit"))) | del(.body.aws_attributes, .body.driver_node_type_id, .body.data_security_mode, .body.enable_elastic_disk)' out.requests.txt rm out.requests.txt @@ -38,7 +38,7 @@ title "Changing autoscale should call update API on stopped cluster\n" update_file.py databricks.yml "min_workers: 2" "min_workers: 3" update_file.py databricks.yml "max_workers: 4" "max_workers: 5" $CLI bundle plan >> out.plan_.$DATABRICKS_BUNDLE_ENGINE.txt -$CLI bundle plan -o json >> out.plan_.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp.py >> out.plan_.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy -q trace jq 'select(.method == "POST" and (.path | contains("/clusters/edit"))) | del(.body.aws_attributes, .body.driver_node_type_id, .body.data_security_mode, .body.enable_elastic_disk)' out.requests.txt rm out.requests.txt @@ -53,7 +53,7 @@ title "Changing autoscale should call resize API on running cluster\n" update_file.py databricks.yml "min_workers: 3" "min_workers: 4" update_file.py databricks.yml "max_workers: 5" "max_workers: 6" $CLI bundle plan >> out.plan_.$DATABRICKS_BUNDLE_ENGINE.txt -$CLI bundle plan -o json >> out.plan_.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp.py >> out.plan_.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy -q trace jq 'select(.method == "POST" and (.path | contains("/clusters/resize")))' out.requests.txt rm out.requests.txt @@ -66,7 +66,7 @@ update_file.py databricks.yml " autoscale: min_workers: 4 max_workers: 6" " num_workers: 3" $CLI bundle plan >> out.plan_.$DATABRICKS_BUNDLE_ENGINE.txt -$CLI bundle plan -o json >> out.plan_.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp.py >> out.plan_.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy -q trace jq 'select(.method == "POST" and (.path | contains("/clusters/resize")))' out.requests.txt rm out.requests.txt diff --git a/acceptance/bundle/resources/clusters/deploy/update-and-resize/out.plan_.direct.json b/acceptance/bundle/resources/clusters/deploy/update-and-resize/out.plan_.direct.json index b4ba1c76152..9d3e514ecfe 100644 --- a/acceptance/bundle/resources/clusters/deploy/update-and-resize/out.plan_.direct.json +++ b/acceptance/bundle/resources/clusters/deploy/update-and-resize/out.plan_.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.clusters.test_cluster": { @@ -20,7 +20,7 @@ } } { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, @@ -92,7 +92,7 @@ } } { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 2, @@ -164,7 +164,7 @@ } } { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 3, diff --git a/acceptance/bundle/resources/clusters/deploy/update-and-resize/script b/acceptance/bundle/resources/clusters/deploy/update-and-resize/script index 14799220348..7643752ef88 100644 --- a/acceptance/bundle/resources/clusters/deploy/update-and-resize/script +++ b/acceptance/bundle/resources/clusters/deploy/update-and-resize/script @@ -10,18 +10,18 @@ trap cleanup EXIT # resize, terraform update), so the per-resource line would diverge. The Resources: # summary agrees, so it is kept rather than using -qq. $CLI bundle plan > out.plan_.$DATABRICKS_BUNDLE_ENGINE.txt -$CLI bundle plan -o json | nostamp > out.plan_.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp.py > out.plan_.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy -q title "Cluster should exist after bundle deployment:\n" -CLUSTER_ID=$($CLI bundle summary -o json | nostamp | jq -r '.resources.clusters.test_cluster.id') +CLUSTER_ID=$($CLI bundle summary -o json | nostamp.py | jq -r '.resources.clusters.test_cluster.id') add_repl "$CLUSTER_ID" CLUSTER_ID $CLI clusters get "${CLUSTER_ID}" | jq '{cluster_name,num_workers}' title "Changing num_workers should call update API on stopped cluster\n" update_file.py databricks.yml "num_workers: 2" "num_workers: 3" $CLI bundle plan >> out.plan_.$DATABRICKS_BUNDLE_ENGINE.txt -$CLI bundle plan -o json >> out.plan_.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp.py >> out.plan_.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy -q trace jq 'select(.method == "POST" and (.path | contains("/clusters/edit"))) | del(.body.aws_attributes, .body.driver_node_type_id, .body.data_security_mode, .body.enable_elastic_disk)' out.requests.txt rm out.requests.txt @@ -35,7 +35,7 @@ $CLI clusters start "${CLUSTER_ID}" title "Changing num_workers should call resize API on running cluster\n" update_file.py databricks.yml "num_workers: 3" "num_workers: 4" $CLI bundle plan >> out.plan_.$DATABRICKS_BUNDLE_ENGINE.txt -$CLI bundle plan -o json >> out.plan_.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp.py >> out.plan_.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy -q trace jq 'select(.method == "POST" and (.path | contains("/clusters/resize")))' out.requests.txt rm out.requests.txt @@ -47,7 +47,7 @@ title "Changing num_workers and spark_conf should call update API\n" update_file.py databricks.yml "num_workers: 4" "num_workers: 5" update_file.py databricks.yml '"spark.executor.memory": "2g"' '"spark.executor.memory": "4g"' $CLI bundle plan >> out.plan_.$DATABRICKS_BUNDLE_ENGINE.txt -$CLI bundle plan -o json >> out.plan_.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp.py >> out.plan_.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy -q trace jq 'select(.method == "POST" and (.path | contains("/clusters/edit"))) | del(.body.aws_attributes, .body.driver_node_type_id, .body.data_security_mode, .body.enable_elastic_disk)' out.requests.txt rm out.requests.txt diff --git a/acceptance/bundle/resources/dashboards/change-serialized-inline/out.plan.direct.json b/acceptance/bundle/resources/dashboards/change-serialized-inline/out.plan.direct.json index e14ec2fba2a..8b496224317 100644 --- a/acceptance/bundle/resources/dashboards/change-serialized-inline/out.plan.direct.json +++ b/acceptance/bundle/resources/dashboards/change-serialized-inline/out.plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resources/dashboards/change-serialized-inline/out.test.toml b/acceptance/bundle/resources/dashboards/change-serialized-inline/out.test.toml index bf4f0e7d3c6..d5b9a6f4a16 100644 --- a/acceptance/bundle/resources/dashboards/change-serialized-inline/out.test.toml +++ b/acceptance/bundle/resources/dashboards/change-serialized-inline/out.test.toml @@ -1,4 +1,4 @@ Cloud = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] -EnvMatrix.DMS = ["", "true"] +EnvMatrix.DMS = [""] EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/resources/dashboards/change-serialized-inline/script b/acceptance/bundle/resources/dashboards/change-serialized-inline/script index 03b8c2ac46b..41629784e3f 100644 --- a/acceptance/bundle/resources/dashboards/change-serialized-inline/script +++ b/acceptance/bundle/resources/dashboards/change-serialized-inline/script @@ -23,7 +23,7 @@ update_file.py databricks.yml "Page 1" "Page One" trace $CLI bundle plan | contains.py "update" "!recreate" -trace $CLI bundle plan -o json > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.plan.$DATABRICKS_BUNDLE_ENGINE.json title "deploy the change\n" $CLI bundle deploy $(readplanarg out.plan.$DATABRICKS_BUNDLE_ENGINE.json) diff --git a/acceptance/bundle/resources/dashboards/change-serialized-inline/test.toml b/acceptance/bundle/resources/dashboards/change-serialized-inline/test.toml index 5cab5faff69..fae8d4e5d0e 100644 --- a/acceptance/bundle/resources/dashboards/change-serialized-inline/test.toml +++ b/acceptance/bundle/resources/dashboards/change-serialized-inline/test.toml @@ -8,3 +8,7 @@ Ignore = [ ] EnvMatrix.READPLAN = ["", "1"] + +# This test dumps a plan and replays it via deploy --plan/readplanarg; a nostamp.py'd dump can't +# carry the DMS fields the replay needs, so opt out of recording (covered by bundle/dms). +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/resources/dashboards/dataset-catalog-schema/out.plan.direct.json b/acceptance/bundle/resources/dashboards/dataset-catalog-schema/out.plan.direct.json index 01e4ab17548..7686609f862 100644 --- a/acceptance/bundle/resources/dashboards/dataset-catalog-schema/out.plan.direct.json +++ b/acceptance/bundle/resources/dashboards/dataset-catalog-schema/out.plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resources/dashboards/dataset-catalog-schema/script b/acceptance/bundle/resources/dashboards/dataset-catalog-schema/script index aa3aa4026b7..e5a1a51185a 100755 --- a/acceptance/bundle/resources/dashboards/dataset-catalog-schema/script +++ b/acceptance/bundle/resources/dashboards/dataset-catalog-schema/script @@ -29,7 +29,7 @@ echo "$DASHBOARD" | jq '{lifecycle_state, parent_path, path}' echo "$DASHBOARD" | jq '.serialized_dashboard | fromjson | .datasets[] | {catalog, schema}' # Verify that there is no drift right after deploy. -trace $CLI bundle plan -o json > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.plan.$DATABRICKS_BUNDLE_ENGINE.json # Modify the direct plan to replace "serialized_dashboard" with a fixture. # It is normalized on the backend so we cannot compare reliably across local and cloud. diff --git a/acceptance/bundle/resources/dashboards/delete-trashed-out-of-band/out.plan.direct.json b/acceptance/bundle/resources/dashboards/delete-trashed-out-of-band/out.plan.direct.json index 6cedd195b0f..c4d8bc38fd9 100644 --- a/acceptance/bundle/resources/dashboards/delete-trashed-out-of-band/out.plan.direct.json +++ b/acceptance/bundle/resources/dashboards/delete-trashed-out-of-band/out.plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resources/dashboards/delete-trashed-out-of-band/script b/acceptance/bundle/resources/dashboards/delete-trashed-out-of-band/script index 2c1396ac618..b8928da1198 100755 --- a/acceptance/bundle/resources/dashboards/delete-trashed-out-of-band/script +++ b/acceptance/bundle/resources/dashboards/delete-trashed-out-of-band/script @@ -25,7 +25,7 @@ trace $CLI lakeview trash $DASHBOARD_ID trace $CLI bundle plan -trace $CLI bundle plan -o json > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.plan.$DATABRICKS_BUNDLE_ENGINE.json # Run destroy - should succeed even though dashboard is already trashed. # -qq drops the summary: terraform refreshes the trashed dashboard away and reports diff --git a/acceptance/bundle/resources/dashboards/detect-change/out.plan.direct.json b/acceptance/bundle/resources/dashboards/detect-change/out.plan.direct.json index c177eb0b450..e11fc5079b5 100644 --- a/acceptance/bundle/resources/dashboards/detect-change/out.plan.direct.json +++ b/acceptance/bundle/resources/dashboards/detect-change/out.plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resources/dashboards/detect-change/script b/acceptance/bundle/resources/dashboards/detect-change/script index 2b4966fd87d..d7c2d18037b 100644 --- a/acceptance/bundle/resources/dashboards/detect-change/script +++ b/acceptance/bundle/resources/dashboards/detect-change/script @@ -38,7 +38,7 @@ add_repl "$(echo "$UPDATE_RESP" | jq -r '.etag')" ETAG_2 title "Try to redeploy the bundle and confirm that the out of band modification is detected:" trace $CLI bundle plan -trace $CLI bundle plan -o json > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.plan.$DATABRICKS_BUNDLE_ENGINE.json trace errcode $CLI bundle deploy title "Redeploy the bundle with the --force flag and confirm that the out of band modification is ignored:" diff --git a/acceptance/bundle/resources/dashboards/generate_inplace/out.test.toml b/acceptance/bundle/resources/dashboards/generate_inplace/out.test.toml index aad5ccf5498..bd29c0328af 100644 --- a/acceptance/bundle/resources/dashboards/generate_inplace/out.test.toml +++ b/acceptance/bundle/resources/dashboards/generate_inplace/out.test.toml @@ -1,4 +1,4 @@ Cloud = true RunsOnDbr = true EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] -EnvMatrix.DMS = ["", "true"] +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/resources/dashboards/generate_inplace/test.toml b/acceptance/bundle/resources/dashboards/generate_inplace/test.toml index 96b263c004a..ca3a5884557 100644 --- a/acceptance/bundle/resources/dashboards/generate_inplace/test.toml +++ b/acceptance/bundle/resources/dashboards/generate_inplace/test.toml @@ -3,3 +3,6 @@ RecordRequests = false RunsOnDbr = true EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] + +# bundle generate reading recorded state from DMS is a fast followup; opt out of recording. +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/resources/dashboards/publish-failure-stale-content/script b/acceptance/bundle/resources/dashboards/publish-failure-stale-content/script index 41022465670..4c455c0c3f5 100644 --- a/acceptance/bundle/resources/dashboards/publish-failure-stale-content/script +++ b/acceptance/bundle/resources/dashboards/publish-failure-stale-content/script @@ -16,7 +16,7 @@ ETAG_1=$(echo "$DASHBOARD" | jq -r '.etag') add_repl "$ETAG_1" ETAG_1 echo "$DASHBOARD" | jq '{display_name, etag}' trace $CLI lakeview get-published $DASHBOARD_ID | jq '{display_name}' -trace $CLI bundle plan -o json | gron.py | grep -E "etag|published" +trace $CLI bundle plan -o json | nostamp.py | gron.py | grep -E "etag|published" rm out.requests.txt # Inject a single publish failure for the update below. @@ -34,7 +34,7 @@ ETAG_2=$(retry --until-not "$ETAG_1" $CLI lakeview get $DASHBOARD_ID | jq -r '.e add_repl "$ETAG_2" ETAG_2 trace $CLI lakeview get $DASHBOARD_ID | jq '{display_name, etag}' trace $CLI lakeview get-published $DASHBOARD_ID | jq '{display_name}' -trace $CLI bundle plan -o json | gron.py | grep -E "etag|published" +trace $CLI bundle plan -o json | nostamp.py | gron.py | grep -E "etag|published" # Bug: re-running deploy fails with "modified remotely" because the stored etag # (pre-PATCH) no longer matches the remote etag (bumped by the PATCH above). diff --git a/acceptance/bundle/resources/dashboards/republish-after-draft-update/script b/acceptance/bundle/resources/dashboards/republish-after-draft-update/script index 9cd9a12869f..cd278bab258 100644 --- a/acceptance/bundle/resources/dashboards/republish-after-draft-update/script +++ b/acceptance/bundle/resources/dashboards/republish-after-draft-update/script @@ -16,7 +16,7 @@ DASHBOARD_ID=$(read_id.py dashboard1) # A clean re-plan is a no-op: the published content matches the draft. title "Plan right after deploy -- published is current, no change:" -trace $CLI bundle plan -o json | jq '.plan["resources.dashboards.dashboard1"].changes.published' +trace $CLI bundle plan -o json | nostamp.py | jq '.plan["resources.dashboards.dashboard1"].changes.published' # Make an out-of-band DRAFT update without republishing. This advances the draft's # update_time past the published revision_create_time, so the published content is now @@ -28,4 +28,4 @@ DASHBOARD_JSON="{\"serialized_dashboard\": \"{}\", \"warehouse_id\": \"$TEST_DEF $CLI lakeview update "$DASHBOARD_ID" --json "${DASHBOARD_JSON}" > /dev/null title "Plan after out-of-band draft update -- a republish is owed:" -trace $CLI bundle plan -o json | jq '.plan["resources.dashboards.dashboard1"].changes.published' +trace $CLI bundle plan -o json | nostamp.py | jq '.plan["resources.dashboards.dashboard1"].changes.published' diff --git a/acceptance/bundle/resources/dashboards/simple/out.plan.direct.json b/acceptance/bundle/resources/dashboards/simple/out.plan.direct.json index 5e6ad9eb0a6..ad1171713f2 100644 --- a/acceptance/bundle/resources/dashboards/simple/out.plan.direct.json +++ b/acceptance/bundle/resources/dashboards/simple/out.plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resources/dashboards/simple/script b/acceptance/bundle/resources/dashboards/simple/script index 4c8b9b55ace..6e2b14529f1 100644 --- a/acceptance/bundle/resources/dashboards/simple/script +++ b/acceptance/bundle/resources/dashboards/simple/script @@ -24,4 +24,4 @@ echo "$DASHBOARD" | jq '{lifecycle_state, parent_path, path, serialized_dashboar # Verify that there is no drift right after deploy. We use |= to update the serialized_dashboard field in place to account # for formatting that the cloud server applies to the serialized_dashboard field. -trace $CLI bundle plan -o json > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.plan.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/dashboards/unpublish-out-of-band/out.plan.direct.json b/acceptance/bundle/resources/dashboards/unpublish-out-of-band/out.plan.direct.json index 852cc492879..cd96018988d 100644 --- a/acceptance/bundle/resources/dashboards/unpublish-out-of-band/out.plan.direct.json +++ b/acceptance/bundle/resources/dashboards/unpublish-out-of-band/out.plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resources/dashboards/unpublish-out-of-band/out.plan_initial.direct.json b/acceptance/bundle/resources/dashboards/unpublish-out-of-band/out.plan_initial.direct.json index b3efa15d5da..42a8d2983bf 100644 --- a/acceptance/bundle/resources/dashboards/unpublish-out-of-band/out.plan_initial.direct.json +++ b/acceptance/bundle/resources/dashboards/unpublish-out-of-band/out.plan_initial.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.dashboards.dashboard1": { diff --git a/acceptance/bundle/resources/dashboards/unpublish-out-of-band/script b/acceptance/bundle/resources/dashboards/unpublish-out-of-band/script index c3af11af2f7..ddb2b9e8b34 100644 --- a/acceptance/bundle/resources/dashboards/unpublish-out-of-band/script +++ b/acceptance/bundle/resources/dashboards/unpublish-out-of-band/script @@ -6,7 +6,7 @@ cleanup() { } trap cleanup EXIT -trace $CLI bundle plan -o json > out.plan_initial.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.plan_initial.$DATABRICKS_BUNDLE_ENGINE.json # Deploy the dashboard trace $CLI bundle deploy -qq @@ -25,7 +25,7 @@ trace $CLI lakeview unpublish $DASHBOARD_ID # Direct: shows "update" because Published field changes from false to true trace $CLI bundle plan > out.plan.$DATABRICKS_BUNDLE_ENGINE.txt -$CLI bundle plan -o json | nostamp > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp.py > out.plan.$DATABRICKS_BUNDLE_ENGINE.json json_in_json_normalize.py out.plan.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy -qq diff --git a/acceptance/bundle/resources/genie_spaces/inline/out.plan.json b/acceptance/bundle/resources/genie_spaces/inline/out.plan.json index 02b174d3869..60aedcde18e 100644 --- a/acceptance/bundle/resources/genie_spaces/inline/out.plan.json +++ b/acceptance/bundle/resources/genie_spaces/inline/out.plan.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resources/genie_spaces/inline/script b/acceptance/bundle/resources/genie_spaces/inline/script index 812cfb9abd4..9e5892a1416 100644 --- a/acceptance/bundle/resources/genie_spaces/inline/script +++ b/acceptance/bundle/resources/genie_spaces/inline/script @@ -15,4 +15,4 @@ add_repl "$GENIE_SPACE_ID" GENIE_SPACE_ID # Without normalization the inline serialized_space leaves a map in the # config struct while state holds a string, and structdiff reports false # drift on every plan. -trace $CLI bundle plan -o json > out.plan.json +trace $CLI bundle plan -o json | nostamp.py > out.plan.json diff --git a/acceptance/bundle/resources/genie_spaces/version_migration/script b/acceptance/bundle/resources/genie_spaces/version_migration/script index 141efd5bef6..5615bbb7034 100644 --- a/acceptance/bundle/resources/genie_spaces/version_migration/script +++ b/acceptance/bundle/resources/genie_spaces/version_migration/script @@ -19,7 +19,7 @@ trace $CLI bundle plan # (and the etag guarding it) is version 2. The serialized_space change is # recorded but skipped via the etag_based rule. title "serialized_space: local version 1, remote version 2, skipped (etag_based)" -$CLI bundle plan -o json \ +$CLI bundle plan -o json | nostamp.py \ | jq '.plan["resources.genie_spaces.foo"].changes.serialized_space | {action, reason, local_version: (.new | fromjson | .version), diff --git a/acceptance/bundle/resources/grants/catalogs/out.plan1.direct.json b/acceptance/bundle/resources/grants/catalogs/out.plan1.direct.json index 51f6d220e56..008571de908 100644 --- a/acceptance/bundle/resources/grants/catalogs/out.plan1.direct.json +++ b/acceptance/bundle/resources/grants/catalogs/out.plan1.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.catalogs.grants_catalog": { diff --git a/acceptance/bundle/resources/grants/catalogs/out.plan2.direct.json b/acceptance/bundle/resources/grants/catalogs/out.plan2.direct.json index fa674375af2..afefaf2f192 100644 --- a/acceptance/bundle/resources/grants/catalogs/out.plan2.direct.json +++ b/acceptance/bundle/resources/grants/catalogs/out.plan2.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resources/grants/catalogs/script b/acceptance/bundle/resources/grants/catalogs/script index 067710b705e..aa47ee0ce94 100755 --- a/acceptance/bundle/resources/grants/catalogs/script +++ b/acceptance/bundle/resources/grants/catalogs/script @@ -1,7 +1,7 @@ envsubst < databricks.yml.tmpl > databricks.yml trace $CLI bundle plan -trace $CLI bundle plan -o json | normalize_uc_payload.py created_at created_by updated_at updated_by metastore_id browse_only effective_predictive_optimization_flag enable_predictive_optimization isolation_mode securable_type > out.plan1.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py | normalize_uc_payload.py created_at created_by updated_at updated_by metastore_id browse_only effective_predictive_optimization_flag enable_predictive_optimization isolation_mode securable_type > out.plan1.$DATABRICKS_BUNDLE_ENGINE.json trace print_requests.py --get //permissions trace $CLI bundle deploy trace print_requests.py //permissions > out.deploy1.requests.$DATABRICKS_BUNDLE_ENGINE.json @@ -13,7 +13,7 @@ trace $CLI grants get catalog catalog_grants_$UNIQUE_NAME | jq --sort-keys update_file.py databricks.yml CREATE_SCHEMA USE_SCHEMA -trace $CLI bundle plan -o json | normalize_uc_payload.py created_at created_by updated_at updated_by metastore_id browse_only effective_predictive_optimization_flag enable_predictive_optimization isolation_mode securable_type > out.plan2.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py | normalize_uc_payload.py created_at created_by updated_at updated_by metastore_id browse_only effective_predictive_optimization_flag enable_predictive_optimization isolation_mode securable_type > out.plan2.$DATABRICKS_BUNDLE_ENGINE.json trace print_requests.py --get //permissions trace $CLI bundle deploy trace print_requests.py //permissions > out.deploy2.requests.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/grants/registered_models/out.plan1.direct.json b/acceptance/bundle/resources/grants/registered_models/out.plan1.direct.json index ab6c291b239..66689f09124 100644 --- a/acceptance/bundle/resources/grants/registered_models/out.plan1.direct.json +++ b/acceptance/bundle/resources/grants/registered_models/out.plan1.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.registered_models.my_registered_model": { diff --git a/acceptance/bundle/resources/grants/registered_models/script b/acceptance/bundle/resources/grants/registered_models/script index 98d73773afc..43ec2d3b87a 100644 --- a/acceptance/bundle/resources/grants/registered_models/script +++ b/acceptance/bundle/resources/grants/registered_models/script @@ -1,6 +1,6 @@ envsubst < databricks.yml.tmpl > databricks.yml -trace $CLI bundle plan -o json > out.plan1.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.plan1.$DATABRICKS_BUNDLE_ENGINE.json trace print_requests.py --get //permissions trace $CLI bundle deploy trace print_requests.py //permissions > out.deploy1.requests.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/grants/schemas/change_privilege/out.plan1.direct.json b/acceptance/bundle/resources/grants/schemas/change_privilege/out.plan1.direct.json index 534d0328aa1..429898e1b67 100644 --- a/acceptance/bundle/resources/grants/schemas/change_privilege/out.plan1.direct.json +++ b/acceptance/bundle/resources/grants/schemas/change_privilege/out.plan1.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.schemas.grants_schema": { diff --git a/acceptance/bundle/resources/grants/schemas/change_privilege/out.plan2.direct.json b/acceptance/bundle/resources/grants/schemas/change_privilege/out.plan2.direct.json index d8bbf97ffd6..46444558c43 100644 --- a/acceptance/bundle/resources/grants/schemas/change_privilege/out.plan2.direct.json +++ b/acceptance/bundle/resources/grants/schemas/change_privilege/out.plan2.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resources/grants/schemas/change_privilege/script b/acceptance/bundle/resources/grants/schemas/change_privilege/script index 737f7491d3a..ec907ef8887 100644 --- a/acceptance/bundle/resources/grants/schemas/change_privilege/script +++ b/acceptance/bundle/resources/grants/schemas/change_privilege/script @@ -1,7 +1,7 @@ envsubst < databricks.yml.tmpl > databricks.yml trace $CLI bundle plan -trace $CLI bundle plan -o json > out.plan1.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.plan1.$DATABRICKS_BUNDLE_ENGINE.json trace print_requests.py --get //permissions trace $CLI bundle deploy trace print_requests.py //permissions > out.deploy1.requests.$DATABRICKS_BUNDLE_ENGINE.json @@ -13,7 +13,7 @@ trace $CLI grants get schema main.schema_grants_$UNIQUE_NAME | jq --sort-keys update_file.py databricks.yml USE_SCHEMA APPLY_TAG -trace $CLI bundle plan -o json | normalize_uc_payload.py effective_predictive_optimization_flag enable_predictive_optimization metastore_id schema_id updated_at updated_by > out.plan2.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py | normalize_uc_payload.py effective_predictive_optimization_flag enable_predictive_optimization metastore_id schema_id updated_at updated_by > out.plan2.$DATABRICKS_BUNDLE_ENGINE.json trace print_requests.py --get //permissions trace $CLI bundle deploy trace print_requests.py //permissions > out.deploy2.requests.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/grants/schemas/duplicate_principals/out.plan.direct.json b/acceptance/bundle/resources/grants/schemas/duplicate_principals/out.plan.direct.json index ff320b18213..f05a65c1a5d 100644 --- a/acceptance/bundle/resources/grants/schemas/duplicate_principals/out.plan.direct.json +++ b/acceptance/bundle/resources/grants/schemas/duplicate_principals/out.plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resources/grants/schemas/duplicate_principals/script b/acceptance/bundle/resources/grants/schemas/duplicate_principals/script index 12a255773b1..1056a114bfb 100644 --- a/acceptance/bundle/resources/grants/schemas/duplicate_principals/script +++ b/acceptance/bundle/resources/grants/schemas/duplicate_principals/script @@ -11,4 +11,4 @@ trace $CLI bundle deploy print_requests.py --get //permissions --keep > out.requests.$DATABRICKS_BUNDLE_ENGINE.txt # effective_predictive_optimization_flag is inherited from the metastore and backend-controlled; drop it so the test does not depend on metastore settings. # updated_at/updated_by reflect UC's post-create system write that populates managed defaults, so on managed-defaults clouds they show a system principal rather than the creator; drop them. -trace $CLI bundle plan -o json | normalize_uc_payload.py effective_predictive_optimization_flag updated_at updated_by > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py | normalize_uc_payload.py effective_predictive_optimization_flag updated_at updated_by > out.plan.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/grants/schemas/out_of_band_principal/out.plan.direct.json b/acceptance/bundle/resources/grants/schemas/out_of_band_principal/out.plan.direct.json index cc1ae0e9cf3..bc969eb2f1c 100644 --- a/acceptance/bundle/resources/grants/schemas/out_of_band_principal/out.plan.direct.json +++ b/acceptance/bundle/resources/grants/schemas/out_of_band_principal/out.plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resources/grants/schemas/out_of_band_principal/out.plan2.direct.json b/acceptance/bundle/resources/grants/schemas/out_of_band_principal/out.plan2.direct.json index bf38acd93fe..8021983f5b5 100644 --- a/acceptance/bundle/resources/grants/schemas/out_of_band_principal/out.plan2.direct.json +++ b/acceptance/bundle/resources/grants/schemas/out_of_band_principal/out.plan2.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 2, diff --git a/acceptance/bundle/resources/grants/schemas/out_of_band_principal/script b/acceptance/bundle/resources/grants/schemas/out_of_band_principal/script index 6f8965cfe1a..fe3b395156d 100644 --- a/acceptance/bundle/resources/grants/schemas/out_of_band_principal/script +++ b/acceptance/bundle/resources/grants/schemas/out_of_band_principal/script @@ -12,7 +12,7 @@ trap cleanup EXIT trace $CLI bundle deploy trace $CLI grants update schema "$SCHEMA_FULL_NAME" --json @update.json > /dev/null $CLI grants get schema "$SCHEMA_FULL_NAME" | gron.py --noindex | sort | contains.py "$CURRENT_USER_NAME" 'deco-test-user@databricks.com' > /dev/null -trace $CLI bundle plan -o json > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.plan.$DATABRICKS_BUNDLE_ENGINE.json # remote_state.__embed__ order is non-deterministic; extract and sort separately into per-engine file jq '.plan["resources.schemas.grants_schema.grants"].remote_state.__embed__' out.plan.$DATABRICKS_BUNDLE_ENGINE.json | gron.py --noindex | sort_lines.py --repl > out.embed.$DATABRICKS_BUNDLE_ENGINE.txt tmp=$(mktemp) @@ -21,4 +21,4 @@ tmp=$(mktemp) # updated_at/updated_by reflect UC's post-create system write that populates managed defaults, so on managed-defaults clouds they show a system principal rather than the creator; drop them. jq 'del(.plan["resources.schemas.grants_schema.grants"].remote_state.__embed__)' out.plan.$DATABRICKS_BUNDLE_ENGINE.json | normalize_uc_payload.py effective_predictive_optimization_flag updated_at updated_by > "$tmp" && mv "$tmp" out.plan.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy -trace $CLI bundle plan -o json | normalize_uc_payload.py effective_predictive_optimization_flag updated_at updated_by > out.plan2.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py | normalize_uc_payload.py effective_predictive_optimization_flag updated_at updated_by > out.plan2.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/grants/volumes/out.plan1.direct.json b/acceptance/bundle/resources/grants/volumes/out.plan1.direct.json index f49832eddc9..4e059e5634b 100644 --- a/acceptance/bundle/resources/grants/volumes/out.plan1.direct.json +++ b/acceptance/bundle/resources/grants/volumes/out.plan1.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.schemas.grants_schema": { diff --git a/acceptance/bundle/resources/grants/volumes/out.plan2.direct.json b/acceptance/bundle/resources/grants/volumes/out.plan2.direct.json index 1daacee25b4..9042bde70e8 100644 --- a/acceptance/bundle/resources/grants/volumes/out.plan2.direct.json +++ b/acceptance/bundle/resources/grants/volumes/out.plan2.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resources/grants/volumes/script b/acceptance/bundle/resources/grants/volumes/script index 725049cda8e..9b8854c8062 100644 --- a/acceptance/bundle/resources/grants/volumes/script +++ b/acceptance/bundle/resources/grants/volumes/script @@ -1,6 +1,6 @@ envsubst < databricks.yml.tmpl > databricks.yml -trace $CLI bundle plan -o json > out.plan1.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.plan1.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py //permissions > out.deploy1.requests.$DATABRICKS_BUNDLE_ENGINE.json @@ -10,7 +10,7 @@ trace $CLI grants get volume main.schema_grants_$UNIQUE_NAME.volume_name | jq -- update_file.py databricks.yml WRITE_VOLUME MANAGE # different on cloud vs local due to remote state -trace $CLI bundle plan -o json | normalize_uc_payload.py effective_predictive_optimization_flag enable_predictive_optimization metastore_id schema_id updated_at updated_by volume_id > out.plan2.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py | normalize_uc_payload.py effective_predictive_optimization_flag enable_predictive_optimization metastore_id schema_id updated_at updated_by volume_id > out.plan2.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py //permissions > out.deploy2.requests.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/job_runs/failed_run/script b/acceptance/bundle/resources/job_runs/failed_run/script index 0fe24556182..2c1ee0fa812 100644 --- a/acceptance/bundle/resources/job_runs/failed_run/script +++ b/acceptance/bundle/resources/job_runs/failed_run/script @@ -15,7 +15,7 @@ musterr trace $CLI bundle deploy # The framework saves the run id before WaitAfterCreate, so FAILED vs SUCCESS is drift. title "the failed run is recorded, and not having succeeded is drift" trace read_id.py my_run -trace $CLI bundle plan -o json | jq '.plan["resources.job_runs.my_run"]' +trace $CLI bundle plan -o json | nostamp.py | jq '.plan["resources.job_runs.my_run"]' # Name the job too, so the run-now bodies below say which number it is. read_id.py my_job > /dev/null diff --git a/acceptance/bundle/resources/job_runs/redeploy/script b/acceptance/bundle/resources/job_runs/redeploy/script index 8ceb3a88052..2c969f0a969 100644 --- a/acceptance/bundle/resources/job_runs/redeploy/script +++ b/acceptance/bundle/resources/job_runs/redeploy/script @@ -17,7 +17,7 @@ trace print_requests.py //jobs/run-now title "change the run configuration and redeploy" trace update_file.py databricks.yml "env: dev" "env: prod" -trace $CLI bundle plan -o json | jq '.plan["resources.job_runs.my_run"]' +trace $CLI bundle plan -o json | nostamp.py | jq '.plan["resources.job_runs.my_run"]' trace $CLI bundle deploy read_id.py my_run > /dev/null trace $CLI bundle summary diff --git a/acceptance/bundle/resources/job_runs/wait/script b/acceptance/bundle/resources/job_runs/wait/script index 70cee1c9082..3a68150df3b 100644 --- a/acceptance/bundle/resources/job_runs/wait/script +++ b/acceptance/bundle/resources/job_runs/wait/script @@ -15,5 +15,5 @@ downstream_id=$(read_id.py downstream_job) trace $CLI jobs get $downstream_id -o json | jq -r ".settings.tags.run_result" title "nothing the run resolved is drift, so a redeploy starts no new run" -trace $CLI bundle plan -o json | jq -r '.plan | to_entries[] | "\(.key) \(.value.action)"' +trace $CLI bundle plan -o json | nostamp.py | jq -r '.plan | to_entries[] | "\(.key) \(.value.action)"' trace $CLI bundle deploy diff --git a/acceptance/bundle/resources/jobs/big_id/out.plan.direct.json b/acceptance/bundle/resources/jobs/big_id/out.plan.direct.json index 6328c4f6dd7..aaf9e6c7458 100644 --- a/acceptance/bundle/resources/jobs/big_id/out.plan.direct.json +++ b/acceptance/bundle/resources/jobs/big_id/out.plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.jobs.foo": { diff --git a/acceptance/bundle/resources/jobs/big_id/script b/acceptance/bundle/resources/jobs/big_id/script index b5823022572..00188cda82d 100644 --- a/acceptance/bundle/resources/jobs/big_id/script +++ b/acceptance/bundle/resources/jobs/big_id/script @@ -1,5 +1,5 @@ trace $CLI bundle validate -o json | jq .resources > out.validate.json -trace $CLI bundle plan -o json > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.plan.$DATABRICKS_BUNDLE_ENGINE.json $CLI bundle deploy $(readplanarg out.plan.direct.json) trace print_requests.py //jobs --nostamp print_state.py > out.state.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/jobs/big_id/test.toml b/acceptance/bundle/resources/jobs/big_id/test.toml index b15d8f9e757..8957ed9ce06 100644 --- a/acceptance/bundle/resources/jobs/big_id/test.toml +++ b/acceptance/bundle/resources/jobs/big_id/test.toml @@ -3,8 +3,8 @@ EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ['direct'] EnvMatrix.READPLAN = ["", "1"] -# `bundle deploy --plan` applies pre-saved state; the stamp isn't written into the plan, -# so the next plan reports it as a change. Recording skipped until stamp reaches saved plans. +# Dumps raw state/plan, which carry the DMS deployment stamp under recording (applied at deploy, +# not saved in the plan). deploy --plan recording is covered by bundle/dms, so opt out here. EnvMatrix.DMS = [""] [[Repls]] diff --git a/acceptance/bundle/resources/jobs/delete_job/out.plan.direct.json b/acceptance/bundle/resources/jobs/delete_job/out.plan.direct.json index f5f1717411e..126e0cbda1b 100644 --- a/acceptance/bundle/resources/jobs/delete_job/out.plan.direct.json +++ b/acceptance/bundle/resources/jobs/delete_job/out.plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resources/jobs/delete_job/script b/acceptance/bundle/resources/jobs/delete_job/script index 1622b3e11cb..f1bf01f596d 100644 --- a/acceptance/bundle/resources/jobs/delete_job/script +++ b/acceptance/bundle/resources/jobs/delete_job/script @@ -1,5 +1,5 @@ trace $CLI bundle deploy cp empty.yml databricks.yml -$CLI bundle plan -o json | nostamp > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp.py > out.plan.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy print_requests.py //jobs --nostamp diff --git a/acceptance/bundle/resources/jobs/delete_task/out.plan_create.direct.json b/acceptance/bundle/resources/jobs/delete_task/out.plan_create.direct.json index a881f097c7a..0539fa4c2de 100644 --- a/acceptance/bundle/resources/jobs/delete_task/out.plan_create.direct.json +++ b/acceptance/bundle/resources/jobs/delete_task/out.plan_create.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.jobs.job": { diff --git a/acceptance/bundle/resources/jobs/delete_task/out.plan_update.direct.json b/acceptance/bundle/resources/jobs/delete_task/out.plan_update.direct.json index 7a64f0a18d6..7e37e4e09e1 100644 --- a/acceptance/bundle/resources/jobs/delete_task/out.plan_update.direct.json +++ b/acceptance/bundle/resources/jobs/delete_task/out.plan_update.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resources/jobs/delete_task/script b/acceptance/bundle/resources/jobs/delete_task/script index 29b856e5ceb..89cf79be7e3 100644 --- a/acceptance/bundle/resources/jobs/delete_task/script +++ b/acceptance/bundle/resources/jobs/delete_task/script @@ -1,6 +1,6 @@ -$CLI bundle plan -o json > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp.py > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json $CLI bundle deploy $(readplanarg out.plan_create.direct.json) grep -v TO_DELETE databricks.yml > tmp.yml && mv tmp.yml databricks.yml trace cat databricks.yml -$CLI bundle plan -o json > out.plan_update.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp.py > out.plan_update.$DATABRICKS_BUNDLE_ENGINE.json $CLI bundle deploy $(readplanarg out.plan_update.direct.json) diff --git a/acceptance/bundle/resources/jobs/delete_task/test.toml b/acceptance/bundle/resources/jobs/delete_task/test.toml index 46a4d91b848..04d6ea1e5b3 100644 --- a/acceptance/bundle/resources/jobs/delete_task/test.toml +++ b/acceptance/bundle/resources/jobs/delete_task/test.toml @@ -1,5 +1,5 @@ RecordRequests = false EnvMatrix.READPLAN = ["", "1"] -# `bundle deploy --plan` applies pre-saved state; the stamp isn't written into the plan, -# so the next plan reports it as a change. Recording skipped until stamp reaches saved plans. +# Dumps raw state/plan, which carry the DMS deployment stamp under recording (applied at deploy, +# not saved in the plan). deploy --plan recording is covered by bundle/dms, so opt out here. EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/resources/jobs/on_failure_empty_slice/script b/acceptance/bundle/resources/jobs/on_failure_empty_slice/script index 6f0606bc346..55c9b9bbf82 100644 --- a/acceptance/bundle/resources/jobs/on_failure_empty_slice/script +++ b/acceptance/bundle/resources/jobs/on_failure_empty_slice/script @@ -1,3 +1,3 @@ trace $CLI bundle plan trace $CLI bundle deploy -trace $CLI bundle plan -o json | nostamp | jq '.plan."resources.jobs.refresh_usage_logs".changes // .' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py | jq '.plan."resources.jobs.refresh_usage_logs".changes // .' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/jobs/remote_add_tag/out.plan_post_update.direct.json b/acceptance/bundle/resources/jobs/remote_add_tag/out.plan_post_update.direct.json index 693492859fc..052acba00b3 100644 --- a/acceptance/bundle/resources/jobs/remote_add_tag/out.plan_post_update.direct.json +++ b/acceptance/bundle/resources/jobs/remote_add_tag/out.plan_post_update.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resources/jobs/remote_add_tag/script b/acceptance/bundle/resources/jobs/remote_add_tag/script index 3a651737979..b2b2fefd0ba 100644 --- a/acceptance/bundle/resources/jobs/remote_add_tag/script +++ b/acceptance/bundle/resources/jobs/remote_add_tag/script @@ -8,4 +8,4 @@ r["tags"]["new_tag"] = "new_value" EOF $CLI bundle plan -$CLI bundle plan -o json | nostamp > out.plan_post_update.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp.py > out.plan_post_update.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/jobs/remote_delete/deploy/out.plan.direct.json b/acceptance/bundle/resources/jobs/remote_delete/deploy/out.plan.direct.json index c9493e8a21c..796dbf0528b 100644 --- a/acceptance/bundle/resources/jobs/remote_delete/deploy/out.plan.direct.json +++ b/acceptance/bundle/resources/jobs/remote_delete/deploy/out.plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resources/jobs/remote_delete/deploy/test.toml b/acceptance/bundle/resources/jobs/remote_delete/deploy/test.toml index 46a4d91b848..04d6ea1e5b3 100644 --- a/acceptance/bundle/resources/jobs/remote_delete/deploy/test.toml +++ b/acceptance/bundle/resources/jobs/remote_delete/deploy/test.toml @@ -1,5 +1,5 @@ RecordRequests = false EnvMatrix.READPLAN = ["", "1"] -# `bundle deploy --plan` applies pre-saved state; the stamp isn't written into the plan, -# so the next plan reports it as a change. Recording skipped until stamp reaches saved plans. +# Dumps raw state/plan, which carry the DMS deployment stamp under recording (applied at deploy, +# not saved in the plan). deploy --plan recording is covered by bundle/dms, so opt out here. EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/resources/jobs/remote_matches_config/out.plan.direct.json b/acceptance/bundle/resources/jobs/remote_matches_config/out.plan.direct.json index 3af240a0ed0..94e8d4554e1 100644 --- a/acceptance/bundle/resources/jobs/remote_matches_config/out.plan.direct.json +++ b/acceptance/bundle/resources/jobs/remote_matches_config/out.plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resources/jobs/remote_matches_config/script b/acceptance/bundle/resources/jobs/remote_matches_config/script index 9ca6a0655b6..b6ed653c0ef 100755 --- a/acceptance/bundle/resources/jobs/remote_matches_config/script +++ b/acceptance/bundle/resources/jobs/remote_matches_config/script @@ -13,7 +13,7 @@ r["max_concurrent_runs"] = 2 EOF trace $CLI bundle plan -$CLI bundle plan -o json | nostamp > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp.py > out.plan.$DATABRICKS_BUNDLE_ENGINE.json rm out.requests.txt # XXX READPLAN diff --git a/acceptance/bundle/resources/jobs/tags_empty_map/script b/acceptance/bundle/resources/jobs/tags_empty_map/script index 4a232723bd0..d3dd04214cf 100644 --- a/acceptance/bundle/resources/jobs/tags_empty_map/script +++ b/acceptance/bundle/resources/jobs/tags_empty_map/script @@ -1,3 +1,3 @@ trace $CLI bundle plan trace $CLI bundle deploy -trace $CLI bundle plan -o json | nostamp | jq '.plan."resources.jobs.test_job".changes // .' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py | jq '.plan."resources.jobs.test_job".changes // .' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/jobs/update/out.plan_create.direct.json b/acceptance/bundle/resources/jobs/update/out.plan_create.direct.json index e13d1b307a5..4a7b31e1946 100644 --- a/acceptance/bundle/resources/jobs/update/out.plan_create.direct.json +++ b/acceptance/bundle/resources/jobs/update/out.plan_create.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.jobs.foo": { diff --git a/acceptance/bundle/resources/jobs/update/out.plan_skip.direct.json b/acceptance/bundle/resources/jobs/update/out.plan_skip.direct.json index 7aecad7102e..87975ef1bc0 100644 --- a/acceptance/bundle/resources/jobs/update/out.plan_skip.direct.json +++ b/acceptance/bundle/resources/jobs/update/out.plan_skip.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resources/jobs/update/out.plan_update.direct.json b/acceptance/bundle/resources/jobs/update/out.plan_update.direct.json index 56eb1211e99..e81f442dbab 100644 --- a/acceptance/bundle/resources/jobs/update/out.plan_update.direct.json +++ b/acceptance/bundle/resources/jobs/update/out.plan_update.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resources/jobs/update/script b/acceptance/bundle/resources/jobs/update/script index 1be869aa509..7a90cf02255 100644 --- a/acceptance/bundle/resources/jobs/update/script +++ b/acceptance/bundle/resources/jobs/update/script @@ -1,12 +1,12 @@ echo "*" > .gitignore trace $CLI bundle plan -$CLI bundle plan -o json > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp.py > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json $CLI bundle deploy $(readplanarg out.plan_create.direct.json) trace print_requests.py //jobs --nostamp > out.create.requests.json print_state.py > out.state.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle plan -trace $CLI bundle plan -o json > out.plan_skip.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.plan_skip.$DATABRICKS_BUNDLE_ENGINE.json $CLI bundle deploy $(readplanarg out.plan_skip.direct.json) trace print_requests.py //jobs --nostamp @@ -14,7 +14,7 @@ trace print_requests.py //jobs --nostamp title "Update trigger.periodic.unit and re-deploy" trace update_file.py databricks.yml DAYS HOURS trace $CLI bundle plan -$CLI bundle plan -o json > out.plan_update.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp.py > out.plan_update.$DATABRICKS_BUNDLE_ENGINE.json $CLI bundle deploy $(readplanarg out.plan_update.direct.json) trace print_requests.py //jobs --nostamp | jq 'del(.body.new_settings.run_as, .body.new_settings.webhook_notifications, .body.new_settings.email_notifications)' > out.update.requests.json diff --git a/acceptance/bundle/resources/jobs/update/test.toml b/acceptance/bundle/resources/jobs/update/test.toml index 20c8733641d..a8a43da097a 100644 --- a/acceptance/bundle/resources/jobs/update/test.toml +++ b/acceptance/bundle/resources/jobs/update/test.toml @@ -1,4 +1,4 @@ EnvMatrix.READPLAN = ["", "1"] -# `bundle deploy --plan` applies pre-saved state; the stamp isn't written into the plan, -# so the next plan reports it as a change. Recording skipped until stamp reaches saved plans. +# Dumps raw state/plan, which carry the DMS deployment stamp under recording (applied at deploy, +# not saved in the plan). deploy --plan recording is covered by bundle/dms, so opt out here. EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/resources/jobs/update_single_node/out.plan_create.direct.json b/acceptance/bundle/resources/jobs/update_single_node/out.plan_create.direct.json index e42c7b24c5f..622487a2539 100644 --- a/acceptance/bundle/resources/jobs/update_single_node/out.plan_create.direct.json +++ b/acceptance/bundle/resources/jobs/update_single_node/out.plan_create.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.jobs.foo": { diff --git a/acceptance/bundle/resources/jobs/update_single_node/out.plan_skip.direct.json b/acceptance/bundle/resources/jobs/update_single_node/out.plan_skip.direct.json index bed35fb5c8c..7e6179fd905 100644 --- a/acceptance/bundle/resources/jobs/update_single_node/out.plan_skip.direct.json +++ b/acceptance/bundle/resources/jobs/update_single_node/out.plan_skip.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resources/jobs/update_single_node/out.plan_update.direct.json b/acceptance/bundle/resources/jobs/update_single_node/out.plan_update.direct.json index 330ae5d5c17..3f616a3754e 100644 --- a/acceptance/bundle/resources/jobs/update_single_node/out.plan_update.direct.json +++ b/acceptance/bundle/resources/jobs/update_single_node/out.plan_update.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resources/jobs/update_single_node/script b/acceptance/bundle/resources/jobs/update_single_node/script index 4deefe32bbc..bfeb03571f8 100644 --- a/acceptance/bundle/resources/jobs/update_single_node/script +++ b/acceptance/bundle/resources/jobs/update_single_node/script @@ -1,15 +1,15 @@ echo "*" > .gitignore trace $CLI bundle plan -$CLI bundle plan -o json | nostamp > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp.py > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy -$CLI bundle plan -o json | nostamp > out.plan_skip.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp.py > out.plan_skip.$DATABRICKS_BUNDLE_ENGINE.json trace print_requests.py //jobs > out.create.requests.txt --nostamp title "Update trigger.periodic.unit and re-deploy" trace update_file.py databricks.yml DAYS HOURS trace $CLI bundle plan -$CLI bundle plan -o json | nostamp > out.plan_update.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp.py > out.plan_update.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py //jobs > out.update.requests.$DATABRICKS_BUNDLE_ENGINE.json --nostamp diff --git a/acceptance/bundle/resources/jobs/webhook-reorder-remote/script b/acceptance/bundle/resources/jobs/webhook-reorder-remote/script index c0257db6330..b3dbd3b782c 100644 --- a/acceptance/bundle/resources/jobs/webhook-reorder-remote/script +++ b/acceptance/bundle/resources/jobs/webhook-reorder-remote/script @@ -14,6 +14,6 @@ EOF # The reordered remote must not produce a phantom diff: on_* lists are diffed by id. trace $CLI bundle plan -$CLI bundle plan -o json | nostamp | jq '.plan."resources.jobs.my_job".changes' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp.py | jq '.plan."resources.jobs.my_job".changes' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json trace print_requests.py //jobs --nostamp diff --git a/acceptance/bundle/resources/model_serving_endpoints/basic/out.first-plan.direct.txt b/acceptance/bundle/resources/model_serving_endpoints/basic/out.first-plan.direct.txt index c1d4fe55d6e..f8f72b2d600 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/basic/out.first-plan.direct.txt +++ b/acceptance/bundle/resources/model_serving_endpoints/basic/out.first-plan.direct.txt @@ -1,4 +1,4 @@ -json.plan_version = 2; +json.plan_version = 3; json.cli_version = "[CLI_VERSION]"; json.plan.resources.model_serving_endpoints.my_endpoint.action = "create"; json.plan.resources.model_serving_endpoints.my_endpoint.new_state.value.name = "[ENDPOINT_NAME_1]"; diff --git a/acceptance/bundle/resources/model_serving_endpoints/basic/out.second-plan.direct.txt b/acceptance/bundle/resources/model_serving_endpoints/basic/out.second-plan.direct.txt index 7b4362bacbe..79919714725 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/basic/out.second-plan.direct.txt +++ b/acceptance/bundle/resources/model_serving_endpoints/basic/out.second-plan.direct.txt @@ -1,4 +1,4 @@ -json.plan_version = 2; +json.plan_version = 3; json.cli_version = "[CLI_VERSION]"; json.lineage = "[UUID]"; json.serial = 1; diff --git a/acceptance/bundle/resources/model_serving_endpoints/basic/script b/acceptance/bundle/resources/model_serving_endpoints/basic/script index 972439f6813..48b1f935b0c 100755 --- a/acceptance/bundle/resources/model_serving_endpoints/basic/script +++ b/acceptance/bundle/resources/model_serving_endpoints/basic/script @@ -14,7 +14,7 @@ trap cleanup EXIT # bundle/direct/dresources/permissions.go), so sorting keeps the plan golden # stable across runs. Same approach as secret_scopes/basic and the requests # sorting below. -trace $CLI bundle plan -o json | gron.py --sort-arrays __embed__ > out.first-plan.$DATABRICKS_BUNDLE_ENGINE.txt +trace $CLI bundle plan -o json | nostamp.py | gron.py --sort-arrays __embed__ > out.first-plan.$DATABRICKS_BUNDLE_ENGINE.txt trace $CLI bundle deploy -qq # Sort access_control_list: the terraform engine stores it as a set whose order # depends on the (unmasked) principal value, so it differs between the local fake @@ -33,7 +33,7 @@ echo "$get_output" | jq '{name, creator}' export ENDPOINT_NAME="test-endpoint-$UNIQUE_NAME-2" envsubst < databricks.yml.tmpl > databricks.yml -trace $CLI bundle plan -o json | gron.py --sort-arrays __embed__ > out.second-plan.$DATABRICKS_BUNDLE_ENGINE.txt +trace $CLI bundle plan -o json | nostamp.py | gron.py --sort-arrays __embed__ > out.second-plan.$DATABRICKS_BUNDLE_ENGINE.txt trace $CLI bundle deploy -qq print_requests.py //serving-endpoints | gron.py --sort-arrays access_control_list > out.second-requests.$DATABRICKS_BUNDLE_ENGINE.txt diff --git a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_unmanaged/script b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_unmanaged/script index 5e30339c21f..7faa6fd31f3 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_unmanaged/script +++ b/acceptance/bundle/resources/model_serving_endpoints/drift/telemetry_config_unmanaged/script @@ -7,7 +7,7 @@ title "Configure telemetry out of band" MSYS_NO_PATHCONV=1 $CLI api patch "/api/2.0/serving-endpoints/test-endpoint/telemetry-config" --json '{"telemetry_config": {"table_names": {"logs_table": "main.default.other_logs"}}}' > /dev/null title "Remote-only telemetry_config is not drift" -trace $CLI bundle plan --output json | jq '.plan[].changes | with_entries(select(.key | startswith("telemetry_config")))' +trace $CLI bundle plan --output json | nostamp.py | jq '.plan[].changes | with_entries(select(.key | startswith("telemetry_config")))' title "Deploy does not remove it" trace $CLI bundle deploy diff --git a/acceptance/bundle/resources/model_serving_endpoints/drift/write_only/script b/acceptance/bundle/resources/model_serving_endpoints/drift/write_only/script index dab44354e0e..03058a0cf1b 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/drift/write_only/script +++ b/acceptance/bundle/resources/model_serving_endpoints/drift/write_only/script @@ -7,7 +7,7 @@ title "Plan is a no-op despite write-only secret and burst_scaling_enabled not r trace $CLI bundle plan | contains.py "Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged" title "The write-only fields are skipped as input_only (confirms the matched rules)" -trace $CLI bundle plan --output json | jq '.plan[].changes' +trace $CLI bundle plan --output json | nostamp.py | jq '.plan[].changes' title "Redeploy is a no-op (no config/ai-gateway/tags update calls)" trace $CLI bundle deploy diff --git a/acceptance/bundle/resources/model_serving_endpoints/recreate/catalog-name/out.first-plan.direct.json b/acceptance/bundle/resources/model_serving_endpoints/recreate/catalog-name/out.first-plan.direct.json index a9353ddba55..79a8a77c640 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/recreate/catalog-name/out.first-plan.direct.json +++ b/acceptance/bundle/resources/model_serving_endpoints/recreate/catalog-name/out.first-plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.model_serving_endpoints.test_endpoint": { diff --git a/acceptance/bundle/resources/model_serving_endpoints/recreate/catalog-name/out.second-plan.direct.json b/acceptance/bundle/resources/model_serving_endpoints/recreate/catalog-name/out.second-plan.direct.json index 86932f72ce6..2143567be39 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/recreate/catalog-name/out.second-plan.direct.json +++ b/acceptance/bundle/resources/model_serving_endpoints/recreate/catalog-name/out.second-plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resources/model_serving_endpoints/recreate/catalog-name/script b/acceptance/bundle/resources/model_serving_endpoints/recreate/catalog-name/script index 919470c4ca4..a37c37a2df5 100755 --- a/acceptance/bundle/resources/model_serving_endpoints/recreate/catalog-name/script +++ b/acceptance/bundle/resources/model_serving_endpoints/recreate/catalog-name/script @@ -6,7 +6,7 @@ cleanup() { } trap cleanup EXIT -trace $CLI bundle plan -o json > out.first-plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.first-plan.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py //serving-endpoints @@ -16,7 +16,7 @@ add_repl "$ENDPOINT_ID" ORIGINAL_ENDPOINT_ID trace $CLI serving-endpoints get "${ENDPOINT_ID}" | jq '.config.auto_capture_config.catalog_name' trace update_file.py databricks.yml "catalog_name: main" "catalog_name: other_catalog" -trace $CLI bundle plan -o json > out.second-plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.second-plan.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py //serving-endpoints diff --git a/acceptance/bundle/resources/model_serving_endpoints/recreate/name-change/out.first-plan.direct.json b/acceptance/bundle/resources/model_serving_endpoints/recreate/name-change/out.first-plan.direct.json index c8e90eb2aad..b48b79aecf3 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/recreate/name-change/out.first-plan.direct.json +++ b/acceptance/bundle/resources/model_serving_endpoints/recreate/name-change/out.first-plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.model_serving_endpoints.test_endpoint": { diff --git a/acceptance/bundle/resources/model_serving_endpoints/recreate/name-change/out.second-plan.direct.json b/acceptance/bundle/resources/model_serving_endpoints/recreate/name-change/out.second-plan.direct.json index 3e424ed3fc8..e44a5467be8 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/recreate/name-change/out.second-plan.direct.json +++ b/acceptance/bundle/resources/model_serving_endpoints/recreate/name-change/out.second-plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resources/model_serving_endpoints/recreate/name-change/script b/acceptance/bundle/resources/model_serving_endpoints/recreate/name-change/script index d48a60c0ace..c27e1968877 100755 --- a/acceptance/bundle/resources/model_serving_endpoints/recreate/name-change/script +++ b/acceptance/bundle/resources/model_serving_endpoints/recreate/name-change/script @@ -6,7 +6,7 @@ cleanup() { } trap cleanup EXIT -trace $CLI bundle plan -o json > out.first-plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.first-plan.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py //serving-endpoints @@ -16,7 +16,7 @@ add_repl "$ENDPOINT_ID" ORIGINAL_ENDPOINT_ID trace $CLI serving-endpoints get "${ENDPOINT_ID}" | jq '.name' trace update_file.py databricks.yml "name: test-endpoint-$UNIQUE_NAME" "name: test-endpoint-2-$UNIQUE_NAME" -trace $CLI bundle plan -o json > out.second-plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.second-plan.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py //serving-endpoints diff --git a/acceptance/bundle/resources/model_serving_endpoints/recreate/route-optimized/out.first-plan.direct.json b/acceptance/bundle/resources/model_serving_endpoints/recreate/route-optimized/out.first-plan.direct.json index 32dd192daab..3e58aa755c8 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/recreate/route-optimized/out.first-plan.direct.json +++ b/acceptance/bundle/resources/model_serving_endpoints/recreate/route-optimized/out.first-plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.model_serving_endpoints.test_endpoint": { diff --git a/acceptance/bundle/resources/model_serving_endpoints/recreate/route-optimized/out.second-plan.direct.json b/acceptance/bundle/resources/model_serving_endpoints/recreate/route-optimized/out.second-plan.direct.json index f8dcb1c5510..19ed3fca2b1 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/recreate/route-optimized/out.second-plan.direct.json +++ b/acceptance/bundle/resources/model_serving_endpoints/recreate/route-optimized/out.second-plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resources/model_serving_endpoints/recreate/route-optimized/script b/acceptance/bundle/resources/model_serving_endpoints/recreate/route-optimized/script index 0b68be7c353..db80789d106 100755 --- a/acceptance/bundle/resources/model_serving_endpoints/recreate/route-optimized/script +++ b/acceptance/bundle/resources/model_serving_endpoints/recreate/route-optimized/script @@ -6,7 +6,7 @@ cleanup() { } trap cleanup EXIT -trace $CLI bundle plan -o json > out.first-plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.first-plan.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy ENDPOINT_ID=$($CLI bundle summary -o json | jq -r '.resources.model_serving_endpoints.test_endpoint.id') @@ -16,7 +16,7 @@ trace $CLI serving-endpoints get "${ENDPOINT_ID}" | jq '{route_optimized, id}' > print_requests.py //serving-endpoints > out.first-requests.$DATABRICKS_BUNDLE_ENGINE.json trace update_file.py databricks.yml "route_optimized: false" "route_optimized: true" -trace $CLI bundle plan -o json > out.second-plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.second-plan.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py //serving-endpoints diff --git a/acceptance/bundle/resources/model_serving_endpoints/recreate/schema-name/out.first-plan.direct.json b/acceptance/bundle/resources/model_serving_endpoints/recreate/schema-name/out.first-plan.direct.json index a9353ddba55..79a8a77c640 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/recreate/schema-name/out.first-plan.direct.json +++ b/acceptance/bundle/resources/model_serving_endpoints/recreate/schema-name/out.first-plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.model_serving_endpoints.test_endpoint": { diff --git a/acceptance/bundle/resources/model_serving_endpoints/recreate/schema-name/out.second-plan.direct.json b/acceptance/bundle/resources/model_serving_endpoints/recreate/schema-name/out.second-plan.direct.json index eeacb3be087..f7b210772cd 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/recreate/schema-name/out.second-plan.direct.json +++ b/acceptance/bundle/resources/model_serving_endpoints/recreate/schema-name/out.second-plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resources/model_serving_endpoints/recreate/schema-name/script b/acceptance/bundle/resources/model_serving_endpoints/recreate/schema-name/script index 0e31f7a4848..cc76c4428c6 100755 --- a/acceptance/bundle/resources/model_serving_endpoints/recreate/schema-name/script +++ b/acceptance/bundle/resources/model_serving_endpoints/recreate/schema-name/script @@ -6,7 +6,7 @@ cleanup() { } trap cleanup EXIT -trace $CLI bundle plan -o json > out.first-plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.first-plan.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py //serving-endpoints @@ -16,7 +16,7 @@ add_repl "$ENDPOINT_ID" ORIGINAL_ENDPOINT_ID trace $CLI serving-endpoints get "${ENDPOINT_ID}" | jq '.config.auto_capture_config.schema_name' trace update_file.py databricks.yml "schema_name: default" "schema_name: other_schema" -trace $CLI bundle plan -o json > out.second-plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.second-plan.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py //serving-endpoints diff --git a/acceptance/bundle/resources/model_serving_endpoints/recreate/table-prefix/out.first-plan.direct.json b/acceptance/bundle/resources/model_serving_endpoints/recreate/table-prefix/out.first-plan.direct.json index a9353ddba55..79a8a77c640 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/recreate/table-prefix/out.first-plan.direct.json +++ b/acceptance/bundle/resources/model_serving_endpoints/recreate/table-prefix/out.first-plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.model_serving_endpoints.test_endpoint": { diff --git a/acceptance/bundle/resources/model_serving_endpoints/recreate/table-prefix/out.second-plan.direct.json b/acceptance/bundle/resources/model_serving_endpoints/recreate/table-prefix/out.second-plan.direct.json index e29d17e4dde..058747f3d60 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/recreate/table-prefix/out.second-plan.direct.json +++ b/acceptance/bundle/resources/model_serving_endpoints/recreate/table-prefix/out.second-plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resources/model_serving_endpoints/recreate/table-prefix/script b/acceptance/bundle/resources/model_serving_endpoints/recreate/table-prefix/script index 7f915920ba1..079bc9a33c9 100755 --- a/acceptance/bundle/resources/model_serving_endpoints/recreate/table-prefix/script +++ b/acceptance/bundle/resources/model_serving_endpoints/recreate/table-prefix/script @@ -6,7 +6,7 @@ cleanup() { } trap cleanup EXIT -trace $CLI bundle plan -o json > out.first-plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.first-plan.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py //serving-endpoints @@ -16,7 +16,7 @@ add_repl "$ENDPOINT_ID" ORIGINAL_ENDPOINT_ID trace $CLI serving-endpoints get "${ENDPOINT_ID}" | jq '.config.auto_capture_config.table_name_prefix' trace update_file.py databricks.yml "table_name_prefix: my_table" "table_name_prefix: other_table" -trace $CLI bundle plan -o json > out.second-plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.second-plan.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py //serving-endpoints diff --git a/acceptance/bundle/resources/model_serving_endpoints/running-endpoint/out.plan.direct.txt b/acceptance/bundle/resources/model_serving_endpoints/running-endpoint/out.plan.direct.txt index 6c3233c9ede..71522c2d87e 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/running-endpoint/out.plan.direct.txt +++ b/acceptance/bundle/resources/model_serving_endpoints/running-endpoint/out.plan.direct.txt @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.model_serving_endpoints.my_endpoint": { diff --git a/acceptance/bundle/resources/model_serving_endpoints/running-endpoint/script b/acceptance/bundle/resources/model_serving_endpoints/running-endpoint/script index 8b420559c3d..8f2e49038ed 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/running-endpoint/script +++ b/acceptance/bundle/resources/model_serving_endpoints/running-endpoint/script @@ -8,7 +8,7 @@ cleanup() { } trap cleanup EXIT -trace $CLI bundle plan -o json > out.plan.$DATABRICKS_BUNDLE_ENGINE.txt +trace $CLI bundle plan -o json | nostamp.py > out.plan.$DATABRICKS_BUNDLE_ENGINE.txt trace $CLI bundle deploy trace print_requests.py //serving-endpoints diff --git a/acceptance/bundle/resources/model_serving_endpoints/update/ai-gateway/out.plan.direct.json b/acceptance/bundle/resources/model_serving_endpoints/update/ai-gateway/out.plan.direct.json index f305ce97719..3f93160a2ab 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/update/ai-gateway/out.plan.direct.json +++ b/acceptance/bundle/resources/model_serving_endpoints/update/ai-gateway/out.plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resources/model_serving_endpoints/update/ai-gateway/script b/acceptance/bundle/resources/model_serving_endpoints/update/ai-gateway/script index 6e556d2b1ab..3f942f4d306 100755 --- a/acceptance/bundle/resources/model_serving_endpoints/update/ai-gateway/script +++ b/acceptance/bundle/resources/model_serving_endpoints/update/ai-gateway/script @@ -15,7 +15,7 @@ trace $CLI serving-endpoints get "${ENDPOINT_ID}" | jq '.ai_gateway' trace update_file.py databricks.yml 'catalog_name: "first-inference-catalog"' 'catalog_name: "second-inference-catalog"' -trace $CLI bundle plan -o json > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.plan.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py //serving-endpoints diff --git a/acceptance/bundle/resources/model_serving_endpoints/update/both_gateway_and_tags/out.plan.direct.json b/acceptance/bundle/resources/model_serving_endpoints/update/both_gateway_and_tags/out.plan.direct.json index 394377f07f8..32f88590a01 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/update/both_gateway_and_tags/out.plan.direct.json +++ b/acceptance/bundle/resources/model_serving_endpoints/update/both_gateway_and_tags/out.plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resources/model_serving_endpoints/update/both_gateway_and_tags/script b/acceptance/bundle/resources/model_serving_endpoints/update/both_gateway_and_tags/script index 763379ad9a6..c9b81cd0d6d 100755 --- a/acceptance/bundle/resources/model_serving_endpoints/update/both_gateway_and_tags/script +++ b/acceptance/bundle/resources/model_serving_endpoints/update/both_gateway_and_tags/script @@ -17,7 +17,7 @@ trace $CLI serving-endpoints get "${ENDPOINT_ID}" | jq '.config' trace update_file.py databricks.yml "value: my-team-one" "value: my-team-two" trace update_file.py databricks.yml 'catalog_name: "first-inference-catalog"' 'catalog_name: "second-inference-catalog"' -trace $CLI bundle plan -o json > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.plan.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py //serving-endpoints diff --git a/acceptance/bundle/resources/model_serving_endpoints/update/config/out.plan.direct.json b/acceptance/bundle/resources/model_serving_endpoints/update/config/out.plan.direct.json index 47e992c1cf9..b4d67db672b 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/update/config/out.plan.direct.json +++ b/acceptance/bundle/resources/model_serving_endpoints/update/config/out.plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resources/model_serving_endpoints/update/config/script b/acceptance/bundle/resources/model_serving_endpoints/update/config/script index 3a28d1dcda2..c4656a110f8 100755 --- a/acceptance/bundle/resources/model_serving_endpoints/update/config/script +++ b/acceptance/bundle/resources/model_serving_endpoints/update/config/script @@ -15,7 +15,7 @@ trace $CLI serving-endpoints get "${ENDPOINT_ID}" | jq '.config' trace update_file.py databricks.yml "name: gpt-4o-mini" "name: gpt-5o-mini" -trace $CLI bundle plan -o json > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.plan.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py //serving-endpoints diff --git a/acceptance/bundle/resources/model_serving_endpoints/update/email-notifications/out.plan.direct.json b/acceptance/bundle/resources/model_serving_endpoints/update/email-notifications/out.plan.direct.json index cf8c8b3f0ca..1ff5746e670 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/update/email-notifications/out.plan.direct.json +++ b/acceptance/bundle/resources/model_serving_endpoints/update/email-notifications/out.plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resources/model_serving_endpoints/update/email-notifications/script b/acceptance/bundle/resources/model_serving_endpoints/update/email-notifications/script index 391d369d686..f049becec06 100755 --- a/acceptance/bundle/resources/model_serving_endpoints/update/email-notifications/script +++ b/acceptance/bundle/resources/model_serving_endpoints/update/email-notifications/script @@ -16,7 +16,7 @@ trace $CLI serving-endpoints get "${ENDPOINT_ID}" | jq '.email_notifications' # Update email_notifications - change the email address trace update_file.py databricks.yml "user1@example.com" "user2@example.com" -trace $CLI bundle plan -o json > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.plan.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy title "There a bug in TF where it does not actually update the email notifications for a serving endpoint." diff --git a/acceptance/bundle/resources/model_serving_endpoints/update/tags/out.plan.direct.json b/acceptance/bundle/resources/model_serving_endpoints/update/tags/out.plan.direct.json index 73202db539d..4e1999adb7e 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/update/tags/out.plan.direct.json +++ b/acceptance/bundle/resources/model_serving_endpoints/update/tags/out.plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resources/model_serving_endpoints/update/tags/script b/acceptance/bundle/resources/model_serving_endpoints/update/tags/script index 6f69dc2f063..01c0e26eedd 100755 --- a/acceptance/bundle/resources/model_serving_endpoints/update/tags/script +++ b/acceptance/bundle/resources/model_serving_endpoints/update/tags/script @@ -15,7 +15,7 @@ trace $CLI serving-endpoints get "${ENDPOINT_ID}" | jq '.tags' trace update_file.py databricks.yml "value: my-team-one" "value: my-team-two" -trace $CLI bundle plan -o json > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.plan.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py //serving-endpoints diff --git a/acceptance/bundle/resources/permissions/_script b/acceptance/bundle/resources/permissions/_script index 8f6a0d9c8cd..df20dfd2fa9 100644 --- a/acceptance/bundle/resources/permissions/_script +++ b/acceptance/bundle/resources/permissions/_script @@ -1,7 +1,7 @@ -trace $CLI bundle validate -o json | nostamp | jq .resources.$RESOURCE.foo.permissions +trace $CLI bundle validate -o json | nostamp.py | jq .resources.$RESOURCE.foo.permissions rm out.requests.txt -$CLI bundle plan -o json | nostamp > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp.py > out.plan.$DATABRICKS_BUNDLE_ENGINE.json print_requests() { jq -c < out.requests.txt | jq 'select(.method != "GET" and (.path | contains("permissions")) and (.path | contains("/api/2.0/bundle") | not))' diff --git a/acceptance/bundle/resources/permissions/apps/current_can_manage/out.plan.direct.json b/acceptance/bundle/resources/permissions/apps/current_can_manage/out.plan.direct.json index 6774f270231..718499a0f66 100644 --- a/acceptance/bundle/resources/permissions/apps/current_can_manage/out.plan.direct.json +++ b/acceptance/bundle/resources/permissions/apps/current_can_manage/out.plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.apps.foo": { diff --git a/acceptance/bundle/resources/permissions/apps/other_can_manage/out.plan.direct.json b/acceptance/bundle/resources/permissions/apps/other_can_manage/out.plan.direct.json index 6774f270231..718499a0f66 100644 --- a/acceptance/bundle/resources/permissions/apps/other_can_manage/out.plan.direct.json +++ b/acceptance/bundle/resources/permissions/apps/other_can_manage/out.plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.apps.foo": { diff --git a/acceptance/bundle/resources/permissions/clusters/current_can_manage/out.plan.direct.json b/acceptance/bundle/resources/permissions/clusters/current_can_manage/out.plan.direct.json index bd13e780112..84f6bcd2feb 100644 --- a/acceptance/bundle/resources/permissions/clusters/current_can_manage/out.plan.direct.json +++ b/acceptance/bundle/resources/permissions/clusters/current_can_manage/out.plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.clusters.foo": { diff --git a/acceptance/bundle/resources/permissions/dashboards/create/out.plan.direct.json b/acceptance/bundle/resources/permissions/dashboards/create/out.plan.direct.json index 423fa9d0eb8..5793e74dd90 100644 --- a/acceptance/bundle/resources/permissions/dashboards/create/out.plan.direct.json +++ b/acceptance/bundle/resources/permissions/dashboards/create/out.plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.dashboards.foo": { diff --git a/acceptance/bundle/resources/permissions/dashboards/create/script b/acceptance/bundle/resources/permissions/dashboards/create/script index 41f26f63604..2d578386e6a 100644 --- a/acceptance/bundle/resources/permissions/dashboards/create/script +++ b/acceptance/bundle/resources/permissions/dashboards/create/script @@ -7,7 +7,7 @@ envsubst < databricks.yml.tmpl > databricks.yml trace $CLI bundle validate -o json | jq .resources.dashboards.foo.permissions rm out.requests.txt -$CLI bundle plan --output json > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan --output json | nostamp.py > out.plan.$DATABRICKS_BUNDLE_ENGINE.json rm out.requests.txt trace $CLI bundle deploy diff --git a/acceptance/bundle/resources/permissions/database_instances/current_can_manage/out.plan.direct.json b/acceptance/bundle/resources/permissions/database_instances/current_can_manage/out.plan.direct.json index cc7dfa8e463..35dfda54426 100644 --- a/acceptance/bundle/resources/permissions/database_instances/current_can_manage/out.plan.direct.json +++ b/acceptance/bundle/resources/permissions/database_instances/current_can_manage/out.plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.database_instances.foo": { diff --git a/acceptance/bundle/resources/permissions/experiments/current_can_manage/out.plan.direct.json b/acceptance/bundle/resources/permissions/experiments/current_can_manage/out.plan.direct.json index 6ce406b3d28..de8c34ac0ca 100644 --- a/acceptance/bundle/resources/permissions/experiments/current_can_manage/out.plan.direct.json +++ b/acceptance/bundle/resources/permissions/experiments/current_can_manage/out.plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.experiments.foo": { diff --git a/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/out.plan.direct.json b/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/out.plan.direct.json index 90736028eb4..fd4fc45a826 100644 --- a/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/out.plan.direct.json +++ b/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/out.plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.genie_spaces.foo": { diff --git a/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/script b/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/script index e589a40bd5b..e681e725cde 100644 --- a/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/script +++ b/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/script @@ -1,7 +1,7 @@ trace $CLI bundle validate -o json | jq .resources.genie_spaces.foo.permissions rm out.requests.txt -$CLI bundle plan -o json | nostamp > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp.py > out.plan.$DATABRICKS_BUNDLE_ENGINE.json rm out.requests.txt trace $CLI bundle deploy diff --git a/acceptance/bundle/resources/permissions/jobs/added_remotely/out.plan.direct.json b/acceptance/bundle/resources/permissions/jobs/added_remotely/out.plan.direct.json index f048f3e6f48..7185b30f356 100644 --- a/acceptance/bundle/resources/permissions/jobs/added_remotely/out.plan.direct.json +++ b/acceptance/bundle/resources/permissions/jobs/added_remotely/out.plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resources/permissions/jobs/added_remotely/script b/acceptance/bundle/resources/permissions/jobs/added_remotely/script index e85bcde5379..3b872d6446b 100644 --- a/acceptance/bundle/resources/permissions/jobs/added_remotely/script +++ b/acceptance/bundle/resources/permissions/jobs/added_remotely/script @@ -10,7 +10,7 @@ title "Add permissions out of band" trace $CLI permissions set jobs "$job_id" --json @remote_add.json trace $CLI bundle plan -trace $CLI bundle plan -o json | nostamp > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.plan.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace $CLI bundle plan diff --git a/acceptance/bundle/resources/permissions/jobs/current_can_manage/out.plan.direct.json b/acceptance/bundle/resources/permissions/jobs/current_can_manage/out.plan.direct.json index b85fdd51a06..9b131bdc4af 100644 --- a/acceptance/bundle/resources/permissions/jobs/current_can_manage/out.plan.direct.json +++ b/acceptance/bundle/resources/permissions/jobs/current_can_manage/out.plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.jobs.foo": { diff --git a/acceptance/bundle/resources/permissions/jobs/current_can_manage_run/out.plan.direct.txt b/acceptance/bundle/resources/permissions/jobs/current_can_manage_run/out.plan.direct.txt index 4b289dbd158..d3c2e3afdca 100644 --- a/acceptance/bundle/resources/permissions/jobs/current_can_manage_run/out.plan.direct.txt +++ b/acceptance/bundle/resources/permissions/jobs/current_can_manage_run/out.plan.direct.txt @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resources/permissions/jobs/current_can_manage_run/script b/acceptance/bundle/resources/permissions/jobs/current_can_manage_run/script index 3c8217c195c..529b94814fa 100644 --- a/acceptance/bundle/resources/permissions/jobs/current_can_manage_run/script +++ b/acceptance/bundle/resources/permissions/jobs/current_can_manage_run/script @@ -1,11 +1,11 @@ envsubst < databricks.yml.tmpl > databricks.yml -trace $CLI bundle validate -t green -o json | nostamp | jq .resources +trace $CLI bundle validate -t green -o json | nostamp.py | jq .resources trace errcode $CLI bundle deploy -t green print_requests.py //jobs --nostamp &> out.deploy.requests.json # check plan to ensure there is not drift -trace $CLI bundle plan -o json -t green | nostamp > out.plan.$DATABRICKS_BUNDLE_ENGINE.txt +trace $CLI bundle plan -o json -t green | nostamp.py > out.plan.$DATABRICKS_BUNDLE_ENGINE.txt trace errcode $CLI bundle destroy -t green --auto-approve print_requests.py //jobs --nostamp &> out.destroy.requests.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/permissions/jobs/current_is_owner/out.plan.direct.json b/acceptance/bundle/resources/permissions/jobs/current_is_owner/out.plan.direct.json index 88e1e115a9c..16944e5b4dd 100644 --- a/acceptance/bundle/resources/permissions/jobs/current_is_owner/out.plan.direct.json +++ b/acceptance/bundle/resources/permissions/jobs/current_is_owner/out.plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.jobs.foo": { diff --git a/acceptance/bundle/resources/permissions/jobs/delete_one/out.plan_create.direct.json b/acceptance/bundle/resources/permissions/jobs/delete_one/out.plan_create.direct.json index a72e11dc488..74cc6c46453 100644 --- a/acceptance/bundle/resources/permissions/jobs/delete_one/out.plan_create.direct.json +++ b/acceptance/bundle/resources/permissions/jobs/delete_one/out.plan_create.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.jobs.job_with_permissions": { diff --git a/acceptance/bundle/resources/permissions/jobs/delete_one/script b/acceptance/bundle/resources/permissions/jobs/delete_one/script index 1f0aa1c80a7..82cd9d0aa94 100644 --- a/acceptance/bundle/resources/permissions/jobs/delete_one/script +++ b/acceptance/bundle/resources/permissions/jobs/delete_one/script @@ -8,7 +8,7 @@ if [ -n "$CLOUD_ENV" ]; then fi rm -f out.requests.txt -trace $CLI bundle plan -o json | nostamp > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json cleanup() { trace errcode $CLI bundle destroy --auto-approve diff --git a/acceptance/bundle/resources/permissions/jobs/deleted_remotely/out.plan_create.direct.json b/acceptance/bundle/resources/permissions/jobs/deleted_remotely/out.plan_create.direct.json index bd43de04ba7..ceb3d6d81cf 100644 --- a/acceptance/bundle/resources/permissions/jobs/deleted_remotely/out.plan_create.direct.json +++ b/acceptance/bundle/resources/permissions/jobs/deleted_remotely/out.plan_create.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.jobs.job_with_permissions": { diff --git a/acceptance/bundle/resources/permissions/jobs/deleted_remotely/out.plan_restore.direct.json b/acceptance/bundle/resources/permissions/jobs/deleted_remotely/out.plan_restore.direct.json index cda86e4ceb4..82626507c1a 100644 --- a/acceptance/bundle/resources/permissions/jobs/deleted_remotely/out.plan_restore.direct.json +++ b/acceptance/bundle/resources/permissions/jobs/deleted_remotely/out.plan_restore.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resources/permissions/jobs/deleted_remotely/script b/acceptance/bundle/resources/permissions/jobs/deleted_remotely/script index cce4d71d0ce..f91513db46e 100644 --- a/acceptance/bundle/resources/permissions/jobs/deleted_remotely/script +++ b/acceptance/bundle/resources/permissions/jobs/deleted_remotely/script @@ -1,4 +1,4 @@ -trace $CLI bundle plan -o json | nostamp > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py --nostamp //jobs/ > out.requests_create.json @@ -11,7 +11,7 @@ title "Delete permissions remotely" trace $CLI permissions set jobs "$job_id" --json @remote_delete.json trace print_requests.py --nostamp //jobs/ -trace $CLI bundle plan -o json | nostamp > out.plan_restore.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.plan_restore.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py --nostamp //jobs/ > out.requests_restore.json diff --git a/acceptance/bundle/resources/permissions/jobs/destroy_without_mgmtperms/test.toml b/acceptance/bundle/resources/permissions/jobs/destroy_without_mgmtperms/test.toml index 921045dfd09..4ca61be3f36 100644 --- a/acceptance/bundle/resources/permissions/jobs/destroy_without_mgmtperms/test.toml +++ b/acceptance/bundle/resources/permissions/jobs/destroy_without_mgmtperms/test.toml @@ -1,4 +1,3 @@ -# Recording needs a bundle it has seen from the start. This test seeds a state file, -# so recording refuses it. TODO(DMS): drop this once existing state can be -# handed over to the service (see the TODO in dstate.Open). +# Deploy runs as the service principal, later commands as the user - who resolves no deployment of +# its own, so bundle summary reads the job id back null. Recording needs state staged for the reader. EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/resources/permissions/jobs/empty_list/out.plan.direct.json b/acceptance/bundle/resources/permissions/jobs/empty_list/out.plan.direct.json index 0ec914589b1..d8a4d90a9b0 100644 --- a/acceptance/bundle/resources/permissions/jobs/empty_list/out.plan.direct.json +++ b/acceptance/bundle/resources/permissions/jobs/empty_list/out.plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.jobs.foo": { diff --git a/acceptance/bundle/resources/permissions/jobs/other_can_manage/out.plan.direct.json b/acceptance/bundle/resources/permissions/jobs/other_can_manage/out.plan.direct.json index b85fdd51a06..9b131bdc4af 100644 --- a/acceptance/bundle/resources/permissions/jobs/other_can_manage/out.plan.direct.json +++ b/acceptance/bundle/resources/permissions/jobs/other_can_manage/out.plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.jobs.foo": { diff --git a/acceptance/bundle/resources/permissions/jobs/other_is_owner/out.plan.direct.json b/acceptance/bundle/resources/permissions/jobs/other_is_owner/out.plan.direct.json index 2976bb3ebe6..5d47f0cc8ec 100644 --- a/acceptance/bundle/resources/permissions/jobs/other_is_owner/out.plan.direct.json +++ b/acceptance/bundle/resources/permissions/jobs/other_is_owner/out.plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.jobs.foo": { diff --git a/acceptance/bundle/resources/permissions/jobs/update/out.plan_create.direct.json b/acceptance/bundle/resources/permissions/jobs/update/out.plan_create.direct.json index 42bd247c77b..993098a0ee6 100644 --- a/acceptance/bundle/resources/permissions/jobs/update/out.plan_create.direct.json +++ b/acceptance/bundle/resources/permissions/jobs/update/out.plan_create.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.jobs.job_with_permissions": { diff --git a/acceptance/bundle/resources/permissions/jobs/update/out.plan_delete_all.direct.json b/acceptance/bundle/resources/permissions/jobs/update/out.plan_delete_all.direct.json index e7bfda922b8..f217c1f7185 100644 --- a/acceptance/bundle/resources/permissions/jobs/update/out.plan_delete_all.direct.json +++ b/acceptance/bundle/resources/permissions/jobs/update/out.plan_delete_all.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 3, diff --git a/acceptance/bundle/resources/permissions/jobs/update/out.plan_delete_one.direct.json b/acceptance/bundle/resources/permissions/jobs/update/out.plan_delete_one.direct.json index a2e93e298eb..5c1c6301698 100644 --- a/acceptance/bundle/resources/permissions/jobs/update/out.plan_delete_one.direct.json +++ b/acceptance/bundle/resources/permissions/jobs/update/out.plan_delete_one.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 2, diff --git a/acceptance/bundle/resources/permissions/jobs/update/out.plan_post_create.direct.json b/acceptance/bundle/resources/permissions/jobs/update/out.plan_post_create.direct.json index 02ffe2db63a..0402134e14c 100644 --- a/acceptance/bundle/resources/permissions/jobs/update/out.plan_post_create.direct.json +++ b/acceptance/bundle/resources/permissions/jobs/update/out.plan_post_create.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resources/permissions/jobs/update/out.plan_restore.direct.json b/acceptance/bundle/resources/permissions/jobs/update/out.plan_restore.direct.json index 1b25b762637..49058decfa6 100644 --- a/acceptance/bundle/resources/permissions/jobs/update/out.plan_restore.direct.json +++ b/acceptance/bundle/resources/permissions/jobs/update/out.plan_restore.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 4, diff --git a/acceptance/bundle/resources/permissions/jobs/update/out.plan_set_empty.direct.json b/acceptance/bundle/resources/permissions/jobs/update/out.plan_set_empty.direct.json index 57ee99c53f4..2c6d989004f 100644 --- a/acceptance/bundle/resources/permissions/jobs/update/out.plan_set_empty.direct.json +++ b/acceptance/bundle/resources/permissions/jobs/update/out.plan_set_empty.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 5, diff --git a/acceptance/bundle/resources/permissions/jobs/update/out.plan_update.direct.json b/acceptance/bundle/resources/permissions/jobs/update/out.plan_update.direct.json index 89b3cf0f302..8cce904f66b 100644 --- a/acceptance/bundle/resources/permissions/jobs/update/out.plan_update.direct.json +++ b/acceptance/bundle/resources/permissions/jobs/update/out.plan_update.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resources/permissions/jobs/update/script b/acceptance/bundle/resources/permissions/jobs/update/script index 28f2c121f2b..fb88499c271 100644 --- a/acceptance/bundle/resources/permissions/jobs/update/script +++ b/acceptance/bundle/resources/permissions/jobs/update/script @@ -1,10 +1,10 @@ cp databricks.yml databricks.yml.saved trace $CLI bundle plan -trace $CLI bundle plan -o json | nostamp > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py --nostamp //jobs/ > out.requests_create.json trace $CLI bundle plan -trace $CLI bundle plan -o json | nostamp > out.plan_post_create.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.plan_post_create.$DATABRICKS_BUNDLE_ENGINE.json job_id="$(read_id.py job_with_permissions)" @@ -13,21 +13,21 @@ rm -f out.requests.txt title "Update one permission and deploy again\n" update_file.py databricks.yml CAN_VIEW CAN_MANAGE -trace $CLI bundle plan -o json | nostamp > out.plan_update.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.plan_update.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py --nostamp //jobs/ > out.requests_update.json trace $CLI bundle plan title "Delete one permission and deploy again\n" grep -v DELETE_ONE databricks.yml > tmp.yml && mv tmp.yml databricks.yml -trace $CLI bundle plan -o json | nostamp > out.plan_delete_one.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.plan_delete_one.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py --nostamp //jobs/ > out.requests_delete_one.json trace $CLI bundle plan title "Delete the whole block and deploy again\n" grep -v PERMISSIONS databricks.yml > tmp.yml && mv tmp.yml databricks.yml -trace $CLI bundle plan -o json | nostamp > out.plan_delete_all.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.plan_delete_all.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py --nostamp //jobs/ > out.requests_delete_all.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle plan @@ -36,7 +36,7 @@ title "Restore original config\n" trace diff.py databricks.yml.saved databricks.yml mv databricks.yml.saved databricks.yml -trace $CLI bundle plan -o json | nostamp > out.plan_restore.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.plan_restore.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py --nostamp //jobs/ > out.requests_restore_original.json trace $CLI bundle plan @@ -44,7 +44,7 @@ trace $CLI bundle plan title "Set permissions: []\n" grep -vE '(PERMISSIONS|DELETE_ONE)' databricks.yml > tmp.yml && mv tmp.yml databricks.yml update_file.py databricks.yml '# permissions: [] # EXPLICIT_EMPTY' 'permissions: [] # EXPLICIT_EMPTY' -trace $CLI bundle plan -o json | nostamp > out.plan_set_empty.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.plan_set_empty.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py --nostamp //jobs/ > out.requests_set_empty.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/permissions/jobs/viewers/out.plan.direct.json b/acceptance/bundle/resources/permissions/jobs/viewers/out.plan.direct.json index 151138ddc8b..a783dd0aa59 100644 --- a/acceptance/bundle/resources/permissions/jobs/viewers/out.plan.direct.json +++ b/acceptance/bundle/resources/permissions/jobs/viewers/out.plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.jobs.foo": { diff --git a/acceptance/bundle/resources/permissions/models/current_can_manage/out.plan.direct.json b/acceptance/bundle/resources/permissions/models/current_can_manage/out.plan.direct.json index 268172af1af..ad2ad5b9245 100644 --- a/acceptance/bundle/resources/permissions/models/current_can_manage/out.plan.direct.json +++ b/acceptance/bundle/resources/permissions/models/current_can_manage/out.plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.models.foo": { diff --git a/acceptance/bundle/resources/permissions/models/current_can_manage/script b/acceptance/bundle/resources/permissions/models/current_can_manage/script index 0a6935c20b8..8159841e4e3 100644 --- a/acceptance/bundle/resources/permissions/models/current_can_manage/script +++ b/acceptance/bundle/resources/permissions/models/current_can_manage/script @@ -1,7 +1,7 @@ trace $CLI bundle validate -o json | jq .resources.$RESOURCE.foo.permissions rm out.requests.txt -$CLI bundle plan -o json | nostamp > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp.py > out.plan.$DATABRICKS_BUNDLE_ENGINE.json rm out.requests.txt trace errcode $CLI bundle deploy &> out.deploy.txt diff --git a/acceptance/bundle/resources/permissions/pipelines/current_can_manage/out.plan.direct.json b/acceptance/bundle/resources/permissions/pipelines/current_can_manage/out.plan.direct.json index 41c0fc3ba64..38cad367876 100644 --- a/acceptance/bundle/resources/permissions/pipelines/current_can_manage/out.plan.direct.json +++ b/acceptance/bundle/resources/permissions/pipelines/current_can_manage/out.plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.pipelines.foo": { diff --git a/acceptance/bundle/resources/permissions/pipelines/current_is_owner/out.plan.direct.json b/acceptance/bundle/resources/permissions/pipelines/current_is_owner/out.plan.direct.json index 029250bd21a..56c72cff931 100644 --- a/acceptance/bundle/resources/permissions/pipelines/current_is_owner/out.plan.direct.json +++ b/acceptance/bundle/resources/permissions/pipelines/current_is_owner/out.plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.pipelines.foo": { diff --git a/acceptance/bundle/resources/permissions/pipelines/empty_list/out.plan.direct.json b/acceptance/bundle/resources/permissions/pipelines/empty_list/out.plan.direct.json index 97493239403..8daa49692ad 100644 --- a/acceptance/bundle/resources/permissions/pipelines/empty_list/out.plan.direct.json +++ b/acceptance/bundle/resources/permissions/pipelines/empty_list/out.plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.pipelines.foo": { diff --git a/acceptance/bundle/resources/permissions/pipelines/other_can_manage/out.plan.direct.json b/acceptance/bundle/resources/permissions/pipelines/other_can_manage/out.plan.direct.json index 41c0fc3ba64..38cad367876 100644 --- a/acceptance/bundle/resources/permissions/pipelines/other_can_manage/out.plan.direct.json +++ b/acceptance/bundle/resources/permissions/pipelines/other_can_manage/out.plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.pipelines.foo": { diff --git a/acceptance/bundle/resources/permissions/pipelines/other_is_owner/out.plan.direct.json b/acceptance/bundle/resources/permissions/pipelines/other_is_owner/out.plan.direct.json index 240341258e8..e696913fa49 100644 --- a/acceptance/bundle/resources/permissions/pipelines/other_is_owner/out.plan.direct.json +++ b/acceptance/bundle/resources/permissions/pipelines/other_is_owner/out.plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.pipelines.foo": { diff --git a/acceptance/bundle/resources/permissions/pipelines/update/out.plan_create.direct.json b/acceptance/bundle/resources/permissions/pipelines/update/out.plan_create.direct.json index 2ee78e63631..ba600d05faa 100644 --- a/acceptance/bundle/resources/permissions/pipelines/update/out.plan_create.direct.json +++ b/acceptance/bundle/resources/permissions/pipelines/update/out.plan_create.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.pipelines.foo": { diff --git a/acceptance/bundle/resources/permissions/pipelines/update/out.plan_delete_all.direct.json b/acceptance/bundle/resources/permissions/pipelines/update/out.plan_delete_all.direct.json index db74caabfca..ac3785d797b 100644 --- a/acceptance/bundle/resources/permissions/pipelines/update/out.plan_delete_all.direct.json +++ b/acceptance/bundle/resources/permissions/pipelines/update/out.plan_delete_all.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 3, diff --git a/acceptance/bundle/resources/permissions/pipelines/update/out.plan_delete_one.direct.json b/acceptance/bundle/resources/permissions/pipelines/update/out.plan_delete_one.direct.json index 4586f292b32..c3518ec2310 100644 --- a/acceptance/bundle/resources/permissions/pipelines/update/out.plan_delete_one.direct.json +++ b/acceptance/bundle/resources/permissions/pipelines/update/out.plan_delete_one.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 2, diff --git a/acceptance/bundle/resources/permissions/pipelines/update/out.plan_restore.direct.json b/acceptance/bundle/resources/permissions/pipelines/update/out.plan_restore.direct.json index 7e20625e706..6ee152943b7 100644 --- a/acceptance/bundle/resources/permissions/pipelines/update/out.plan_restore.direct.json +++ b/acceptance/bundle/resources/permissions/pipelines/update/out.plan_restore.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 4, diff --git a/acceptance/bundle/resources/permissions/pipelines/update/out.plan_update.direct.json b/acceptance/bundle/resources/permissions/pipelines/update/out.plan_update.direct.json index f4419563280..a43fcd03904 100644 --- a/acceptance/bundle/resources/permissions/pipelines/update/out.plan_update.direct.json +++ b/acceptance/bundle/resources/permissions/pipelines/update/out.plan_update.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resources/permissions/pipelines/update/script b/acceptance/bundle/resources/permissions/pipelines/update/script index 9ace44addb4..33224b30de3 100644 --- a/acceptance/bundle/resources/permissions/pipelines/update/script +++ b/acceptance/bundle/resources/permissions/pipelines/update/script @@ -1,5 +1,5 @@ cp databricks.yml databricks.yml.saved -trace $CLI bundle plan -o json | nostamp > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py --nostamp //pipeline '^//api/2.0/bundle' > out.requests_create.json trace $CLI bundle plan @@ -11,14 +11,14 @@ rm -f out.requests.txt title "Update one permission and deploy again\n" update_file.py databricks.yml CAN_VIEW CAN_MANAGE -trace $CLI bundle plan -o json | nostamp > out.plan_update.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.plan_update.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py --nostamp //pipeline '^//api/2.0/bundle' > out.requests_update.json trace $CLI bundle plan title "Delete one permission and deploy again\n" grep -v DELETE_ONE databricks.yml > tmp.yml && mv tmp.yml databricks.yml -trace $CLI bundle plan -o json | nostamp > out.plan_delete_one.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.plan_delete_one.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py --nostamp //pipeline '^//api/2.0/bundle' > out.requests_delete_one.json trace $CLI bundle plan @@ -26,14 +26,14 @@ trace $CLI bundle plan title "Delete the whole block and deploy again\n" grep -v PERMISSIONS databricks.yml > tmp.yml && mv tmp.yml databricks.yml trace cat databricks.yml -trace $CLI bundle plan -o json | nostamp > out.plan_delete_all.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.plan_delete_all.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py --nostamp //pipeline '^//api/2.0/bundle' --sort > out.requests_delete_all.json trace $CLI bundle plan title "Restore original config\n" mv databricks.yml.saved databricks.yml -trace $CLI bundle plan -o json | nostamp > out.plan_restore.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.plan_restore.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py --nostamp //pipeline '^//api/2.0/bundle' > out.requests_restore_original.json trace $CLI bundle plan diff --git a/acceptance/bundle/resources/permissions/postgres_projects/current_can_manage/out.plan.direct.json b/acceptance/bundle/resources/permissions/postgres_projects/current_can_manage/out.plan.direct.json index 302eeb3c913..0a0293bf156 100644 --- a/acceptance/bundle/resources/permissions/postgres_projects/current_can_manage/out.plan.direct.json +++ b/acceptance/bundle/resources/permissions/postgres_projects/current_can_manage/out.plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.postgres_projects.foo": { diff --git a/acceptance/bundle/resources/permissions/sql_warehouses/current_can_manage/out.plan.direct.json b/acceptance/bundle/resources/permissions/sql_warehouses/current_can_manage/out.plan.direct.json index 6720a13fdaa..f1f44e53bc6 100644 --- a/acceptance/bundle/resources/permissions/sql_warehouses/current_can_manage/out.plan.direct.json +++ b/acceptance/bundle/resources/permissions/sql_warehouses/current_can_manage/out.plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.sql_warehouses.foo": { diff --git a/acceptance/bundle/resources/permissions/target_permissions/out.plan.direct.json b/acceptance/bundle/resources/permissions/target_permissions/out.plan.direct.json index 71d7ae5cbef..61eb85a3f0b 100644 --- a/acceptance/bundle/resources/permissions/target_permissions/out.plan.direct.json +++ b/acceptance/bundle/resources/permissions/target_permissions/out.plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.jobs.foo": { diff --git a/acceptance/bundle/resources/permissions/target_permissions/script b/acceptance/bundle/resources/permissions/target_permissions/script index 7a443d8cc2d..a2aa063789d 100644 --- a/acceptance/bundle/resources/permissions/target_permissions/script +++ b/acceptance/bundle/resources/permissions/target_permissions/script @@ -1,5 +1,5 @@ trace $CLI bundle plan -$CLI bundle plan -o json | nostamp > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp.py > out.plan.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy print_requests.py --nostamp //jobs/ > out.requests_create.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/permissions/vector_search_endpoints/current_can_manage/out.plan.direct.json b/acceptance/bundle/resources/permissions/vector_search_endpoints/current_can_manage/out.plan.direct.json index eea6e75cb51..d49433933ee 100644 --- a/acceptance/bundle/resources/permissions/vector_search_endpoints/current_can_manage/out.plan.direct.json +++ b/acceptance/bundle/resources/permissions/vector_search_endpoints/current_can_manage/out.plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.vector_search_endpoints.foo": { diff --git a/acceptance/bundle/resources/pipelines/auto-approve/script b/acceptance/bundle/resources/pipelines/auto-approve/script index f0b6c4f9044..c98e2f8e5b3 100644 --- a/acceptance/bundle/resources/pipelines/auto-approve/script +++ b/acceptance/bundle/resources/pipelines/auto-approve/script @@ -9,12 +9,12 @@ trap cleanup EXIT trace $CLI bundle deploy title "Assert the pipeline is created" -PIPELINE_ID=$($CLI bundle summary -o json | nostamp | jq -r '.resources.pipelines.bar.id') -trace $CLI pipelines get "${PIPELINE_ID}" | nostamp| jq "{spec}" +PIPELINE_ID=$($CLI bundle summary -o json | nostamp.py | jq -r '.resources.pipelines.bar.id') +trace $CLI pipelines get "${PIPELINE_ID}" | nostamp.py| jq "{spec}" title "Assert the job is created" -JOB_ID=$($CLI bundle summary -o json | nostamp | jq -r '.resources.jobs.foo.id') -$CLI jobs get "${JOB_ID}" | nostamp| jq '{name: .settings.name}' +JOB_ID=$($CLI bundle summary -o json | nostamp.py | jq -r '.resources.jobs.foo.id') +$CLI jobs get "${JOB_ID}" | nostamp.py| jq '{name: .settings.name}' title "Remove resources from configuration." trace rm resources.yml diff --git a/acceptance/bundle/resources/pipelines/lakeflow-pipeline/script b/acceptance/bundle/resources/pipelines/lakeflow-pipeline/script index a319f62fb18..79927ff6447 100644 --- a/acceptance/bundle/resources/pipelines/lakeflow-pipeline/script +++ b/acceptance/bundle/resources/pipelines/lakeflow-pipeline/script @@ -8,5 +8,5 @@ cleanup() { trap cleanup EXIT trace $CLI bundle deploy -trace jq -s '.[] | select(.path=="/api/2.0/pipelines" and .method == "POST") | .body' out.requests.txt | nostamp +trace jq -s '.[] | select(.path=="/api/2.0/pipelines" and .method == "POST") | .body' out.requests.txt | nostamp.py rm out.requests.txt diff --git a/acceptance/bundle/resources/pipelines/num-workers-zero/script b/acceptance/bundle/resources/pipelines/num-workers-zero/script index d23c1e063de..58d7db9fa5b 100644 --- a/acceptance/bundle/resources/pipelines/num-workers-zero/script +++ b/acceptance/bundle/resources/pipelines/num-workers-zero/script @@ -1,3 +1,3 @@ trace $CLI bundle deploy > out.deploy.txt 2>&1 -trace errcode jq -s '.[] | select(.method == "POST" and (.path | contains("/pipelines")))' out.requests.txt | nostamp > out.requests.$DATABRICKS_BUNDLE_ENGINE.txt 2>&1 +trace errcode jq -s '.[] | select(.method == "POST" and (.path | contains("/pipelines")))' out.requests.txt | nostamp.py > out.requests.$DATABRICKS_BUNDLE_ENGINE.txt 2>&1 rm out.requests.txt diff --git a/acceptance/bundle/resources/pipelines/recreate-keys/_script b/acceptance/bundle/resources/pipelines/recreate-keys/_script index 45c53d86743..1121836f497 100644 --- a/acceptance/bundle/resources/pipelines/recreate-keys/_script +++ b/acceptance/bundle/resources/pipelines/recreate-keys/_script @@ -2,7 +2,7 @@ trace cat databricks.yml touch foo.py touch bar.py trace $CLI bundle plan # should show 'create' -$CLI bundle plan -o json | nostamp > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp.py > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy ppid1=`read_id.py my` @@ -10,7 +10,7 @@ ppid1=`read_id.py my` print_requests() { # Excludes /api/2.0/bundle: a DMS operation path embeds the resource key, so it also # contains "/pipelines". - jq --sort-keys 'select(.method != "GET" and (.path | contains("/pipelines")) and (.path | contains("/api/2.0/bundle") | not))' < out.requests.txt | nostamp + jq --sort-keys 'select(.method != "GET" and (.path | contains("/pipelines")) and (.path | contains("/api/2.0/bundle") | not))' < out.requests.txt | nostamp.py rm -f out.requests.txt } @@ -18,14 +18,14 @@ trace print_requests trace update_file.py databricks.yml $CONFIG_UPDATE trace $CLI bundle plan # should show 'recreate' -$CLI bundle plan -o json | nostamp > out.plan_recreate.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp.py > out.plan_recreate.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy --auto-approve trace print_requests title "Fetch pipeline ID and verify remote state" ppid2=`read_id.py my` -trace $CLI pipelines get $ppid2 | nostamp +trace $CLI pipelines get $ppid2 | nostamp.py title "Verify that original pipeline is gone" trace musterr $CLI pipelines get $ppid1 diff --git a/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/out.plan_create.direct.json b/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/out.plan_create.direct.json index 1a34eb33dd4..03f9b803d55 100644 --- a/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/out.plan_create.direct.json +++ b/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/out.plan_create.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.pipelines.my": { diff --git a/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/out.plan_recreate.direct.json b/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/out.plan_recreate.direct.json index a2bf3b724c7..c15ac5eae0d 100644 --- a/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/out.plan_recreate.direct.json +++ b/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/out.plan_recreate.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resources/pipelines/recreate-keys/change-storage/out.plan_create.direct.json b/acceptance/bundle/resources/pipelines/recreate-keys/change-storage/out.plan_create.direct.json index d5bd316376b..7c768907755 100644 --- a/acceptance/bundle/resources/pipelines/recreate-keys/change-storage/out.plan_create.direct.json +++ b/acceptance/bundle/resources/pipelines/recreate-keys/change-storage/out.plan_create.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.pipelines.my": { diff --git a/acceptance/bundle/resources/pipelines/recreate-keys/change-storage/out.plan_recreate.direct.json b/acceptance/bundle/resources/pipelines/recreate-keys/change-storage/out.plan_recreate.direct.json index 6987c3ad761..da70dac7a17 100644 --- a/acceptance/bundle/resources/pipelines/recreate-keys/change-storage/out.plan_recreate.direct.json +++ b/acceptance/bundle/resources/pipelines/recreate-keys/change-storage/out.plan_recreate.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resources/pipelines/recreate/script b/acceptance/bundle/resources/pipelines/recreate/script index 0a814d08985..247959cb281 100644 --- a/acceptance/bundle/resources/pipelines/recreate/script +++ b/acceptance/bundle/resources/pipelines/recreate/script @@ -8,8 +8,8 @@ trap cleanup EXIT trace $CLI bundle deploy title "Assert the pipeline is created with catalog" -PIPELINE_ID=$($CLI bundle summary -o json | nostamp | jq -r '.resources.pipelines.foo.id') -trace $CLI pipelines get "${PIPELINE_ID}" | nostamp| jq "{spec}" +PIPELINE_ID=$($CLI bundle summary -o json | nostamp.py | jq -r '.resources.pipelines.foo.id') +trace $CLI pipelines get "${PIPELINE_ID}" | nostamp.py| jq "{spec}" # Note: In Terraform provider v1.98.0+, changing catalog no longer triggers recreation. # We switch to using storage location instead, which still triggers recreation. diff --git a/acceptance/bundle/resources/pipelines/remote_matches_config/out.plan.direct.json b/acceptance/bundle/resources/pipelines/remote_matches_config/out.plan.direct.json index c07de448831..cb852987772 100644 --- a/acceptance/bundle/resources/pipelines/remote_matches_config/out.plan.direct.json +++ b/acceptance/bundle/resources/pipelines/remote_matches_config/out.plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resources/pipelines/remote_matches_config/script b/acceptance/bundle/resources/pipelines/remote_matches_config/script index 6a6a334022e..267aa0cc3b7 100644 --- a/acceptance/bundle/resources/pipelines/remote_matches_config/script +++ b/acceptance/bundle/resources/pipelines/remote_matches_config/script @@ -15,6 +15,6 @@ r["run_as"] = {"user_name": "changed@example.test"} EOF trace $CLI bundle plan -$CLI bundle plan -o json | nostamp > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp.py > out.plan.$DATABRICKS_BUNDLE_ENGINE.json rm out.requests.txt diff --git a/acceptance/bundle/resources/pipelines/update/script b/acceptance/bundle/resources/pipelines/update/script index d9eb259cbb9..f23ae69f62a 100644 --- a/acceptance/bundle/resources/pipelines/update/script +++ b/acceptance/bundle/resources/pipelines/update/script @@ -21,7 +21,7 @@ rm out.requests.txt title "Fetch pipeline ID and verify remote state" ppid=`read_id.py my` -trace $CLI pipelines get $ppid | nostamp +trace $CLI pipelines get $ppid | nostamp.py rm out.requests.txt title "Destroy the pipeline and verify that it's removed from the state and from remote" diff --git a/acceptance/bundle/resources/postgres_branches/purge_on_delete_transitions/script b/acceptance/bundle/resources/postgres_branches/purge_on_delete_transitions/script index 919fe6e4f4f..f743dc45177 100644 --- a/acceptance/bundle/resources/postgres_branches/purge_on_delete_transitions/script +++ b/acceptance/bundle/resources/postgres_branches/purge_on_delete_transitions/script @@ -16,7 +16,7 @@ trap cleanup EXIT # resources.json. The field is omitted from JSON when unset (omitempty + # ForceSendFields tracking), so an absent value falls back to "(unset)". get_purge() { - gron.py < .databricks/bundle/default/resources.json | grep 'postgres_branches.branch.state.purge_on_delete' || echo '(unset)' + print_state.py | gron.py | grep 'postgres_branches.branch.state.purge_on_delete' || echo '(unset)' } title "Step 1: deploy with purge_on_delete unset" diff --git a/acceptance/bundle/resources/postgres_branches/recreate_source_branch_time/script b/acceptance/bundle/resources/postgres_branches/recreate_source_branch_time/script index a65cb5ee98b..f86eb93b434 100644 --- a/acceptance/bundle/resources/postgres_branches/recreate_source_branch_time/script +++ b/acceptance/bundle/resources/postgres_branches/recreate_source_branch_time/script @@ -13,7 +13,7 @@ rm -f out.requests.txt title "Change the fork point: source_branch_time is immutable, so this recreates" trace update_file.py databricks.yml "2026-01-01T00:00:00Z" "2026-02-01T00:00:00Z" trace $CLI bundle plan -trace $CLI bundle plan -o json | jq '.plan."resources.postgres_branches.dev_branch" | {action, changes}' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py | jq '.plan."resources.postgres_branches.dev_branch" | {action, changes}' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt trace $CLI bundle deploy --auto-approve trace print_requests.py --del-body project_id,branch_id --sort '//postgres' '^//workspace-files/' '^//workspace/' '^//telemetry-ext' '^//operations/' > out.requests.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/postgres_branches/update_expire_time/script b/acceptance/bundle/resources/postgres_branches/update_expire_time/script index 6c393dc60f0..c35cbbe34f0 100644 --- a/acceptance/bundle/resources/postgres_branches/update_expire_time/script +++ b/acceptance/bundle/resources/postgres_branches/update_expire_time/script @@ -19,7 +19,7 @@ trace update_file.py databricks.yml "2030-01-01T00:00:00Z" "2031-06-15T12:00:00Z # Only terraform applies it: expire_time is a member of the # expiration oneof, and the direct engine masks it under its own name. See test.toml. trace $CLI bundle plan -trace $CLI bundle plan -o json | jq '.plan."resources.postgres_branches.dev_branch" | {action, changes}' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py | jq '.plan."resources.postgres_branches.dev_branch" | {action, changes}' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt trace $CLI bundle deploy trace print_requests.py --del-body project_id,branch_id --sort '//postgres' '^//workspace-files/' '^//workspace/' '^//telemetry-ext' '^//operations/' > out.requests.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/postgres_branches/update_protected/script b/acceptance/bundle/resources/postgres_branches/update_protected/script index 1566c885831..7235a56c82c 100755 --- a/acceptance/bundle/resources/postgres_branches/update_protected/script +++ b/acceptance/bundle/resources/postgres_branches/update_protected/script @@ -15,7 +15,7 @@ print_requests() { title "Initial deployment" trace $CLI bundle validate trace $CLI bundle plan -trace $CLI bundle plan -o json | nostamp | jq '.plan."resources.postgres_branches.dev_branch" // .' > out.plan.create.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py | jq '.plan."resources.postgres_branches.dev_branch" // .' > out.plan.create.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt trace $CLI bundle deploy @@ -28,7 +28,7 @@ trace $CLI postgres get-branch "${branch_name}" | branch_fields title "Verify no_change (no changes)" trace $CLI bundle plan -trace $CLI bundle plan -o json | nostamp | jq '(.plan."resources.postgres_branches.dev_branch" // .) | del(.remote_state.status.logical_size_bytes)' > out.plan.no_change.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py | jq '(.plan."resources.postgres_branches.dev_branch" // .) | del(.remote_state.status.logical_size_bytes)' > out.plan.no_change.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt trace $CLI bundle deploy @@ -39,7 +39,7 @@ trace update_file.py databricks.yml "is_protected: false" "is_protected: true" trace cat databricks.yml trace $CLI bundle plan -trace $CLI bundle plan -o json | nostamp | jq '(.plan."resources.postgres_branches.dev_branch" // .) | del(.remote_state.status.logical_size_bytes)' > out.plan.update.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py | jq '(.plan."resources.postgres_branches.dev_branch" // .) | del(.remote_state.status.logical_size_bytes)' > out.plan.update.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt trace $CLI bundle deploy @@ -51,7 +51,7 @@ trace $CLI postgres get-branch "${branch_name}" | branch_fields title "Restore is_protected to false" trace update_file.py databricks.yml "is_protected: true" "is_protected: false" trace $CLI bundle plan -trace $CLI bundle plan -o json | nostamp | jq '(.plan."resources.postgres_branches.dev_branch" // .) | del(.remote_state.status.logical_size_bytes)' > out.plan.restore.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py | jq '(.plan."resources.postgres_branches.dev_branch" // .) | del(.remote_state.status.logical_size_bytes)' > out.plan.restore.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt trace $CLI bundle deploy diff --git a/acceptance/bundle/resources/postgres_branches/update_ttl/script b/acceptance/bundle/resources/postgres_branches/update_ttl/script index a450a54d745..ee3e60d1597 100644 --- a/acceptance/bundle/resources/postgres_branches/update_ttl/script +++ b/acceptance/bundle/resources/postgres_branches/update_ttl/script @@ -24,7 +24,7 @@ trace update_file.py databricks.yml "ttl: 604800s" "ttl: 259200s" # creation_time + ttl and never echoes ttl, which the testserver does not model. # The update_mask each engine sends is what this test is about. trace $CLI bundle plan -trace $CLI bundle plan -o json | jq '.plan."resources.postgres_branches.dev_branch" | {action, changes}' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py | jq '.plan."resources.postgres_branches.dev_branch" | {action, changes}' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt trace $CLI bundle deploy trace print_requests.py --del-body project_id,branch_id --sort '//postgres' '^//workspace-files/' '^//workspace/' '^//telemetry-ext' '^//operations/' > out.requests.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/postgres_databases/update/script b/acceptance/bundle/resources/postgres_databases/update/script index bdbad2b67c6..22677273763 100644 --- a/acceptance/bundle/resources/postgres_databases/update/script +++ b/acceptance/bundle/resources/postgres_databases/update/script @@ -15,7 +15,7 @@ print_requests() { title "Initial deployment" trace $CLI bundle validate trace $CLI bundle plan -trace $CLI bundle plan -o json | nostamp | jq '.plan."resources.postgres_databases.my_database" // .' > out.plan.create.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py | jq '.plan."resources.postgres_databases.my_database" // .' > out.plan.create.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt trace $CLI bundle deploy @@ -29,7 +29,7 @@ trace $CLI postgres get-database "${database_name}" | database_fields title "Verify no changes" trace $CLI bundle plan -trace $CLI bundle plan -o json | nostamp | jq '.plan."resources.postgres_databases.my_database" // .' > out.plan.no_change.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py | jq '.plan."resources.postgres_databases.my_database" // .' > out.plan.no_change.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt trace $CLI bundle deploy @@ -40,7 +40,7 @@ trace update_file.py databricks.yml "postgres_database: initial_db_name" "postgr trace cat databricks.yml trace $CLI bundle plan -trace $CLI bundle plan -o json | nostamp | jq '.plan."resources.postgres_databases.my_database" // .' > out.plan.update.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py | jq '.plan."resources.postgres_databases.my_database" // .' > out.plan.update.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt trace $CLI bundle deploy @@ -52,7 +52,7 @@ trace $CLI postgres get-database "${database_name}" | database_fields title "Restore postgres_database to original value" trace update_file.py databricks.yml "postgres_database: renamed_db_name" "postgres_database: initial_db_name" trace $CLI bundle plan -trace $CLI bundle plan -o json | nostamp | jq '.plan."resources.postgres_databases.my_database" // .' > out.plan.restore.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py | jq '.plan."resources.postgres_databases.my_database" // .' > out.plan.restore.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt trace $CLI bundle deploy diff --git a/acceptance/bundle/resources/postgres_endpoints/remove_suspend_timeout/script b/acceptance/bundle/resources/postgres_endpoints/remove_suspend_timeout/script index 65696066385..43c5a6172d4 100644 --- a/acceptance/bundle/resources/postgres_endpoints/remove_suspend_timeout/script +++ b/acceptance/bundle/resources/postgres_endpoints/remove_suspend_timeout/script @@ -22,7 +22,7 @@ trace update_file.py databricks.yml " suspend_timeout_duration: 300s" "" # Terraform provider masks the whole spec, which the API accepts and then ignores. See # test.toml. trace $CLI bundle plan -trace $CLI bundle plan -o json | jq '.plan."resources.postgres_endpoints.my_endpoint" | {action, changes}' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py | jq '.plan."resources.postgres_endpoints.my_endpoint" | {action, changes}' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt errcode $CLI bundle deploy &> out.deploy.$DATABRICKS_BUNDLE_ENGINE.txt trace print_requests.py --del-body project_id,branch_id,endpoint_id --sort '//postgres' '^//workspace-files/' '^//workspace/' '^//telemetry-ext' '^//operations/' > out.requests.$DATABRICKS_BUNDLE_ENGINE.json @@ -32,4 +32,4 @@ trace print_requests.py --del-body project_id,branch_id,endpoint_id --sort '//po # fail again on every deploy, terraform applied it and is clean. trace $CLI postgres get-endpoint "${endpoint_name}" | endpoint_fields trace $CLI bundle plan &> out.plan.drift.$DATABRICKS_BUNDLE_ENGINE.txt -trace $CLI bundle plan -o json | jq '.plan."resources.postgres_endpoints.my_endpoint" | {action, changes}' > out.plan.drift.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py | jq '.plan."resources.postgres_endpoints.my_endpoint" | {action, changes}' > out.plan.drift.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/postgres_endpoints/update_autoscaling/script b/acceptance/bundle/resources/postgres_endpoints/update_autoscaling/script index f2506c21a01..74bd7ed7077 100755 --- a/acceptance/bundle/resources/postgres_endpoints/update_autoscaling/script +++ b/acceptance/bundle/resources/postgres_endpoints/update_autoscaling/script @@ -15,7 +15,7 @@ print_requests() { title "Initial deployment" trace $CLI bundle validate trace $CLI bundle plan -trace $CLI bundle plan -o json | nostamp | jq '.plan."resources.postgres_endpoints.my_endpoint" // . | del(.remote_state)' > out.plan.create.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py | jq '.plan."resources.postgres_endpoints.my_endpoint" // . | del(.remote_state)' > out.plan.create.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt trace $CLI bundle deploy @@ -29,7 +29,7 @@ trace $CLI postgres get-endpoint "${endpoint_name}" | endpoint_fields title "Verify no changes" trace $CLI bundle plan -trace $CLI bundle plan -o json | nostamp | jq '.plan."resources.postgres_endpoints.my_endpoint" // . | del(.remote_state)' > out.plan.no_change.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py | jq '.plan."resources.postgres_endpoints.my_endpoint" // . | del(.remote_state)' > out.plan.no_change.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt trace $CLI bundle deploy @@ -40,7 +40,7 @@ trace update_file.py databricks.yml "autoscaling_limit_max_cu: 8" "autoscaling_l trace cat databricks.yml trace $CLI bundle plan -trace $CLI bundle plan -o json | nostamp | jq '.plan."resources.postgres_endpoints.my_endpoint" // . | del(.remote_state)' > out.plan.update.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py | jq '.plan."resources.postgres_endpoints.my_endpoint" // . | del(.remote_state)' > out.plan.update.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt trace $CLI bundle deploy @@ -52,7 +52,7 @@ trace $CLI postgres get-endpoint "${endpoint_name}" | endpoint_fields title "Restore endpoint autoscaling to original value" trace update_file.py databricks.yml "autoscaling_limit_max_cu: 4" "autoscaling_limit_max_cu: 8" trace $CLI bundle plan -trace $CLI bundle plan -o json | nostamp | jq '.plan."resources.postgres_endpoints.my_endpoint" // . | del(.remote_state)' > out.plan.restore.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py | jq '.plan."resources.postgres_endpoints.my_endpoint" // . | del(.remote_state)' > out.plan.restore.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt trace $CLI bundle deploy diff --git a/acceptance/bundle/resources/postgres_endpoints/update_pg_settings/script b/acceptance/bundle/resources/postgres_endpoints/update_pg_settings/script index ca616cee3f1..e179f96bc5f 100644 --- a/acceptance/bundle/resources/postgres_endpoints/update_pg_settings/script +++ b/acceptance/bundle/resources/postgres_endpoints/update_pg_settings/script @@ -22,7 +22,7 @@ trace update_file.py databricks.yml "statement_timeout: 8000" "statement_timeout # spec.settings.pg_settings are both accepted, while an indexed path is answered with # 400 "Unknown field path in update_mask". trace $CLI bundle plan -trace $CLI bundle plan -o json | jq '.plan."resources.postgres_endpoints.my_endpoint" | {action, changes}' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py | jq '.plan."resources.postgres_endpoints.my_endpoint" | {action, changes}' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt trace $CLI bundle deploy diff --git a/acceptance/bundle/resources/postgres_endpoints/update_suspend_timeout/script b/acceptance/bundle/resources/postgres_endpoints/update_suspend_timeout/script index 2f383859007..5b57e845759 100644 --- a/acceptance/bundle/resources/postgres_endpoints/update_suspend_timeout/script +++ b/acceptance/bundle/resources/postgres_endpoints/update_suspend_timeout/script @@ -26,7 +26,7 @@ title "Plan and deploy" # Both engines apply the change; only the mask they send differs, so out.requests # is the one per-engine file left. trace $CLI bundle plan -trace $CLI bundle plan -o json | jq '.plan."resources.postgres_endpoints.my_endpoint" | {action, changes}' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py | jq '.plan."resources.postgres_endpoints.my_endpoint" | {action, changes}' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt trace $CLI bundle deploy diff --git a/acceptance/bundle/resources/postgres_projects/purge_on_delete_transitions/script b/acceptance/bundle/resources/postgres_projects/purge_on_delete_transitions/script index 1bdc077e6e1..958d4e2d980 100644 --- a/acceptance/bundle/resources/postgres_projects/purge_on_delete_transitions/script +++ b/acceptance/bundle/resources/postgres_projects/purge_on_delete_transitions/script @@ -12,7 +12,7 @@ trap cleanup EXIT # resources.json. The field is omitted from JSON when unset (omitempty + # ForceSendFields tracking), so an absent value falls back to "(unset)". get_purge() { - gron.py < .databricks/bundle/default/resources.json | grep purge_on_delete || echo '(unset)' + print_state.py | gron.py | grep purge_on_delete || echo '(unset)' } title "Step 1: deploy with purge_on_delete unset" diff --git a/acceptance/bundle/resources/postgres_projects/remove_history_retention/script b/acceptance/bundle/resources/postgres_projects/remove_history_retention/script index 4d426ae616f..a4f41995e28 100644 --- a/acceptance/bundle/resources/postgres_projects/remove_history_retention/script +++ b/acceptance/bundle/resources/postgres_projects/remove_history_retention/script @@ -16,7 +16,7 @@ trace $CLI postgres get-project "${project_name}" | project_fields title "Remove history_retention_duration and re-deploy" trace update_file.py databricks.yml " history_retention_duration: 604800s" "" trace $CLI bundle plan -trace $CLI bundle plan -o json | jq '.plan."resources.postgres_projects.my_project" | {action, changes}' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py | jq '.plan."resources.postgres_projects.my_project" | {action, changes}' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt errcode $CLI bundle deploy &> out.deploy.$DATABRICKS_BUNDLE_ENGINE.txt trace print_requests.py --del-body project_id --sort '//postgres' '^//workspace-files/' '^//workspace/' '^//telemetry-ext' '^//operations/' > out.requests.$DATABRICKS_BUNDLE_ENGINE.json @@ -25,4 +25,4 @@ trace print_requests.py --del-body project_id --sort '//postgres' '^//workspace- # record the drift it leaves behind. trace $CLI postgres get-project "${project_name}" | project_fields > out.project.$DATABRICKS_BUNDLE_ENGINE.txt trace $CLI bundle plan &> out.plan.drift.$DATABRICKS_BUNDLE_ENGINE.txt -trace $CLI bundle plan -o json | jq '.plan."resources.postgres_projects.my_project" | {action, changes}' > out.plan.drift.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py | jq '.plan."resources.postgres_projects.my_project" | {action, changes}' > out.plan.drift.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/postgres_projects/update_default_endpoint_suspend/script b/acceptance/bundle/resources/postgres_projects/update_default_endpoint_suspend/script index 574a6cf5301..d3aeb95ebfa 100644 --- a/acceptance/bundle/resources/postgres_projects/update_default_endpoint_suspend/script +++ b/acceptance/bundle/resources/postgres_projects/update_default_endpoint_suspend/script @@ -25,7 +25,7 @@ trace update_file.py databricks.yml "suspend_timeout_duration: 300s" "suspend_ti # member of the suspension oneof, nested under default_endpoint_settings, and the # direct engine masks it under its own name. See test.toml. trace $CLI bundle plan -trace $CLI bundle plan -o json | jq '.plan."resources.postgres_projects.my_project" | {action, changes}' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py | jq '.plan."resources.postgres_projects.my_project" | {action, changes}' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt trace $CLI bundle deploy trace print_requests.py --del-body project_id --sort '//postgres' '^//workspace-files/' '^//workspace/' '^//telemetry-ext' '^//operations/' > out.requests.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/postgres_projects/update_display_name/script b/acceptance/bundle/resources/postgres_projects/update_display_name/script index 657eca21869..5601e360c32 100755 --- a/acceptance/bundle/resources/postgres_projects/update_display_name/script +++ b/acceptance/bundle/resources/postgres_projects/update_display_name/script @@ -15,7 +15,7 @@ print_requests() { title "Initial deployment" trace $CLI bundle validate trace $CLI bundle plan -trace $CLI bundle plan -o json | nostamp | jq '.plan."resources.postgres_projects.my_project" // .' > out.plan.create.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py | jq '.plan."resources.postgres_projects.my_project" // .' > out.plan.create.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt trace $CLI bundle deploy @@ -27,7 +27,7 @@ trace $CLI postgres get-project "${project_name}" | project_fields title "Verify no changes" trace $CLI bundle plan -trace $CLI bundle plan -o json | nostamp | jq '.plan."resources.postgres_projects.my_project" // .' > out.plan.no_change.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py | jq '.plan."resources.postgres_projects.my_project" // .' > out.plan.no_change.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt trace $CLI bundle deploy @@ -38,7 +38,7 @@ trace update_file.py databricks.yml "Original Name" "Updated Name" trace cat databricks.yml trace $CLI bundle plan -trace $CLI bundle plan -o json | nostamp | jq '.plan."resources.postgres_projects.my_project" // .' > out.plan.update.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py | jq '.plan."resources.postgres_projects.my_project" // .' > out.plan.update.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt trace $CLI bundle deploy @@ -50,7 +50,7 @@ trace $CLI postgres get-project "${project_name}" | project_fields title "Restore display_name to original value" trace update_file.py databricks.yml "Updated Name" "Original Name" trace $CLI bundle plan -trace $CLI bundle plan -o json | nostamp | jq '.plan."resources.postgres_projects.my_project" // .' > out.plan.restore.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py | jq '.plan."resources.postgres_projects.my_project" // .' > out.plan.restore.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt trace $CLI bundle deploy diff --git a/acceptance/bundle/resources/postgres_projects/update_history_retention/script b/acceptance/bundle/resources/postgres_projects/update_history_retention/script index 7a7ae267a73..d67c658c710 100644 --- a/acceptance/bundle/resources/postgres_projects/update_history_retention/script +++ b/acceptance/bundle/resources/postgres_projects/update_history_retention/script @@ -16,7 +16,7 @@ trace $CLI postgres get-project "${project_name}" | project_fields title "Change history_retention_duration to 259200s and re-deploy" trace update_file.py databricks.yml "history_retention_duration: 604800s" "history_retention_duration: 259200s" trace $CLI bundle plan -trace $CLI bundle plan -o json | jq '.plan."resources.postgres_projects.my_project" | {action, changes}' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py | jq '.plan."resources.postgres_projects.my_project" | {action, changes}' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt trace $CLI bundle deploy diff --git a/acceptance/bundle/resources/postgres_roles/inherited-role-bind/out.test.toml b/acceptance/bundle/resources/postgres_roles/inherited-role-bind/out.test.toml index ce8be6534d3..26f5e3debde 100644 --- a/acceptance/bundle/resources/postgres_roles/inherited-role-bind/out.test.toml +++ b/acceptance/bundle/resources/postgres_roles/inherited-role-bind/out.test.toml @@ -2,4 +2,4 @@ Cloud = false CloudEnvs.azure = false CloudEnvs.gcp = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -EnvMatrix.DMS = ["", "true"] +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/resources/postgres_roles/inherited-role-bind/test.toml b/acceptance/bundle/resources/postgres_roles/inherited-role-bind/test.toml index 6de86c5f2b6..01b05580cec 100644 --- a/acceptance/bundle/resources/postgres_roles/inherited-role-bind/test.toml +++ b/acceptance/bundle/resources/postgres_roles/inherited-role-bind/test.toml @@ -7,3 +7,7 @@ Cloud = false # bind/deploy are engine-agnostic here; run on direct for a single stable output. EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] + +# bind and unbind refuse to run when deployment history recording is on, so the +# escape hatch this test covers is not available there yet. +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/resources/postgres_roles/update/script b/acceptance/bundle/resources/postgres_roles/update/script index 5eea84136a6..d98081efd6f 100644 --- a/acceptance/bundle/resources/postgres_roles/update/script +++ b/acceptance/bundle/resources/postgres_roles/update/script @@ -15,7 +15,7 @@ print_requests() { title "Initial deployment" trace $CLI bundle validate trace $CLI bundle plan -trace $CLI bundle plan -o json | nostamp | jq '.plan."resources.postgres_roles.my_role" // .' > out.plan.create.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py | jq '.plan."resources.postgres_roles.my_role" // .' > out.plan.create.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt trace $CLI bundle deploy @@ -28,7 +28,7 @@ trace $CLI postgres get-role "${role_name}" | role_fields title "Verify no changes" trace $CLI bundle plan -trace $CLI bundle plan -o json | nostamp | jq '.plan."resources.postgres_roles.my_role" // .' > out.plan.no_change.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py | jq '.plan."resources.postgres_roles.my_role" // .' > out.plan.no_change.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt trace $CLI bundle deploy @@ -39,7 +39,7 @@ trace update_file.py databricks.yml "createdb: false" "createdb: true" trace cat databricks.yml trace $CLI bundle plan -trace $CLI bundle plan -o json | nostamp | jq '.plan."resources.postgres_roles.my_role" // .' > out.plan.update.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py | jq '.plan."resources.postgres_roles.my_role" // .' > out.plan.update.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt trace $CLI bundle deploy @@ -50,7 +50,7 @@ trace $CLI postgres get-role "${role_name}" | role_fields title "Restore attributes.createdb to original value" trace update_file.py databricks.yml "createdb: true" "createdb: false" trace $CLI bundle plan -trace $CLI bundle plan -o json | nostamp | jq '.plan."resources.postgres_roles.my_role" // .' > out.plan.restore.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py | jq '.plan."resources.postgres_roles.my_role" // .' > out.plan.restore.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt trace $CLI bundle deploy diff --git a/acceptance/bundle/resources/quality_monitors/change_assets_dir/out.plan.direct.json b/acceptance/bundle/resources/quality_monitors/change_assets_dir/out.plan.direct.json index e840a5f3161..ce08323b631 100644 --- a/acceptance/bundle/resources/quality_monitors/change_assets_dir/out.plan.direct.json +++ b/acceptance/bundle/resources/quality_monitors/change_assets_dir/out.plan.direct.json @@ -1,7 +1,7 @@ >>> errcode [CLI] bundle plan -o json { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resources/quality_monitors/change_assets_dir/out.test.toml b/acceptance/bundle/resources/quality_monitors/change_assets_dir/out.test.toml index cef45fe553a..9b8ddfcd2bd 100644 --- a/acceptance/bundle/resources/quality_monitors/change_assets_dir/out.test.toml +++ b/acceptance/bundle/resources/quality_monitors/change_assets_dir/out.test.toml @@ -1,3 +1,3 @@ Cloud = true EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] -EnvMatrix.DMS = ["", "true"] +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/resources/quality_monitors/change_assets_dir/output.txt b/acceptance/bundle/resources/quality_monitors/change_assets_dir/output.txt index 15c9c619892..796ce31264a 100644 --- a/acceptance/bundle/resources/quality_monitors/change_assets_dir/output.txt +++ b/acceptance/bundle/resources/quality_monitors/change_assets_dir/output.txt @@ -17,7 +17,7 @@ Table main.qm_test_[UNIQUE_NAME].test_table is now visible (catalog_name=main) >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/quality-monitor-update-[UNIQUE_NAME]/default/files... Created quality_monitors.monitor1 -Files: 5 uploaded, 0 deleted +Files: 6 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged >>> [CLI] bundle plan diff --git a/acceptance/bundle/resources/quality_monitors/change_assets_dir/test.toml b/acceptance/bundle/resources/quality_monitors/change_assets_dir/test.toml new file mode 100644 index 00000000000..2acd446f3cd --- /dev/null +++ b/acceptance/bundle/resources/quality_monitors/change_assets_dir/test.toml @@ -0,0 +1,3 @@ +# The plan is captured with `errcode ... &> out` (trace line + JSON together), which nostamp.py +# cannot filter; opt out of the recording variant (deploy --plan recording is covered by bundle/dms). +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/resources/quality_monitors/change_output_schema_name/out.plan.direct.json b/acceptance/bundle/resources/quality_monitors/change_output_schema_name/out.plan.direct.json index cfaff9efccb..ae5c05d00ac 100644 --- a/acceptance/bundle/resources/quality_monitors/change_output_schema_name/out.plan.direct.json +++ b/acceptance/bundle/resources/quality_monitors/change_output_schema_name/out.plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resources/quality_monitors/change_output_schema_name/script b/acceptance/bundle/resources/quality_monitors/change_output_schema_name/script index a14879d79f1..a51f2233f7d 100644 --- a/acceptance/bundle/resources/quality_monitors/change_output_schema_name/script +++ b/acceptance/bundle/resources/quality_monitors/change_output_schema_name/script @@ -22,7 +22,7 @@ trace $CLI bundle plan | contains.py "1 unchanged" update_file.py databricks.yml "output_schema_name: main.qm_test_${UNIQUE_NAME}" "output_schema_name: main.qm_test_${UNIQUE_NAME}_2" trace $CLI bundle plan | contains.py "1 to change" -trace $CLI bundle plan -o json > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.plan.$DATABRICKS_BUNDLE_ENGINE.json rm out.requests.txt trace $CLI bundle deploy diff --git a/acceptance/bundle/resources/quality_monitors/change_table_name/out.plan.direct.json b/acceptance/bundle/resources/quality_monitors/change_table_name/out.plan.direct.json index 9b233a1fdbb..eb98b10555c 100644 --- a/acceptance/bundle/resources/quality_monitors/change_table_name/out.plan.direct.json +++ b/acceptance/bundle/resources/quality_monitors/change_table_name/out.plan.direct.json @@ -1,7 +1,7 @@ >>> errcode [CLI] bundle plan -o json { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resources/quality_monitors/change_table_name/out.test.toml b/acceptance/bundle/resources/quality_monitors/change_table_name/out.test.toml index ae5c7bd798f..d73c45e3119 100644 --- a/acceptance/bundle/resources/quality_monitors/change_table_name/out.test.toml +++ b/acceptance/bundle/resources/quality_monitors/change_table_name/out.test.toml @@ -1,3 +1,3 @@ Cloud = true EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -EnvMatrix.DMS = ["", "true"] +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/resources/quality_monitors/change_table_name/test.toml b/acceptance/bundle/resources/quality_monitors/change_table_name/test.toml index 7e03ffedd9a..808576f4791 100644 --- a/acceptance/bundle/resources/quality_monitors/change_table_name/test.toml +++ b/acceptance/bundle/resources/quality_monitors/change_table_name/test.toml @@ -1,2 +1,6 @@ # this does not work on terraform EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] + +# The plan is captured with `errcode ... &> out` (trace line + JSON together), which nostamp.py +# cannot filter; opt out of the recording variant (deploy --plan recording is covered by bundle/dms). +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/resources/quality_monitors/create/out.plan_create.direct.json b/acceptance/bundle/resources/quality_monitors/create/out.plan_create.direct.json index b4a49e6fcd7..133e967a99c 100644 --- a/acceptance/bundle/resources/quality_monitors/create/out.plan_create.direct.json +++ b/acceptance/bundle/resources/quality_monitors/create/out.plan_create.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.quality_monitors.monitor1": { diff --git a/acceptance/bundle/resources/quality_monitors/create/out.plan_noop.direct.json b/acceptance/bundle/resources/quality_monitors/create/out.plan_noop.direct.json index bb26c592f1d..2cb6b0913a4 100644 --- a/acceptance/bundle/resources/quality_monitors/create/out.plan_noop.direct.json +++ b/acceptance/bundle/resources/quality_monitors/create/out.plan_noop.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resources/quality_monitors/create/script b/acceptance/bundle/resources/quality_monitors/create/script index 4f3596f8375..36e85d2164c 100644 --- a/acceptance/bundle/resources/quality_monitors/create/script +++ b/acceptance/bundle/resources/quality_monitors/create/script @@ -17,7 +17,7 @@ trap cleanup EXIT envsubst < databricks.yml.tmpl > databricks.yml -trace $CLI bundle plan -o json > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json rm out.requests.txt trace $CLI bundle deploy &> out.deploy.txt @@ -26,4 +26,4 @@ trace print_requests.py '^//api/2.0/bundle' '^//import-file/' '^//workspace/' '^ # store state to ensure we have table_name there print_state.py | grep name > out.state.$DATABRICKS_BUNDLE_ENGINE.txt -trace $CLI bundle plan -o json > out.plan_noop.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | nostamp.py > out.plan_noop.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/registered_models/aliases_converge/script b/acceptance/bundle/resources/registered_models/aliases_converge/script index ca9dc8f700a..7271aea0090 100644 --- a/acceptance/bundle/resources/registered_models/aliases_converge/script +++ b/acceptance/bundle/resources/registered_models/aliases_converge/script @@ -15,7 +15,7 @@ title "Plan is a no-op: GET never echoes aliases, so they must not drift" trace $CLI bundle plan | contains.py "Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged" title "The config-set aliases are skipped as input_only (confirms the matched rule)" -trace $CLI bundle plan --output json | jq '.plan[].changes.aliases' +trace $CLI bundle plan --output json | nostamp.py | jq '.plan[].changes.aliases' title "Edit the comment: an in-place update, not a recreate" export COMMENT="updated comment" diff --git a/acceptance/bundle/resources/registered_models/basic/script b/acceptance/bundle/resources/registered_models/basic/script index 61dff74182c..c0d1d5e648d 100644 --- a/acceptance/bundle/resources/registered_models/basic/script +++ b/acceptance/bundle/resources/registered_models/basic/script @@ -20,7 +20,7 @@ trap cleanup EXIT deploy_registered_model() { trace $CLI bundle plan trace $CLI bundle deploy - registered_model_id=$($CLI bundle summary --output json | nostamp | jq -r '.resources.registered_models.my_registered_model.id') + registered_model_id=$($CLI bundle summary --output json | nostamp.py | jq -r '.resources.registered_models.my_registered_model.id') trace $CLI registered-models get "${registered_model_id}" | jq '{name, comment, catalog_name, schema_name}' } diff --git a/acceptance/bundle/resources/registered_models/drift/browse_only/script b/acceptance/bundle/resources/registered_models/drift/browse_only/script index a3d66f24b0b..84986d5df48 100644 --- a/acceptance/bundle/resources/registered_models/drift/browse_only/script +++ b/acceptance/bundle/resources/registered_models/drift/browse_only/script @@ -7,7 +7,7 @@ title "Plan is a no-op despite the backend computing browse_only" trace $CLI bundle plan | contains.py "Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged" title "The remote-only browse_only is skipped as output_only (confirms the matched rule)" -trace $CLI bundle plan --output json | jq '.plan[].changes.browse_only' +trace $CLI bundle plan --output json | nostamp.py | jq '.plan[].changes.browse_only' title "Redeploy is a no-op (no UpdateRegisteredModel call)" trace $CLI bundle deploy diff --git a/acceptance/bundle/resources/schemas/auto-approve/script b/acceptance/bundle/resources/schemas/auto-approve/script index f2f2b6cbade..098ef3327b8 100644 --- a/acceptance/bundle/resources/schemas/auto-approve/script +++ b/acceptance/bundle/resources/schemas/auto-approve/script @@ -20,8 +20,8 @@ title "Assert the schema is created" trace $CLI schemas get "${CATALOG_NAME}.${SCHEMA_NAME}" | jq "{full_name, comment}" title "Assert the pipeline is created and uses the schema" -PIPELINE_ID=$($CLI bundle summary -o json | nostamp | jq -r '.resources.pipelines.foo.id') -trace $CLI pipelines get "${PIPELINE_ID}" | nostamp| jq "{spec}" +PIPELINE_ID=$($CLI bundle summary -o json | nostamp.py | jq -r '.resources.pipelines.foo.id') +trace $CLI pipelines get "${PIPELINE_ID}" | nostamp.py| jq "{spec}" title "Create a volume in the schema, and add a file to it. This ensures that the schema has some data in it and deletion will fail unless the generated diff --git a/acceptance/bundle/resources/schemas/comment_out_of_band/script b/acceptance/bundle/resources/schemas/comment_out_of_band/script index 70871c43a9b..4ac6a877cce 100644 --- a/acceptance/bundle/resources/schemas/comment_out_of_band/script +++ b/acceptance/bundle/resources/schemas/comment_out_of_band/script @@ -18,7 +18,7 @@ title "Set the comment out of band, the way Catalog Explorer does" MSYS_NO_PATHCONV=1 $CLI api patch "/api/2.1/unity-catalog/schemas/$SCHEMA" --json '{"comment":"set outside the bundle"}' > /dev/null title "The remote comment is drift, so the plan updates the schema" -trace $CLI bundle plan --output json | jq '.plan[].changes' +trace $CLI bundle plan --output json | nostamp.py | jq '.plan[].changes' title "Redeploy clears the comment" trace $CLI bundle deploy diff --git a/acceptance/bundle/resources/schemas/drift/managed_properties/script b/acceptance/bundle/resources/schemas/drift/managed_properties/script index fadd886cd48..f4aa3e83214 100644 --- a/acceptance/bundle/resources/schemas/drift/managed_properties/script +++ b/acceptance/bundle/resources/schemas/drift/managed_properties/script @@ -7,7 +7,7 @@ title "Plan is a no-op despite UC auto-populating managed properties" trace $CLI bundle plan | contains.py "Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged" title "The remote-only properties map is skipped as a backend default (confirms the matched rule)" -trace $CLI bundle plan --output json | jq '.plan[].changes' +trace $CLI bundle plan --output json | nostamp.py | jq '.plan[].changes' title "Redeploy is a no-op (no UpdateSchema call)" trace $CLI bundle deploy diff --git a/acceptance/bundle/resources/secret_scopes/basic/out.plan1.direct.txt b/acceptance/bundle/resources/secret_scopes/basic/out.plan1.direct.txt index 2c6ded2a841..668da79bfb5 100644 --- a/acceptance/bundle/resources/secret_scopes/basic/out.plan1.direct.txt +++ b/acceptance/bundle/resources/secret_scopes/basic/out.plan1.direct.txt @@ -1,4 +1,4 @@ -json.plan_version = 2; +json.plan_version = 3; json.cli_version = "[CLI_VERSION]"; json.plan.resources.secret_scopes.my_scope.action = "create"; json.plan.resources.secret_scopes.my_scope.new_state.value.scope = "test-scope-[UNIQUE_NAME]-1"; diff --git a/acceptance/bundle/resources/secret_scopes/basic/out.plan2.direct.txt b/acceptance/bundle/resources/secret_scopes/basic/out.plan2.direct.txt index 44b7e2831ca..f26d6cba67b 100644 --- a/acceptance/bundle/resources/secret_scopes/basic/out.plan2.direct.txt +++ b/acceptance/bundle/resources/secret_scopes/basic/out.plan2.direct.txt @@ -1,4 +1,4 @@ -json.plan_version = 2; +json.plan_version = 3; json.cli_version = "[CLI_VERSION]"; json.lineage = "[UUID]"; json.serial = 1; diff --git a/acceptance/bundle/resources/secret_scopes/basic/out.plan_verify_no_drift.direct.txt b/acceptance/bundle/resources/secret_scopes/basic/out.plan_verify_no_drift.direct.txt index 8745d467735..2782639ba91 100644 --- a/acceptance/bundle/resources/secret_scopes/basic/out.plan_verify_no_drift.direct.txt +++ b/acceptance/bundle/resources/secret_scopes/basic/out.plan_verify_no_drift.direct.txt @@ -1,4 +1,4 @@ -json.plan_version = 2; +json.plan_version = 3; json.cli_version = "[CLI_VERSION]"; json.lineage = "[UUID]"; json.serial = 2; diff --git a/acceptance/bundle/resources/secret_scopes/basic/script b/acceptance/bundle/resources/secret_scopes/basic/script index e8aebc1853f..ebaaf04d898 100755 --- a/acceptance/bundle/resources/secret_scopes/basic/script +++ b/acceptance/bundle/resources/secret_scopes/basic/script @@ -9,7 +9,7 @@ cleanup() { trap cleanup EXIT title "create the secret scope" -trace $CLI bundle plan -o json | gron.py --sort-arrays acls > out.plan1.$DATABRICKS_BUNDLE_ENGINE.txt +trace $CLI bundle plan -o json | nostamp.py | gron.py --sort-arrays acls > out.plan1.$DATABRICKS_BUNDLE_ENGINE.txt trace $CLI bundle deploy -qq scope_name=$($CLI bundle summary --output json | jq -r '.resources.secret_scopes.my_scope.name') @@ -26,7 +26,7 @@ title "update the name of the scope (should recreate)" export SECRET_SCOPE_NAME="test-scope-$UNIQUE_NAME-2" envsubst < databricks.yml.tmpl > databricks.yml -trace $CLI bundle plan -o json | gron.py --sort-arrays acls > out.plan2.$DATABRICKS_BUNDLE_ENGINE.txt +trace $CLI bundle plan -o json | nostamp.py | gron.py --sort-arrays acls > out.plan2.$DATABRICKS_BUNDLE_ENGINE.txt trace $CLI bundle deploy -qq # Capture API requests for verification. Terraform cleans up ACLs before deleting the scope, but direct does not, hence the difference in requests. @@ -43,4 +43,4 @@ trace $CLI secrets list-acls $scope_name | jq -c '.[]' | sort trace print_requests.py //secrets title "verify there's no persistent drift" -trace $CLI bundle plan -o json | gron.py --sort-arrays acls > out.plan_verify_no_drift.$DATABRICKS_BUNDLE_ENGINE.txt +trace $CLI bundle plan -o json | nostamp.py | gron.py --sort-arrays acls > out.plan_verify_no_drift.$DATABRICKS_BUNDLE_ENGINE.txt diff --git a/acceptance/bundle/resources/secrets/remove-expire-time/script b/acceptance/bundle/resources/secrets/remove-expire-time/script index 5f507f1f39a..2f461ba392b 100644 --- a/acceptance/bundle/resources/secrets/remove-expire-time/script +++ b/acceptance/bundle/resources/secrets/remove-expire-time/script @@ -7,13 +7,13 @@ read_state.py secrets secret1 name expire_time title "Remove expire_time and re-deploy" trace update_file.py databricks.yml ' expire_time: "2030-01-01T00:00:00Z"' "" trace $CLI bundle plan --var secret_value=secret-value -trace $CLI bundle plan --var secret_value=secret-value -o json | jq '.plan."resources.secrets.secret1" | {action, changes}' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan --var secret_value=secret-value -o json | nostamp.py | jq '.plan."resources.secrets.secret1" | {action, changes}' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy --var secret_value=secret-value trace print_requests.py //unity-catalog read_state.py secrets secret1 name expire_time title "Plan again to record any drift the removal leaves behind" trace $CLI bundle plan --var secret_value=secret-value -trace $CLI bundle plan --var secret_value=secret-value -o json | jq '.plan."resources.secrets.secret1" | {action, changes}' > out.plan.drift.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan --var secret_value=secret-value -o json | nostamp.py | jq '.plan."resources.secrets.secret1" | {action, changes}' > out.plan.drift.$DATABRICKS_BUNDLE_ENGINE.json rm -f out.requests.txt diff --git a/acceptance/bundle/resources/secrets/update-expire-time/script b/acceptance/bundle/resources/secrets/update-expire-time/script index 9b915b11936..b9321fd8782 100644 --- a/acceptance/bundle/resources/secrets/update-expire-time/script +++ b/acceptance/bundle/resources/secrets/update-expire-time/script @@ -7,7 +7,7 @@ read_state.py secrets secret1 name expire_time title "Change expire_time and re-deploy" trace update_file.py databricks.yml "2030-01-01T00:00:00Z" "2031-06-15T12:00:00Z" trace $CLI bundle plan --var secret_value=secret-value -trace $CLI bundle plan --var secret_value=secret-value -o json | jq '.plan."resources.secrets.secret1" | {action, changes}' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan --var secret_value=secret-value -o json | nostamp.py | jq '.plan."resources.secrets.secret1" | {action, changes}' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy --var secret_value=secret-value trace print_requests.py //unity-catalog read_state.py secrets secret1 name expire_time diff --git a/acceptance/bundle/resources/secrets/update-value/output.txt b/acceptance/bundle/resources/secrets/update-value/output.txt index 9e94921e2f3..e6cd1b57674 100644 --- a/acceptance/bundle/resources/secrets/update-value/output.txt +++ b/acceptance/bundle/resources/secrets/update-value/output.txt @@ -1,7 +1,7 @@ >>> [CLI] bundle plan --var secret_value=initial-secret-value -o json { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.secrets.secret1": { @@ -42,7 +42,7 @@ secrets secret1 catalog_name='main' schema_name='default' name='test_secret' com === Update secret value by passing a different variable value >>> [CLI] bundle plan --var secret_value=updated-secret-value -o json { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, @@ -117,20 +117,14 @@ secrets secret1 catalog_name='main' schema_name='default' name='test_secret' com === Verify state does not contain actual secret value >>> print_state.py { - "state_version": 2, - "cli_version": "[CLI_VERSION]", - "lineage": "[UUID]", - "serial": 2, - "state": { "resources.secrets.secret1": { - "__id__": "main.default.test_secret", - "state": { - "catalog_name": "main", - "comment": "Test secret", - "name": "test_secret", - "schema_name": "default", - "value": "" - } + "__id__": "main.default.test_secret", + "state": { + "catalog_name": "main", + "comment": "Test secret", + "name": "test_secret", + "schema_name": "default", + "value": "" + } } - } } diff --git a/acceptance/bundle/resources/secrets/update-value/script b/acceptance/bundle/resources/secrets/update-value/script index a5c1596e29c..224073934fc 100755 --- a/acceptance/bundle/resources/secrets/update-value/script +++ b/acceptance/bundle/resources/secrets/update-value/script @@ -1,16 +1,16 @@ -trace $CLI bundle plan --var secret_value=initial-secret-value -o json +trace $CLI bundle plan --var secret_value=initial-secret-value -o json | nostamp.py trace $CLI bundle deploy --var secret_value=initial-secret-value trace print_requests.py //unity-catalog read_state.py secrets secret1 catalog_name schema_name name comment value title "Update secret value by passing a different variable value" -trace $CLI bundle plan --var secret_value=updated-secret-value -o json +trace $CLI bundle plan --var secret_value=updated-secret-value -o json | nostamp.py trace $CLI bundle deploy --var secret_value=updated-secret-value trace print_requests.py //unity-catalog read_state.py secrets secret1 catalog_name schema_name name comment value title "Verify state does not contain actual secret value" -{ trace print_state.py | contains.py "!initial-secret-value" "!updated-secret-value"; } || true +{ trace print_state.py | jq .state | contains.py "!initial-secret-value" "!updated-secret-value"; } || true rm -f out.requests.txt diff --git a/acceptance/bundle/resources/vector_search_endpoints/drift/target_qps/script b/acceptance/bundle/resources/vector_search_endpoints/drift/target_qps/script index 58ce3a2f5ad..eaf11e08529 100644 --- a/acceptance/bundle/resources/vector_search_endpoints/drift/target_qps/script +++ b/acceptance/bundle/resources/vector_search_endpoints/drift/target_qps/script @@ -16,7 +16,7 @@ trace $CLI vector-search-endpoints patch-endpoint "${endpoint_name}" --target-qp title "Plan detects drift and proposes update" trace $CLI bundle plan | contains.py "Plan: 0 to add, 1 to change, 0 to delete, 0 unchanged" -$CLI bundle plan --output json | jq '{plan: .plan | map_values({action, changes})}' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan --output json | nostamp.py | jq '{plan: .plan | map_values({action, changes})}' > out.plan.$DATABRICKS_BUNDLE_ENGINE.json title "Deploy restores target_qps to 1" rm -f out.requests.txt diff --git a/acceptance/bundle/resources/vector_search_indexes/recreate/with_endpoint/output.txt b/acceptance/bundle/resources/vector_search_indexes/recreate/with_endpoint/output.txt index 9f3980b3e80..77b87027314 100644 --- a/acceptance/bundle/resources/vector_search_indexes/recreate/with_endpoint/output.txt +++ b/acceptance/bundle/resources/vector_search_indexes/recreate/with_endpoint/output.txt @@ -17,7 +17,7 @@ Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged >>> [CLI] bundle plan --output json { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, @@ -71,7 +71,7 @@ Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged "name": "vector" } ], - "schema_json": "{\"id\":\"int\",\"vector\":\"array\u003cfloat\u003e\"}" + "schema_json": "{\"id\":\"int\",\"vector\":\"array\"}" }, "endpoint_name": "vs-endpoint-[UNIQUE_NAME]", "endpoint_uuid": "[UUID]", @@ -87,9 +87,9 @@ Plan: 1 to add, 0 to change, 1 to delete, 1 unchanged "direct_access_index_spec.schema_json": { "action": "skip", "reason": "normalized_by_backend", - "old": "{\"id\":\"integer\",\"vector\":\"array\u003cfloat\u003e\"}", - "new": "{\"id\":\"integer\",\"vector\":\"array\u003cfloat\u003e\"}", - "remote": "{\"id\":\"int\",\"vector\":\"array\u003cfloat\u003e\"}" + "old": "{\"id\":\"integer\",\"vector\":\"array\"}", + "new": "{\"id\":\"integer\",\"vector\":\"array\"}", + "remote": "{\"id\":\"int\",\"vector\":\"array\"}" }, "endpoint_uuid": { "action": "skip", diff --git a/acceptance/bundle/resources/vector_search_indexes/recreate/with_endpoint/script b/acceptance/bundle/resources/vector_search_indexes/recreate/with_endpoint/script index b6170f5b613..5761eaae689 100644 --- a/acceptance/bundle/resources/vector_search_indexes/recreate/with_endpoint/script +++ b/acceptance/bundle/resources/vector_search_indexes/recreate/with_endpoint/script @@ -14,7 +14,7 @@ title "Change endpoint_type (recreates endpoint; index recreate cascade is not y trace update_file.py databricks.yml "endpoint_type: STANDARD" "endpoint_type: STORAGE_OPTIMIZED" trace $CLI bundle plan -trace $CLI bundle plan --output json +trace $CLI bundle plan --output json | nostamp.py rm -f out.requests.txt trace $CLI bundle deploy --auto-approve diff --git a/acceptance/bundle/resources/volumes/catalog-var-ref/output.txt b/acceptance/bundle/resources/volumes/catalog-var-ref/output.txt index cccd121c387..4b9a961521b 100644 --- a/acceptance/bundle/resources/volumes/catalog-var-ref/output.txt +++ b/acceptance/bundle/resources/volumes/catalog-var-ref/output.txt @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.catalogs.main_catalog": { diff --git a/acceptance/bundle/resources/volumes/catalog-var-ref/script b/acceptance/bundle/resources/volumes/catalog-var-ref/script index 7f1b6190766..5e6053edc06 100644 --- a/acceptance/bundle/resources/volumes/catalog-var-ref/script +++ b/acceptance/bundle/resources/volumes/catalog-var-ref/script @@ -1 +1 @@ -$CLI bundle plan -o json +$CLI bundle plan -o json | nostamp.py diff --git a/acceptance/bundle/resources/volumes/change-name/out.plan.direct.json b/acceptance/bundle/resources/volumes/change-name/out.plan.direct.json index 68e26c30d44..682a73bf1f1 100644 --- a/acceptance/bundle/resources/volumes/change-name/out.plan.direct.json +++ b/acceptance/bundle/resources/volumes/change-name/out.plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/resources/volumes/change-name/script b/acceptance/bundle/resources/volumes/change-name/script index 396ffb15155..61d993f6516 100644 --- a/acceptance/bundle/resources/volumes/change-name/script +++ b/acceptance/bundle/resources/volumes/change-name/script @@ -10,7 +10,7 @@ trace update_file.py databricks.yml myvolume mynewvolume trace $CLI bundle plan # terraform marks this as "update", direct marks this as "update_with_id" -$CLI bundle plan -o json | nostamp > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp.py > out.plan.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace print_requests.py //unity diff --git a/acceptance/bundle/resources/volumes/comment_out_of_band/script b/acceptance/bundle/resources/volumes/comment_out_of_band/script index 3c01424f369..97ce5529786 100644 --- a/acceptance/bundle/resources/volumes/comment_out_of_band/script +++ b/acceptance/bundle/resources/volumes/comment_out_of_band/script @@ -18,7 +18,7 @@ title "Set the comment out of band, the way Catalog Explorer does" MSYS_NO_PATHCONV=1 $CLI api patch "/api/2.1/unity-catalog/volumes/$VOLUME" --json '{"comment":"set outside the bundle"}' > /dev/null title "The remote comment is drift, so the plan updates the volume" -trace $CLI bundle plan --output json | jq '.plan["resources.volumes.volume1"].changes' +trace $CLI bundle plan --output json | nostamp.py | jq '.plan["resources.volumes.volume1"].changes' title "Redeploy clears the comment" trace $CLI bundle deploy diff --git a/acceptance/bundle/resources/volumes/remote-change-name/out.plan_create.direct.json b/acceptance/bundle/resources/volumes/remote-change-name/out.plan_create.direct.json index b5ef8e7d906..cd7a6dfd044 100644 --- a/acceptance/bundle/resources/volumes/remote-change-name/out.plan_create.direct.json +++ b/acceptance/bundle/resources/volumes/remote-change-name/out.plan_create.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.volumes.foo": { diff --git a/acceptance/bundle/resources/volumes/remote-change-name/script b/acceptance/bundle/resources/volumes/remote-change-name/script index 536ad6d50b8..3337fd0c6c5 100644 --- a/acceptance/bundle/resources/volumes/remote-change-name/script +++ b/acceptance/bundle/resources/volumes/remote-change-name/script @@ -1,4 +1,4 @@ -$CLI bundle plan -o json | nostamp > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json | nostamp.py > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace $CLI volumes update mycatalog.myschema.myname --json '{"new_name": "my_new_name"}' diff --git a/acceptance/bundle/resources/volumes/uppercase-name/script b/acceptance/bundle/resources/volumes/uppercase-name/script index b5403a291f8..d2931527477 100644 --- a/acceptance/bundle/resources/volumes/uppercase-name/script +++ b/acceptance/bundle/resources/volumes/uppercase-name/script @@ -21,7 +21,7 @@ if [ "$DATABRICKS_BUNDLE_ENGINE" = "direct" ]; then # seconds after create, so the schema's `properties` change is present # nondeterministically. Prune it with normalize_uc_payload.py before reshaping so # the golden is stable. See #6027 and #6040. - $CLI bundle plan -o json \ + $CLI bundle plan -o json | nostamp.py \ | normalize_uc_payload.py \ | jq '.plan | to_entries | map({resource: .key, changes: (.value.changes // {} | map_values({action, reason}))})' \ >> out.deploy.$DATABRICKS_BUNDLE_ENGINE.txt diff --git a/acceptance/bundle/run_as/pipelines/_script b/acceptance/bundle/run_as/pipelines/_script index 293600ded7d..d7cf3f76a2a 100644 --- a/acceptance/bundle/run_as/pipelines/_script +++ b/acceptance/bundle/run_as/pipelines/_script @@ -1,5 +1,5 @@ print_requests() { - jq --sort-keys 'select(.method != "GET" and (.path | contains("/pipelines")) and (.path | contains("/api/2.0/bundle") | not))' < out.requests.txt | nostamp + jq --sort-keys 'select(.method != "GET" and (.path | contains("/pipelines")) and (.path | contains("/api/2.0/bundle") | not))' < out.requests.txt | nostamp.py rm out.requests.txt } @@ -15,7 +15,7 @@ for target in "${targets[@]}"; do trace $CLI bundle plan -t $target # Debug plan - $CLI bundle plan -o json -t $target | nostamp > out.plan_$target.$DATABRICKS_BUNDLE_ENGINE.json + $CLI bundle plan -o json -t $target | nostamp.py > out.plan_$target.$DATABRICKS_BUNDLE_ENGINE.json # Deploy rm -f out.requests.txt diff --git a/acceptance/bundle/run_as/pipelines/regular_user/out.plan_t_service_principal_name.direct.json b/acceptance/bundle/run_as/pipelines/regular_user/out.plan_t_service_principal_name.direct.json index 6bd7cce4936..b29f4242bc9 100644 --- a/acceptance/bundle/run_as/pipelines/regular_user/out.plan_t_service_principal_name.direct.json +++ b/acceptance/bundle/run_as/pipelines/regular_user/out.plan_t_service_principal_name.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.pipelines.nyc_taxi_pipeline": { diff --git a/acceptance/bundle/run_as/pipelines/regular_user/out.plan_t_service_principal_name_different.direct.json b/acceptance/bundle/run_as/pipelines/regular_user/out.plan_t_service_principal_name_different.direct.json index 98e33a7a82b..fdcbed9bea6 100644 --- a/acceptance/bundle/run_as/pipelines/regular_user/out.plan_t_service_principal_name_different.direct.json +++ b/acceptance/bundle/run_as/pipelines/regular_user/out.plan_t_service_principal_name_different.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.pipelines.nyc_taxi_pipeline": { diff --git a/acceptance/bundle/run_as/pipelines/regular_user/out.plan_t_user_name.direct.json b/acceptance/bundle/run_as/pipelines/regular_user/out.plan_t_user_name.direct.json index 168d9852fb1..85d6696a669 100644 --- a/acceptance/bundle/run_as/pipelines/regular_user/out.plan_t_user_name.direct.json +++ b/acceptance/bundle/run_as/pipelines/regular_user/out.plan_t_user_name.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.pipelines.nyc_taxi_pipeline": { diff --git a/acceptance/bundle/run_as/pipelines/regular_user/out.plan_t_user_name_different.direct.json b/acceptance/bundle/run_as/pipelines/regular_user/out.plan_t_user_name_different.direct.json index a489222a9bd..02c785a484d 100644 --- a/acceptance/bundle/run_as/pipelines/regular_user/out.plan_t_user_name_different.direct.json +++ b/acceptance/bundle/run_as/pipelines/regular_user/out.plan_t_user_name_different.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.pipelines.nyc_taxi_pipeline": { diff --git a/acceptance/bundle/run_as/pipelines/service_principal/out.plan_t_service_principal_name.direct.json b/acceptance/bundle/run_as/pipelines/service_principal/out.plan_t_service_principal_name.direct.json index 5f886cd55ab..f1e14de6938 100644 --- a/acceptance/bundle/run_as/pipelines/service_principal/out.plan_t_service_principal_name.direct.json +++ b/acceptance/bundle/run_as/pipelines/service_principal/out.plan_t_service_principal_name.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.pipelines.nyc_taxi_pipeline": { diff --git a/acceptance/bundle/run_as/pipelines/service_principal/out.plan_t_service_principal_name_different.direct.json b/acceptance/bundle/run_as/pipelines/service_principal/out.plan_t_service_principal_name_different.direct.json index ef31749eae8..7dab8bf4cbb 100644 --- a/acceptance/bundle/run_as/pipelines/service_principal/out.plan_t_service_principal_name_different.direct.json +++ b/acceptance/bundle/run_as/pipelines/service_principal/out.plan_t_service_principal_name_different.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.pipelines.nyc_taxi_pipeline": { diff --git a/acceptance/bundle/run_as/pipelines/service_principal/out.plan_t_user_name.direct.json b/acceptance/bundle/run_as/pipelines/service_principal/out.plan_t_user_name.direct.json index 1e780047697..48c2635f87a 100644 --- a/acceptance/bundle/run_as/pipelines/service_principal/out.plan_t_user_name.direct.json +++ b/acceptance/bundle/run_as/pipelines/service_principal/out.plan_t_user_name.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.pipelines.nyc_taxi_pipeline": { diff --git a/acceptance/bundle/run_as/pipelines/service_principal/out.plan_t_user_name_different.direct.json b/acceptance/bundle/run_as/pipelines/service_principal/out.plan_t_user_name_different.direct.json index 4640930e261..dd0a3e931c2 100644 --- a/acceptance/bundle/run_as/pipelines/service_principal/out.plan_t_user_name_different.direct.json +++ b/acceptance/bundle/run_as/pipelines/service_principal/out.plan_t_user_name_different.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.pipelines.nyc_taxi_pipeline": { diff --git a/acceptance/bundle/select/basic/out.test.toml b/acceptance/bundle/select/basic/out.test.toml index a420f50c421..b5dac5e552c 100644 --- a/acceptance/bundle/select/basic/out.test.toml +++ b/acceptance/bundle/select/basic/out.test.toml @@ -1,4 +1,4 @@ Cloud = true EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -EnvMatrix.DMS = ["", "true"] +EnvMatrix.DMS = [""] EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/select/basic/script b/acceptance/bundle/select/basic/script index 12db6ca0ba0..e8b7741af95 100644 --- a/acceptance/bundle/select/basic/script +++ b/acceptance/bundle/select/basic/script @@ -22,7 +22,7 @@ trace $CLI bundle plan --select jobs.foo # JSON embeds remote state that differs between local and cloud), then deploy it: # inline, or via --plan (READPLAN=1). The deploy is not traced because readplanarg # varies the command line between READPLAN variants, which must produce identical output. -$CLI bundle plan --select jobs.foo -o json | nostamp > plan.json +$CLI bundle plan --select jobs.foo -o json > plan.json title "bundle deploy --select jobs.foo\n" $CLI bundle deploy --select jobs.foo $(readplanarg plan.json) # The deploy reports that --select was used via telemetry. @@ -37,7 +37,7 @@ trace $CLI bundle summary # foo and bar are already deployed, so only baz remains to create. title "Full plan after partial deploy" trace $CLI bundle plan -$CLI bundle plan -o json | nostamp > plan-full.json +$CLI bundle plan -o json > plan-full.json title "Full deploy\n" $CLI bundle deploy $(readplanarg plan-full.json) # Only baz is created this time. diff --git a/acceptance/bundle/select/basic/test.toml b/acceptance/bundle/select/basic/test.toml index fe41d5f871a..7166a9e7946 100644 --- a/acceptance/bundle/select/basic/test.toml +++ b/acceptance/bundle/select/basic/test.toml @@ -8,3 +8,7 @@ EnvMatrix.READPLAN = ["", "1"] # databricks.yml and the serialized plans are generated at runtime; the plan JSON # is kept out of the recorded output because it embeds local/cloud-specific state. Ignore = [".databricks", ".gitignore", "databricks.yml", "plan.json", "plan-full.json"] + +# This test dumps a plan and replays it via deploy --plan/readplanarg; a nostamp.py'd dump can't +# carry the DMS fields the replay needs, so opt out of recording (covered by bundle/dms). +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/select/grants_permissions/out.test.toml b/acceptance/bundle/select/grants_permissions/out.test.toml index dd03baaabb6..bf0f576652e 100644 --- a/acceptance/bundle/select/grants_permissions/out.test.toml +++ b/acceptance/bundle/select/grants_permissions/out.test.toml @@ -1,4 +1,4 @@ Cloud = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -EnvMatrix.DMS = ["", "true"] +EnvMatrix.DMS = [""] EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/select/grants_permissions/test.toml b/acceptance/bundle/select/grants_permissions/test.toml index 4b4735b927d..544f8f1e442 100644 --- a/acceptance/bundle/select/grants_permissions/test.toml +++ b/acceptance/bundle/select/grants_permissions/test.toml @@ -11,3 +11,7 @@ RecordRequests = true EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] EnvMatrix.READPLAN = ["", "1"] Ignore = [".databricks", ".gitignore", "databricks.yml", "plan-job.json", "plan-schema.json"] + +# Dumps plans and replays them via deploy --plan/readplanarg; the replay needs the DMS fields, +# so opt out of recording (covered by bundle/dms). +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/state/feature_flags/out.test.toml b/acceptance/bundle/state/feature_flags/out.test.toml index 59b56a2037c..27ec2a7fcd6 100644 --- a/acceptance/bundle/state/feature_flags/out.test.toml +++ b/acceptance/bundle/state/feature_flags/out.test.toml @@ -1,3 +1,3 @@ Cloud = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -EnvMatrix.DMS = ["", "true"] +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/state/feature_flags/output.txt b/acceptance/bundle/state/feature_flags/output.txt index df55e7c6cee..1ed17f9b23e 100644 --- a/acceptance/bundle/state/feature_flags/output.txt +++ b/acceptance/bundle/state/feature_flags/output.txt @@ -1,12 +1,12 @@ -=== a version-3 state recording a feature is rejected (this CLI records no features yet) +=== a state depending on a feature this CLI does not recognize is rejected >>> errcode [CLI] bundle plan -Error: migrating state [TEST_TMP_DIR]/.databricks/bundle/default/resources.json: the deployment state requires features this CLI does not support: future_feature; upgrade to the latest CLI version and see https://docs.databricks.com/aws/en/dev-tools/bundles/state-features#state-features for more information +Error: the deployment state requires features this CLI does not support: future_feature; upgrade to the latest CLI version and see https://docs.databricks.com/aws/en/dev-tools/bundles/state-features#state-features for more information Exit code: 1 -=== a version-3 state with an empty features map is accepted, and a deploy keeps it at version 3 (no flip to 2) +=== a state with an empty features map is accepted, and a deploy keeps it at version 2 >>> [CLI] bundle plan create jobs.my_job @@ -19,4 +19,4 @@ Files: 6 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged >>> gron.py .databricks/bundle/default/resources.json -json.state_version = 3; +json.state_version = 2; diff --git a/acceptance/bundle/state/feature_flags/resources.empty_features.json b/acceptance/bundle/state/feature_flags/resources.empty_features.json index b20c97aa074..876fdaf19ce 100644 --- a/acceptance/bundle/state/feature_flags/resources.empty_features.json +++ b/acceptance/bundle/state/feature_flags/resources.empty_features.json @@ -1,5 +1,5 @@ { - "state_version": 3, + "state_version": 2, "features": {}, "cli_version": "0.0.0-dev", "lineage": "test-lineage", diff --git a/acceptance/bundle/state/feature_flags/resources.with_feature.json b/acceptance/bundle/state/feature_flags/resources.with_feature.json index b844b098a70..acc2ec9823d 100644 --- a/acceptance/bundle/state/feature_flags/resources.with_feature.json +++ b/acceptance/bundle/state/feature_flags/resources.with_feature.json @@ -1,5 +1,5 @@ { - "state_version": 3, + "state_version": 2, "features": { "future_feature": {} }, diff --git a/acceptance/bundle/state/feature_flags/script b/acceptance/bundle/state/feature_flags/script index e70b562f2ff..ef9570e318c 100644 --- a/acceptance/bundle/state/feature_flags/script +++ b/acceptance/bundle/state/feature_flags/script @@ -1,10 +1,10 @@ mkdir -p .databricks/bundle/default -title "a version-3 state recording a feature is rejected (this CLI records no features yet)" +title "a state depending on a feature this CLI does not recognize is rejected" cp resources.with_feature.json .databricks/bundle/default/resources.json trace errcode $CLI bundle plan 2>&1 | contains.py "requires features this CLI does not support: future_feature" "upgrade to the latest CLI version" "https://docs.databricks.com/aws/en/dev-tools/bundles/state-features#state-features" -title "a version-3 state with an empty features map is accepted, and a deploy keeps it at version 3 (no flip to 2)" +title "a state with an empty features map is accepted, and a deploy keeps it at version 2" cp resources.empty_features.json .databricks/bundle/default/resources.json trace $CLI bundle plan | contains.py "Plan:" trace $CLI bundle deploy diff --git a/acceptance/bundle/state/feature_flags/test.toml b/acceptance/bundle/state/feature_flags/test.toml index 5640bffa4dc..dfe3377a2b7 100644 --- a/acceptance/bundle/state/feature_flags/test.toml +++ b/acceptance/bundle/state/feature_flags/test.toml @@ -1,3 +1,6 @@ Ignore = [".databricks"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +# State feature-flag handling, not DMS recording: opt out of the DMS variant so the deploy in the +# empty-features case does not record (which would diverge from the non-recording variant). +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/state/future_version/output.txt b/acceptance/bundle/state/future_version/output.txt index 0a16971f472..7cf98129ee9 100644 --- a/acceptance/bundle/state/future_version/output.txt +++ b/acceptance/bundle/state/future_version/output.txt @@ -1,3 +1,3 @@ -state version 999 is newer than supported version 3; upgrade the CLI +state version 999 is newer than supported version 2; upgrade the CLI Exit code: 1 diff --git a/acceptance/bundle/state/permission_level_migration/out.plan.direct.json b/acceptance/bundle/state/permission_level_migration/out.plan.direct.json index 33d6194c91e..1492e489619 100644 --- a/acceptance/bundle/state/permission_level_migration/out.plan.direct.json +++ b/acceptance/bundle/state/permission_level_migration/out.plan.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "test-lineage", "serial": 1, diff --git a/acceptance/bundle/state/permission_level_migration/script b/acceptance/bundle/state/permission_level_migration/script index 32211ebeb49..55fb4aa769f 100644 --- a/acceptance/bundle/state/permission_level_migration/script +++ b/acceptance/bundle/state/permission_level_migration/script @@ -9,10 +9,10 @@ mkdir -p .databricks/bundle/default cp resources.v1.json .databricks/bundle/default/resources.json title "Plan with old v1 state" -trace $CLI bundle plan -o json > out.plan.direct.json +trace $CLI bundle plan -o json | nostamp.py > out.plan.direct.json title "Deploy (migrates state)" trace $CLI bundle deploy title "Print state after deploy" -trace print_state.py | nostamp --indent 1 +trace print_state.py | nostamp.py --indent 1 diff --git a/acceptance/bundle/summary/modified_status/script b/acceptance/bundle/summary/modified_status/script index 22149b2c76b..fd509c95af0 100644 --- a/acceptance/bundle/summary/modified_status/script +++ b/acceptance/bundle/summary/modified_status/script @@ -1,14 +1,14 @@ title "Initial view of resources without id and modified_status=created" -trace $CLI bundle summary -o json | nostamp | jq .resources +trace $CLI bundle summary -o json | nostamp.py | jq .resources trace $CLI bundle deploy title "Post-deployment view of resources with id and without modified_status" -trace $CLI bundle summary -o json | nostamp | jq .resources +trace $CLI bundle summary -o json | nostamp.py | jq .resources mv $VARIANT databricks.yml title "Expecting all resources to have modified_status=deleted" -trace $CLI bundle summary -o json | nostamp | jq .resources +trace $CLI bundle summary -o json | nostamp.py | jq .resources trace $CLI bundle destroy --auto-approve -trace $CLI bundle summary -o json | nostamp | jq .resources +trace $CLI bundle summary -o json | nostamp.py | jq .resources diff --git a/acceptance/bundle/templates/default-python/classic/out.plan_after_deploy_dev.direct.json b/acceptance/bundle/templates/default-python/classic/out.plan_after_deploy_dev.direct.json index 49c706fc239..5607a8fc757 100644 --- a/acceptance/bundle/templates/default-python/classic/out.plan_after_deploy_dev.direct.json +++ b/acceptance/bundle/templates/default-python/classic/out.plan_after_deploy_dev.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/templates/default-python/classic/out.plan_after_deploy_prod.direct.json b/acceptance/bundle/templates/default-python/classic/out.plan_after_deploy_prod.direct.json index 30918164e2c..4ec8e61237d 100644 --- a/acceptance/bundle/templates/default-python/classic/out.plan_after_deploy_prod.direct.json +++ b/acceptance/bundle/templates/default-python/classic/out.plan_after_deploy_prod.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/templates/default-python/classic/out.plan_dev.direct.json b/acceptance/bundle/templates/default-python/classic/out.plan_dev.direct.json index b150391beeb..3fb9ec72a4f 100644 --- a/acceptance/bundle/templates/default-python/classic/out.plan_dev.direct.json +++ b/acceptance/bundle/templates/default-python/classic/out.plan_dev.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.jobs.sample_job": { diff --git a/acceptance/bundle/templates/default-python/classic/out.plan_prod.direct.json b/acceptance/bundle/templates/default-python/classic/out.plan_prod.direct.json index a153855fe26..3b20349f64d 100644 --- a/acceptance/bundle/templates/default-python/classic/out.plan_prod.direct.json +++ b/acceptance/bundle/templates/default-python/classic/out.plan_prod.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.jobs.sample_job": { diff --git a/acceptance/bundle/templates/default-python/classic/script b/acceptance/bundle/templates/default-python/classic/script index 6b79c6f2251..883a405fa3e 100644 --- a/acceptance/bundle/templates/default-python/classic/script +++ b/acceptance/bundle/templates/default-python/classic/script @@ -7,20 +7,20 @@ trace $CLI bundle validate -t prod trace $CLI bundle plan -t dev trace $CLI bundle plan -t prod -$CLI bundle plan -o json -t dev > ../../out.plan_dev.$DATABRICKS_BUNDLE_ENGINE.json -$CLI bundle plan -o json -t prod > ../../out.plan_prod.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json -t dev | nostamp.py > ../../out.plan_dev.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json -t prod | nostamp.py > ../../out.plan_prod.$DATABRICKS_BUNDLE_ENGINE.json rm ../../out.requests.txt # With --plan variant we don't build artifacts, so we can filter out relevant log lines $CLI bundle deploy -t dev $(readplanarg ../../out.plan_dev.direct.json) 2>&1 | grep -vE '^Building python_artifact|^Uploading .databricks' print_requests.py --sort '^//import-file/' '^//telemetry-ext' > ../../out.requests.dev.$DATABRICKS_BUNDLE_ENGINE.txt trace $CLI bundle plan -t dev # check if if there is drift -trace $CLI bundle plan -t dev -o json > ../../out.plan_after_deploy_dev.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -t dev -o json | nostamp.py > ../../out.plan_after_deploy_dev.$DATABRICKS_BUNDLE_ENGINE.json $CLI bundle deploy -t prod $(readplanarg ../../out.plan_prod.direct.json) 2>&1 | grep -vE '^Building python_artifact|^Uploading dist' print_requests.py --sort '^//import-file/' '^//telemetry-ext' > ../../out.requests.prod.$DATABRICKS_BUNDLE_ENGINE.txt trace $CLI bundle plan -t prod # check if there is drift -trace $CLI bundle plan -t prod -o json > ../../out.plan_after_deploy_prod.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -t prod -o json | nostamp.py > ../../out.plan_after_deploy_prod.$DATABRICKS_BUNDLE_ENGINE.json # Do not affect this repository's git behaviour #2318 mv .gitignore out.gitignore diff --git a/acceptance/bundle/templates/default-python/integration_classic/out.plan_dev.direct.json b/acceptance/bundle/templates/default-python/integration_classic/out.plan_dev.direct.json index 0c6800b20b3..b69ca087696 100644 --- a/acceptance/bundle/templates/default-python/integration_classic/out.plan_dev.direct.json +++ b/acceptance/bundle/templates/default-python/integration_classic/out.plan_dev.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.jobs.sample_job": { diff --git a/acceptance/bundle/templates/default-python/integration_classic/out.plan_prod.direct.json b/acceptance/bundle/templates/default-python/integration_classic/out.plan_prod.direct.json index e637e98dde5..1bb26918106 100644 --- a/acceptance/bundle/templates/default-python/integration_classic/out.plan_prod.direct.json +++ b/acceptance/bundle/templates/default-python/integration_classic/out.plan_prod.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.jobs.sample_job": { diff --git a/acceptance/bundle/templates/default-python/integration_classic/script b/acceptance/bundle/templates/default-python/integration_classic/script index 50b39d6ffce..98c5b97123c 100644 --- a/acceptance/bundle/templates/default-python/integration_classic/script +++ b/acceptance/bundle/templates/default-python/integration_classic/script @@ -17,7 +17,7 @@ trap "trace $CLI bundle destroy -t dev --auto-approve" EXIT trace $CLI bundle validate -t dev trace $CLI bundle validate -t dev -o json | jq -S "del($JQ_DELETE)" > ../out.validate.dev.json -$CLI bundle plan -o json -t dev > ../out.plan_dev.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json -t dev | nostamp.py > ../out.plan_dev.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy -t dev trace $CLI bundle summary -t dev trace $CLI bundle summary -t dev -o json | jq -S "del($JQ_DELETE)" > ../out.summary.dev.json @@ -30,7 +30,7 @@ trace $CLI bundle validate -t prod trace $CLI bundle validate -t prod -o json | jq -S "del($JQ_DELETE)" > ../out.validate.prod.json trace diff.py ../out.validate.dev.json ../out.validate.prod.json rm ../out.validate.prod.json -$CLI bundle plan -o json -t prod > ../out.plan_prod.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json -t prod | nostamp.py > ../out.plan_prod.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy -t prod trace $CLI bundle summary -t prod trace $CLI bundle summary -t prod -o json | jq -S "del($JQ_DELETE)" > ../out.summary.prod.json diff --git a/acceptance/bundle/templates/default-python/serverless/out.plan_after_deploy_dev.direct.json b/acceptance/bundle/templates/default-python/serverless/out.plan_after_deploy_dev.direct.json index 9967e015883..eac2e21abdf 100644 --- a/acceptance/bundle/templates/default-python/serverless/out.plan_after_deploy_dev.direct.json +++ b/acceptance/bundle/templates/default-python/serverless/out.plan_after_deploy_dev.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/templates/default-python/serverless/out.plan_after_deploy_prod.direct.json b/acceptance/bundle/templates/default-python/serverless/out.plan_after_deploy_prod.direct.json index 57a12ae5bb3..c9b06ebd2c5 100644 --- a/acceptance/bundle/templates/default-python/serverless/out.plan_after_deploy_prod.direct.json +++ b/acceptance/bundle/templates/default-python/serverless/out.plan_after_deploy_prod.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, diff --git a/acceptance/bundle/templates/default-python/serverless/out.plan_dev.direct.json b/acceptance/bundle/templates/default-python/serverless/out.plan_dev.direct.json index dd0e402c7df..f68705b05d9 100644 --- a/acceptance/bundle/templates/default-python/serverless/out.plan_dev.direct.json +++ b/acceptance/bundle/templates/default-python/serverless/out.plan_dev.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.jobs.sample_job": { diff --git a/acceptance/bundle/templates/default-python/serverless/out.plan_prod.direct.json b/acceptance/bundle/templates/default-python/serverless/out.plan_prod.direct.json index a9db4ff38f4..42a1c6a6351 100644 --- a/acceptance/bundle/templates/default-python/serverless/out.plan_prod.direct.json +++ b/acceptance/bundle/templates/default-python/serverless/out.plan_prod.direct.json @@ -1,5 +1,5 @@ { - "plan_version": 2, + "plan_version": 3, "cli_version": "[CLI_VERSION]", "plan": { "resources.jobs.sample_job": { diff --git a/acceptance/bundle/templates/default-python/serverless/script b/acceptance/bundle/templates/default-python/serverless/script index 86272b4a266..c0f2e58f4f4 100644 --- a/acceptance/bundle/templates/default-python/serverless/script +++ b/acceptance/bundle/templates/default-python/serverless/script @@ -7,19 +7,19 @@ trace $CLI bundle validate -t prod trace $CLI bundle plan -t dev trace $CLI bundle plan -t prod -$CLI bundle plan -o json -t dev > ../../out.plan_dev.$DATABRICKS_BUNDLE_ENGINE.json -$CLI bundle plan -o json -t prod > ../../out.plan_prod.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json -t dev | nostamp.py > ../../out.plan_dev.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle plan -o json -t prod | nostamp.py > ../../out.plan_prod.$DATABRICKS_BUNDLE_ENGINE.json rm ../../out.requests.txt trace $CLI bundle deploy -t dev print_requests.py --sort '^//import-file/' '^//telemetry-ext' > ../../out.requests.dev.$DATABRICKS_BUNDLE_ENGINE.txt trace $CLI bundle plan -t dev # check if if there is drift -trace $CLI bundle plan -t dev -o json > ../../out.plan_after_deploy_dev.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -t dev -o json | nostamp.py > ../../out.plan_after_deploy_dev.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy -t prod print_requests.py --sort '^//import-file/' '^//telemetry-ext' > ../../out.requests.prod.$DATABRICKS_BUNDLE_ENGINE.txt trace $CLI bundle plan -t prod # check if there is drift -trace $CLI bundle plan -t prod -o json > ../../out.plan_after_deploy_prod.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -t prod -o json | nostamp.py > ../../out.plan_after_deploy_prod.$DATABRICKS_BUNDLE_ENGINE.json # Do not affect this repository's git behaviour #2318 mv .gitignore out.gitignore diff --git a/acceptance/bundle/templates/record-deployment-history/out.test.toml b/acceptance/bundle/templates/record-deployment-history/out.test.toml index 27ec2a7fcd6..9921e91a794 100644 --- a/acceptance/bundle/templates/record-deployment-history/out.test.toml +++ b/acceptance/bundle/templates/record-deployment-history/out.test.toml @@ -1,3 +1,3 @@ Cloud = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -EnvMatrix.DMS = [""] +EnvMatrix.DMS = ["true"] diff --git a/acceptance/bundle/templates/record-deployment-history/test.toml b/acceptance/bundle/templates/record-deployment-history/test.toml index a1d3232167a..89b85620c2a 100644 --- a/acceptance/bundle/templates/record-deployment-history/test.toml +++ b/acceptance/bundle/templates/record-deployment-history/test.toml @@ -2,3 +2,7 @@ Env.DATABRICKS_BUNDLE_INIT_RECORD_DEPLOYMENT_HISTORY = "true" # What is under test is what `bundle init` writes, so one engine is enough. EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] + +# The generated project sets experimental.record_deployment_history; DMS=true permits it so +# the validate step below loads instead of rejecting it. +EnvMatrix.DMS = ["true"] diff --git a/acceptance/bundle/test.toml b/acceptance/bundle/test.toml index 6a259762f65..2db98f4b429 100644 --- a/acceptance/bundle/test.toml +++ b/acceptance/bundle/test.toml @@ -10,14 +10,6 @@ EnvMatrix.DMS = ["", "true"] # DMS recording is only supported by the direct engine; it is a no-op on terraform. EnvMatrixExclude.dms_needs_direct = ["DMS=true", "DATABRICKS_BUNDLE_ENGINE=terraform"] -# Saved plans don't carry the deployment stamp. A first plan writes the deployment record, -# so `deploy --plan` creates resources without it and reports drift on the next plan. -EnvMatrixExclude.dms_no_readplan = ["DMS=true", "READPLAN=1"] - -# Recording is gated off for users (see validate.ValidateRecordDeploymentHistory), so -# force it on: the point of this run is to exercise it. -Env.DATABRICKS_BUNDLE_FORCE_ALLOW_RECORD_DEPLOYMENT_HISTORY = "1" - # The DMS run asserts the same golden files as the engine runs. EnvRepl.DMS = false diff --git a/bundle/bundle.go b/bundle/bundle.go index 1eb9ce15654..1832be861c3 100644 --- a/bundle/bundle.go +++ b/bundle/bundle.go @@ -317,6 +317,14 @@ func (b *Bundle) WorkspaceClient(ctx context.Context) *databricks.WorkspaceClien return client } +// ConfiguresDeploymentHistory reports whether this bundle is configured to record deployment history with the +// deployment metadata service, from experimental.record_deployment_history or +// DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY. +func (b *Bundle) ConfiguresDeploymentHistory(ctx context.Context) bool { + configured := b.Config.Experimental != nil && b.Config.Experimental.RecordDeploymentHistory + return env.RecordsDeploymentHistory(ctx, configured) +} + // SetWorkpaceClient sets the workspace client for this bundle. // This is used to inject a mock client for testing. func (b *Bundle) SetWorkpaceClient(w *databricks.WorkspaceClient) { diff --git a/bundle/config/deployment.go b/bundle/config/deployment.go index b7efb4456f9..b59d1b1da02 100644 --- a/bundle/config/deployment.go +++ b/bundle/config/deployment.go @@ -7,4 +7,20 @@ type Deployment struct { // Lock configures locking behavior on deployment. Lock Lock `json:"lock,omitempty"` + + // History reports what the deployment metadata service has recorded for this + // bundle. Output only: it is read from the service for 'bundle summary' and is + // unset when the bundle does not record deployment history. + History *DeploymentHistory `json:"history,omitempty" bundle:"readonly"` +} + +// DeploymentHistory identifies the bundle's deployment in the deployment +// metadata service. +type DeploymentHistory struct { + // DeploymentID is the ID the service assigned to this bundle's deployment. + DeploymentID string `json:"deployment_id,omitempty"` + + // LatestVersionID is the most recent version recorded for the deployment. It is + // unset when the deployment exists but has no version yet. + LatestVersionID string `json:"latest_version_id,omitempty"` } diff --git a/bundle/config/experimental.go b/bundle/config/experimental.go index 658f1cea819..005c4dfcade 100644 --- a/bundle/config/experimental.go +++ b/bundle/config/experimental.go @@ -53,6 +53,9 @@ type Experimental struct { // RecordDeploymentHistory opts the bundle into the deployment metadata // service (DMS), which records deployment history and tracks what changed // across deployments. + // + // Only for bundles with no deployed resources yet: DMS becomes the source of + // truth for their state. RecordDeploymentHistory bool `json:"record_deployment_history,omitempty"` } diff --git a/bundle/configsync/diff.go b/bundle/configsync/diff.go index ea45903508b..40511ae1dcd 100644 --- a/bundle/configsync/diff.go +++ b/bundle/configsync/diff.go @@ -149,7 +149,7 @@ func OpenDeploymentState(ctx context.Context, b *bundle.Bundle, engine engine.En deployBundle := &direct.DeploymentBundle{} _, statePath := b.StateFilenameConfigSnapshot(ctx) - if err := deployBundle.StateDB.Open(ctx, statePath, dstate.WithRecovery(true), dstate.WithWrite(false)); err != nil { + if err := deployBundle.StateDB.Open(ctx, statePath, dstate.WithRecovery(true), dstate.WithWrite(false), nil, dstate.WithDeploymentHistory(false), ""); err != nil { return nil, fmt.Errorf("failed to open state: %w", err) } return deployBundle, nil diff --git a/bundle/configsync/variables.go b/bundle/configsync/variables.go index 65cddeaff19..c16655a3518 100644 --- a/bundle/configsync/variables.go +++ b/bundle/configsync/variables.go @@ -156,7 +156,7 @@ func resourceIDLookup(ctx context.Context, b *bundle.Bundle) func(string) string } _, statePath := b.StateFilenameConfigSnapshot(ctx) db := &dstate.DeploymentState{} - if err := db.Open(ctx, statePath, dstate.WithRecovery(false), dstate.WithWrite(false)); err != nil { + if err := db.Open(ctx, statePath, dstate.WithRecovery(false), dstate.WithWrite(false), nil, dstate.WithDeploymentHistory(false), ""); err != nil { log.Debugf(ctx, "variable restoration: failed to open state DB at %s: %v", statePath, err) return nil } diff --git a/bundle/deploy/metadata/annotate_deployment_version.go b/bundle/deploy/metadata/annotate_deployment_version.go new file mode 100644 index 00000000000..c08f7b52345 --- /dev/null +++ b/bundle/deploy/metadata/annotate_deployment_version.go @@ -0,0 +1,65 @@ +package metadata + +import ( + "context" + "strconv" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/libs/diag" +) + +type annotateDeployment struct { + deploymentID string +} + +// AnnotateDeployment stamps the DMS deployment onto every job and pipeline, so a workspace +// resource points back at the deployment that produced it - how lineage resolves a job to its +// bundle. It runs before the plan, or the stamp would show as drift against local config. +func AnnotateDeployment(deploymentID string) bundle.Mutator { + return &annotateDeployment{deploymentID: deploymentID} +} + +func (m *annotateDeployment) Name() string { + return "metadata.AnnotateDeployment" +} + +func (m *annotateDeployment) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { + for _, job := range b.Config.Resources.Jobs { + // Deployment is set by AnnotateJobs, which runs during initialize. + job.Deployment.DeploymentId = m.deploymentID + } + + for _, pipeline := range b.Config.Resources.Pipelines { + pipeline.Deployment.DeploymentId = m.deploymentID + } + + return nil +} + +type annotateDeploymentVersion struct { + version int64 +} + +// AnnotateDeploymentVersion stamps the DMS version onto every job and pipeline. +// Separate from AnnotateDeployment because version only exists after CreateVersion runs. +func AnnotateDeploymentVersion(version int64) bundle.Mutator { + return &annotateDeploymentVersion{version: version} +} + +func (m *annotateDeploymentVersion) Name() string { + return "metadata.AnnotateDeploymentVersion" +} + +func (m *annotateDeploymentVersion) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { + versionID := strconv.FormatInt(m.version, 10) + + for _, job := range b.Config.Resources.Jobs { + job.Deployment.VersionId = versionID + } + + for _, pipeline := range b.Config.Resources.Pipelines { + pipeline.Deployment.VersionId = versionID + } + + return nil +} diff --git a/bundle/deploy/metadata/annotate_deployment_version_test.go b/bundle/deploy/metadata/annotate_deployment_version_test.go new file mode 100644 index 00000000000..297ba95f8ee --- /dev/null +++ b/bundle/deploy/metadata/annotate_deployment_version_test.go @@ -0,0 +1,50 @@ +package metadata + +import ( + "testing" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/databricks-sdk-go/service/jobs" + "github.com/databricks/databricks-sdk-go/service/pipelines" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestAnnotateDeploymentVersion(t *testing.T) { + b := &bundle.Bundle{ + Config: config.Root{ + Resources: config.Resources{ + Jobs: map[string]*resources.Job{ + "my-job": { + JobSettings: jobs.JobSettings{ + Deployment: &jobs.JobDeployment{Kind: jobs.JobDeploymentKindBundle}, + }, + }, + }, + Pipelines: map[string]*resources.Pipeline{ + "my-pipeline": { + CreatePipeline: pipelines.CreatePipeline{ + Deployment: &pipelines.PipelineDeployment{Kind: pipelines.DeploymentKindBundle}, + }, + }, + }, + }, + }, + } + + diags := bundle.ApplySeq(t.Context(), b, AnnotateDeployment("dep-123"), AnnotateDeploymentVersion(7)) + require.NoError(t, diags.Error()) + + job := b.Config.Resources.Jobs["my-job"].Deployment + assert.Equal(t, "dep-123", job.DeploymentId) + assert.Equal(t, "7", job.VersionId) + // The kind set by AnnotateJobs is preserved. + assert.Equal(t, jobs.JobDeploymentKindBundle, job.Kind) + + pipeline := b.Config.Resources.Pipelines["my-pipeline"].Deployment + assert.Equal(t, "dep-123", pipeline.DeploymentId) + assert.Equal(t, "7", pipeline.VersionId) + assert.Equal(t, pipelines.DeploymentKindBundle, pipeline.Kind) +} diff --git a/bundle/deployplan/plan.go b/bundle/deployplan/plan.go index c9f5bec3516..3f24c82ee43 100644 --- a/bundle/deployplan/plan.go +++ b/bundle/deployplan/plan.go @@ -13,14 +13,23 @@ import ( "github.com/databricks/cli/libs/structs/structvar" ) -const currentPlanVersion = 2 +const currentPlanVersion = 3 type Plan struct { - PlanVersion int `json:"plan_version,omitempty"` - CLIVersion string `json:"cli_version,omitempty"` - Lineage string `json:"lineage,omitempty"` - Serial int `json:"serial,omitempty"` - Plan map[string]*PlanEntry `json:"plan,omitzero"` + PlanVersion int `json:"plan_version,omitempty"` + CLIVersion string `json:"cli_version,omitempty"` + Lineage string `json:"lineage,omitempty"` + Serial int `json:"serial,omitempty"` + + // DMS fields, set only when the bundle records deployment history. The plan targets DeploymentId + // and will create NextVersionId; LastVersionId is the deployment's most recent version at plan + // time. deploy --plan rejects the plan if the deployment moved on from these (see process.go), + // and passes LastVersionId as previous_version_id so the service rejects a stale version too. + DeploymentId string `json:"deployment_id,omitempty"` + NextVersionId string `json:"next_version_id,omitempty"` + LastVersionId string `json:"last_version_id,omitempty"` + + Plan map[string]*PlanEntry `json:"plan,omitzero"` // NotSelected is the number of resources removed by FilterToSelected via the // --select flag. Serialized so the summary survives a deploy from a plan file diff --git a/bundle/direct/apply.go b/bundle/direct/apply.go index 6b2fffe9a16..af2a7277c77 100644 --- a/bundle/direct/apply.go +++ b/bundle/direct/apply.go @@ -58,6 +58,7 @@ func (d *DeploymentUnit) Deploy(ctx context.Context, db *dstate.DeploymentState, } } +// Create creates the resource and records its state. func (d *DeploymentUnit) Create(ctx context.Context, db *dstate.DeploymentState, newState any) error { var newID string var remoteState any @@ -83,7 +84,7 @@ func (d *DeploymentUnit) Create(ctx context.Context, db *dstate.DeploymentState, return err } - err = d.saveState(db, newID, newState, d.DependsOn) + err = d.saveState(ctx, db, newID, newState, d.DependsOn) if err != nil { return fmt.Errorf("saving state after creating id=%s: %w", newID, err) } @@ -126,7 +127,10 @@ func (d *DeploymentUnit) Recreate(ctx context.Context, db *dstate.DeploymentStat // Drop the state entry so a subsequent failure of Create or WaitAfterDelete // leaves no malformed (empty-ID) entry behind. The next plan will see "no // state" and retry as Create. - err = db.DeleteState(d.ResourceKey) + // + // Recorded as a recreate rather than a delete: if the create below fails, this is the + // operation DMS is left with, and it says the resource is mid-recreate. + err = db.DeleteState(ctx, d.ResourceKey, true) if err != nil { return fmt.Errorf("deleting state: %w", err) } @@ -168,12 +172,12 @@ func (d *DeploymentUnit) Update(ctx context.Context, db *dstate.DeploymentState, // The update emptied the resource out (e.g. all grants revoked). Keeping an entry // would report the node as tracked-and-unchanged forever, while a fresh deploy of // the same config plans no node at all; drop it so the two agree. - err = db.DeleteState(d.ResourceKey) + err = db.DeleteState(ctx, d.ResourceKey, false) if err != nil { return fmt.Errorf("deleting state id=%s: %w", id, err) } } else { - err = d.saveState(db, id, newState, d.DependsOn) + err = d.saveState(ctx, db, id, newState, d.DependsOn) if err != nil { return fmt.Errorf("saving state id=%s: %w", id, err) } @@ -218,7 +222,7 @@ func (d *DeploymentUnit) UpdateWithID(ctx context.Context, db *dstate.Deployment return err } - err = d.saveState(db, newID, newState, d.DependsOn) + err = d.saveState(ctx, db, newID, newState, d.DependsOn) if err != nil { return fmt.Errorf("saving state id=%s: %w", oldID, err) } @@ -256,11 +260,15 @@ func (d *DeploymentUnit) Delete(ctx context.Context, db *dstate.DeploymentState, } else if d.deleteConfirmedGone(ctx, oldID) { log.Warnf(ctx, "Treating %s id=%s as already deleted despite delete error: %s", d.ResourceKey, oldID, err) } else { - return fmt.Errorf("deleting id=%s: %w", oldID, err) + // The resource is still there, so the history says why rather than leaving its + // operation pending. The state write below is what records a delete that worked. + err = fmt.Errorf("deleting id=%s: %w", oldID, err) + db.RecordFailure(d.ResourceKey, oldID, err) + return err } } - err = db.DeleteState(d.ResourceKey) + err = db.DeleteState(ctx, d.ResourceKey, false) if err != nil { return fmt.Errorf("deleting state id=%s: %w", oldID, err) } @@ -305,7 +313,7 @@ func (d *DeploymentUnit) Resize(ctx context.Context, db *dstate.DeploymentState, return fmt.Errorf("resizing id=%s: %w", id, err) } - err = d.saveState(db, id, newState, d.DependsOn) + err = d.saveState(ctx, db, id, newState, d.DependsOn) if err != nil { return fmt.Errorf("saving state id=%s: %w", id, err) } @@ -315,11 +323,11 @@ func (d *DeploymentUnit) Resize(ctx context.Context, db *dstate.DeploymentState, // saveState saves a state with sensitive fields replaced by a placeholder value so secrets are never written // to disk in plaintext. -func (d *DeploymentUnit) saveState(db *dstate.DeploymentState, newID string, state any, dependsOn []deployplan.DependsOnEntry) error { +func (d *DeploymentUnit) saveState(ctx context.Context, db *dstate.DeploymentState, newID string, state any, dependsOn []deployplan.DependsOnEntry) error { if err := zeroSensitiveFields(d.Adapter, state); err != nil { return fmt.Errorf("redacting state: %w", err) } - return db.SaveState(d.ResourceKey, newID, state, dependsOn) + return db.SaveState(ctx, d.ResourceKey, newID, state, dependsOn) } func parseState(destType reflect.Type, raw json.RawMessage) (any, error) { diff --git a/bundle/direct/bind.go b/bundle/direct/bind.go index 9760ce95666..9c4ab260e03 100644 --- a/bundle/direct/bind.go +++ b/bundle/direct/bind.go @@ -61,8 +61,10 @@ type BindResult struct { // Call Finalize to commit the state or Cancel to discard. func (b *DeploymentBundle) Bind(ctx context.Context, client *databricks.WorkspaceClient, configRoot *config.Root, statePath, resourceKey, resourceID string) (*BindResult, error) { // Check if the resource is already managed (bound to a different ID) + // The state is opened without a DMS client, so the writes below record nothing; + // phases.Bind and phases.Unbind refuse to run at all when recording is enabled. var checkStateDB dstate.DeploymentState - if err := checkStateDB.Open(ctx, statePath, dstate.WithRecovery(true), dstate.WithWrite(false)); err == nil { + if err := checkStateDB.Open(ctx, statePath, dstate.WithRecovery(true), dstate.WithWrite(false), nil, dstate.WithDeploymentHistory(false), ""); err == nil { existingID := checkStateDB.GetResourceID(resourceKey) if _, err := checkStateDB.Finalize(ctx); err != nil { log.Warnf(ctx, "failed to finalize state: %v", err) @@ -86,14 +88,14 @@ func (b *DeploymentBundle) Bind(ctx context.Context, client *databricks.Workspac } // Open temp state - err := b.StateDB.Open(ctx, tmpStatePath, dstate.WithRecovery(false), dstate.WithWrite(true)) + err := b.StateDB.Open(ctx, tmpStatePath, dstate.WithRecovery(false), dstate.WithWrite(true), nil, dstate.WithDeploymentHistory(false), "") if err != nil { os.Remove(tmpStatePath) return nil, err } // Save state with ID and empty state (like migrate does) - err = b.StateDB.SaveState(resourceKey, resourceID, struct{}{}, nil) + err = b.StateDB.SaveState(ctx, resourceKey, resourceID, struct{}{}, nil) if err != nil { os.Remove(tmpStatePath) return nil, err @@ -109,7 +111,7 @@ func (b *DeploymentBundle) Bind(ctx context.Context, client *databricks.Workspac log.Infof(ctx, "Bound %s to id=%s (in temp state)", resourceKey, resourceID) // First plan + update: populate state with resolved config - err = b.StateDB.Open(ctx, tmpStatePath, dstate.WithRecovery(true), dstate.WithWrite(false)) + err = b.StateDB.Open(ctx, tmpStatePath, dstate.WithRecovery(true), dstate.WithWrite(false), nil, dstate.WithDeploymentHistory(false), "") if err != nil { os.Remove(tmpStatePath) return nil, err @@ -145,13 +147,13 @@ func (b *DeploymentBundle) Bind(ctx context.Context, client *databricks.Workspac } } - err = b.StateDB.Open(ctx, tmpStatePath, dstate.WithRecovery(true), dstate.WithWrite(true)) + err = b.StateDB.Open(ctx, tmpStatePath, dstate.WithRecovery(true), dstate.WithWrite(true), nil, dstate.WithDeploymentHistory(false), "") if err != nil { os.Remove(tmpStatePath) return nil, err } - err = b.StateDB.SaveState(resourceKey, resourceID, sv.Value, dependsOn) + err = b.StateDB.SaveState(ctx, resourceKey, resourceID, sv.Value, dependsOn) if err != nil { os.Remove(tmpStatePath) return nil, err @@ -165,7 +167,7 @@ func (b *DeploymentBundle) Bind(ctx context.Context, client *databricks.Workspac } // Second plan: this is the plan to present to the user (change between remote resource and config) - err = b.StateDB.Open(ctx, tmpStatePath, dstate.WithRecovery(true), dstate.WithWrite(false)) + err = b.StateDB.Open(ctx, tmpStatePath, dstate.WithRecovery(true), dstate.WithWrite(false), nil, dstate.WithDeploymentHistory(false), "") if err != nil { os.Remove(tmpStatePath) return nil, err @@ -215,13 +217,13 @@ func (result *BindResult) Cancel() { // Unbind removes a resource from direct engine state without deleting // the workspace resource. Also removes associated permissions/grants entries. func (b *DeploymentBundle) Unbind(ctx context.Context, statePath, resourceKey string) error { - err := b.StateDB.Open(ctx, statePath, dstate.WithRecovery(true), dstate.WithWrite(true)) + err := b.StateDB.Open(ctx, statePath, dstate.WithRecovery(true), dstate.WithWrite(true), nil, dstate.WithDeploymentHistory(false), "") if err != nil { return err } // Delete the main resource - err = b.StateDB.DeleteState(resourceKey) + err = b.StateDB.DeleteState(ctx, resourceKey, false) if err != nil { return err } @@ -235,7 +237,7 @@ func (b *DeploymentBundle) Unbind(ctx context.Context, statePath, resourceKey st for key := range b.StateDB.Data.State { if key == permissionsKey || key == grantsKey || strings.HasPrefix(key, resourceKey+".") { - err = b.StateDB.DeleteState(key) + err = b.StateDB.DeleteState(ctx, key, false) if err != nil { return err } diff --git a/bundle/direct/bundle_apply.go b/bundle/direct/bundle_apply.go index 424ae2bdec9..72465fad346 100644 --- a/bundle/direct/bundle_apply.go +++ b/bundle/direct/bundle_apply.go @@ -43,6 +43,9 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa return } + // The state DB records every write with DMS from here on (via the buffer StartRecording opened + // in the deploy phase), so the service mirrors the WAL. Writes go out on one background + // goroutine, off the apply path, and are drained below once every worker has finished recording. g.Run(defaultParallelism, func(resourceKey string, failedDependency *string) bool { entry, err := plan.WriteLockEntry(resourceKey) if err != nil { @@ -73,6 +76,14 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa return false } + // Stop resource CRUD once recording state with DMS has failed. + if buf := b.DmsAsyncOperationClient; buf != nil { + if err := buf.Err(); err != nil { + logdiag.LogError(ctx, fmt.Errorf("%s: %w", errorPrefix, err)) + return false + } + } + adapter, err := b.getAdapterForKey(resourceKey) if adapter == nil { logdiag.LogError(ctx, fmt.Errorf("%s: internal error: cannot get adapter: %w", errorPrefix, err)) @@ -103,7 +114,7 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa if entry.Gone { // Planning confirmed the resource is already deleted remotely; only // remove it from the state, without calling the delete API. - err = b.StateDB.DeleteState(resourceKey) + err = b.StateDB.DeleteState(ctx, resourceKey, false) } else { err = d.Destroy(ctx, &b.StateDB) } @@ -134,8 +145,15 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa } // TODO: redo calcDiff to downgrade planned action if possible (?) + // + // Success is recorded by the state writes inside Deploy, so a recreate reports + // each of its steps. err = d.Deploy(ctx, &b.StateDB, sv.Value, action, entry) if err != nil { + // Empty for a create that never got an ID, and for a recreate whose delete + // step already dropped it. + failedID := b.StateDB.GetResourceID(resourceKey) + b.StateDB.RecordFailure(resourceKey, failedID, err) logdiag.LogError(ctx, fmt.Errorf("%s: %w", errorPrefix, err)) return false } @@ -162,6 +180,14 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa return true }) + + // Wait for the uploads and report a failure here, with the deploy's other errors. The phase + // completes the version afterwards and drains again, quietly, so this is not printed twice. + if buf := b.DmsAsyncOperationClient; buf != nil { + if err := buf.Drain(); err != nil { + logdiag.LogError(ctx, err) + } + } } func (b *DeploymentBundle) LookupReferencePostDeploy(ctx context.Context, path *structpath.PathNode) (any, error) { diff --git a/bundle/direct/bundle_plan.go b/bundle/direct/bundle_plan.go index 739ca3bd8e7..45d5b2d3663 100644 --- a/bundle/direct/bundle_plan.go +++ b/bundle/direct/bundle_plan.go @@ -8,6 +8,7 @@ import ( "maps" "reflect" "slices" + "strconv" "strings" "github.com/databricks/cli/bundle/config" @@ -15,6 +16,7 @@ import ( "github.com/databricks/cli/bundle/direct/dresources" "github.com/databricks/cli/bundle/direct/dstate" "github.com/databricks/cli/bundle/terraform_dabs_map" + "github.com/databricks/cli/libs/dms" "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/dyn/dynvar" "github.com/databricks/cli/libs/log" @@ -25,6 +27,7 @@ import ( "github.com/databricks/cli/libs/structs/structvar" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/apierr" + "github.com/databricks/databricks-sdk-go/service/jobs" ) var errDelayed = errors.New("must be resolved after apply") @@ -58,8 +61,10 @@ func ValidatePlanAgainstState(stateDB *dstate.DeploymentState, plan *deployplan. return nil } -// InitForApply initializes the DeploymentBundle for applying a pre-computed plan. -// StateDB must already be open for write before calling this function. +// InitForApply initializes the DeploymentBundle for applying a pre-computed plan: it parses the +// plan's entries (loaded as JSON when the plan came from a file) into the typed state cache the +// apply reads. A first deployment's id does not exist until the deploy creates it after approval; +// StampDeploymentID fills it in then. StateDB must already be open for write before calling this. func (b *DeploymentBundle) InitForApply(ctx context.Context, client *databricks.WorkspaceClient, plan *deployplan.Plan) error { b.StateDB.AssertOpenedForWrite() @@ -113,6 +118,41 @@ func (b *DeploymentBundle) InitForApply(ctx context.Context, client *databricks. return nil } +// StampDeploymentID fills deploymentID into the plan's jobs and pipelines that still lack one, in +// both the state cache the apply reads and the plan JSON. It is called after approval creates the +// deployment, for a first deploy whose id did not exist at plan time; the version and any +// pre-existing id were stamped then. +func (b *DeploymentBundle) StampDeploymentID(deploymentID string) error { + for resourceKey, entry := range b.Plan.Plan { + if entry.NewState == nil || len(entry.NewState.Value) == 0 { + continue + } + sv, ok := b.StateCache.Load(resourceKey) + if !ok { + continue + } + var stamped bool + switch v := sv.Value.(type) { + case *jobs.JobSettings: + if v.Deployment.DeploymentId == "" { + v.Deployment.DeploymentId = deploymentID + stamped = true + } + case *dresources.PipelineState: + if v.Deployment.DeploymentId == "" { + v.Deployment.DeploymentId = deploymentID + stamped = true + } + } + if stamped { + if err := sv.SyncToJSON(entry.NewState); err != nil { + return fmt.Errorf("%s: stamping deployment into plan: %w", resourceKey, err) + } + } + } + return nil +} + // CalculatePlan computes the deployment plan by comparing local config against remote state. // StateDB must already be open for read before calling this function. func (b *DeploymentBundle) CalculatePlan(ctx context.Context, client *databricks.WorkspaceClient, configRoot *config.Root) (*deployplan.Plan, error) { @@ -128,6 +168,21 @@ func (b *DeploymentBundle) CalculatePlan(ctx context.Context, client *databricks return nil, fmt.Errorf("reading config: %w", err) } + // Record the DMS deployment and version this plan targets. A saved plan carries them so + // deploy --plan can reject a plan the deployment has moved on from, and pass last_version_id + // as previous_version_id. History is set only while recording, so other plans leave these empty. + // configRoot is nil for destroy, which records its version separately. + if configRoot != nil && configRoot.Bundle.Deployment.History != nil { + h := configRoot.Bundle.Deployment.History + next, err := dms.NextVersion(h.LatestVersionID) + if err != nil { + return nil, fmt.Errorf("computing next deployment version: %w", err) + } + plan.DeploymentId = h.DeploymentID + plan.LastVersionId = h.LatestVersionID + plan.NextVersionId = strconv.FormatInt(next, 10) + } + b.Plan = plan g, err := makeGraph(plan) diff --git a/bundle/direct/dstate/dms.go b/bundle/direct/dstate/dms.go new file mode 100644 index 00000000000..c0d4ffb2a7f --- /dev/null +++ b/bundle/direct/dstate/dms.go @@ -0,0 +1,58 @@ +package dstate + +import ( + "encoding/json" + "fmt" + + "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/cli/libs/dms" +) + +// RecordedState is what the CLI serializes into the DMS operation's state field. It wraps the +// config so depends_on survives the round trip: DMS has no field for dependency edges, and +// nesting them in the config would collide with resource fields of the same name. +type RecordedState struct { + State json.RawMessage `json:"state"` + DependsOn []deployplan.DependsOnEntry `json:"depends_on,omitempty"` +} + +// applyDMSState replaces the file-derived resource state with what DMS recorded. DMS owns the +// resource set outright, so this runs on every recorded open and the file's copy is never read +// back as the truth: an empty set means the service tracks nothing, whether because the deploy +// created nothing or because the deployment is gone. The caller holds db.mu. +func (db *DeploymentState) applyDMSState(recorded []dms.Resource) error { + // Built first and assigned together, so a malformed envelope leaves the state as it was + // rather than half replaced. + resources := make(map[string]ResourceEntry, len(recorded)) + stateIDs := make(map[string]string, len(recorded)) + for _, res := range recorded { + entry, err := stateEntry(res) + if err != nil { + return err + } + resources[res.Key] = entry + stateIDs[res.Key] = entry.ID + } + + db.Data.State = resources + db.stateIDs = stateIDs + return nil +} + +// stateEntry unwraps the envelope the write path recorded for a resource. +func stateEntry(res dms.Resource) (ResourceEntry, error) { + var recorded RecordedState + if res.State != "" { + // The service stores state as an opaque string, so it arrives as the serialized + // envelope the write side sent. + if err := json.Unmarshal([]byte(res.State), &recorded); err != nil { + return ResourceEntry{}, fmt.Errorf("interpreting state recorded for %s: %w", res.Key, err) + } + } + + return ResourceEntry{ + ID: res.ID, + State: recorded.State, + DependsOn: recorded.DependsOn, + }, nil +} diff --git a/bundle/direct/dstate/dms_test.go b/bundle/direct/dstate/dms_test.go new file mode 100644 index 00000000000..8edcf563ee8 --- /dev/null +++ b/bundle/direct/dstate/dms_test.go @@ -0,0 +1,100 @@ +package dstate + +import ( + "encoding/json" + "testing" + + "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/cli/libs/dms" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestApplyDMSState(t *testing.T) { + // The service stores state as an opaque string, so the envelope the write path sent + // arrives verbatim. + envelope := `{"state":{"name":"foo"},"depends_on":[{"node":"resources.pipelines.bar","label":"${resources.pipelines.bar.id}"}]}` + + // What the file loaded, for the cases that check it is replaced or survives. + fileState := map[string]ResourceEntry{ + "resources.jobs.bar": {ID: "file-id", State: json.RawMessage(`{"name":"from-file"}`)}, + } + + tests := []struct { + name string + existing map[string]ResourceEntry + recorded []dms.Resource + want map[string]ResourceEntry + wantErr string + }{ + { + name: "the envelope is unwrapped, depends_on and all", + recorded: []dms.Resource{ + {Key: "resources.jobs.foo", ID: "123", State: envelope}, + {Key: "resources.pipelines.bar", ID: "456"}, + }, + // depends_on comes back from the envelope, so a bundle whose local state was + // wiped still has the edges needed for delete ordering. + want: map[string]ResourceEntry{ + "resources.jobs.foo": { + ID: "123", + State: json.RawMessage(`{"name":"foo"}`), + DependsOn: []deployplan.DependsOnEntry{{Node: "resources.pipelines.bar", Label: "${resources.pipelines.bar.id}"}}, + }, + "resources.pipelines.bar": {ID: "456"}, + }, + }, + { + name: "what DMS holds replaces what the file loaded", + existing: fileState, + recorded: []dms.Resource{{Key: "resources.jobs.foo", ID: "dms-id", State: `{"state":{"name":"from-dms"}}`}}, + want: map[string]ResourceEntry{"resources.jobs.foo": {ID: "dms-id", State: json.RawMessage(`{"name":"from-dms"}`)}}, + }, + { + // Nil is what a list holding nothing returns, and it means a successful deploy of + // nothing rather than missing data. + name: "nothing recorded empties the state", + existing: fileState, + want: map[string]ResourceEntry{}, + }, + { + // The good resource comes first, so the error lands mid-way: what the file loaded + // has to survive whole rather than end up half replaced. + name: "a malformed envelope leaves the state as it was", + existing: fileState, + recorded: []dms.Resource{ + {Key: "resources.jobs.ok", ID: "999", State: `{"state":{"name":"ok"}}`}, + {Key: "resources.jobs.foo", ID: "123", State: "not json"}, + }, + wantErr: "interpreting state recorded for resources.jobs.foo", + want: fileState, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var db DeploymentState + db.Data.State = tt.existing + db.stateIDs = stateIDsOf(tt.existing) + + err := db.applyDMSState(tt.recorded) + if tt.wantErr != "" { + assert.ErrorContains(t, err, tt.wantErr) + } else { + require.NoError(t, err) + } + + assert.Equal(t, tt.want, db.Data.State) + assert.Equal(t, stateIDsOf(tt.want), db.stateIDs) + }) + } +} + +// stateIDsOf is the id index the state keeps alongside its entries. +func stateIDsOf(entries map[string]ResourceEntry) map[string]string { + ids := make(map[string]string, len(entries)) + for key, entry := range entries { + ids[key] = entry.ID + } + return ids +} diff --git a/bundle/direct/dstate/migrate.go b/bundle/direct/dstate/migrate.go index e4d21a7054a..dccb06686a5 100644 --- a/bundle/direct/dstate/migrate.go +++ b/bundle/direct/dstate/migrate.go @@ -3,8 +3,6 @@ package dstate import ( "encoding/json" "fmt" - "slices" - "strings" "github.com/databricks/cli/bundle/direct/dresources" "github.com/databricks/cli/libs/structs/structpath" @@ -15,24 +13,6 @@ import ( // migrateState runs all necessary migrations on the database. // It is called after loading state from disk. func migrateState(db *Database) error { - // featureStateVersion states carry a feature list this CLI does not yet write or - // understand (see the featureStateVersion doc comment). A featureStateVersion - // state with no features is equivalent to currentStateVersion, so accept it and - // return without running the migrations below, leaving the on-disk version at - // featureStateVersion rather than flipping it down. One that records any feature - // depends on capabilities this CLI lacks, so refuse it and tell the user to upgrade. - if db.StateVersion == featureStateVersion { - if len(db.Features) == 0 { - return nil - } - features := make([]string, 0, len(db.Features)) - for name := range db.Features { - features = append(features, name) - } - slices.Sort(features) - return fmt.Errorf("the deployment state requires features this CLI does not support: %s; upgrade to the latest CLI version and see %s for more information", strings.Join(features, ", "), featuresDocURL) - } - if db.StateVersion == currentStateVersion { return nil } diff --git a/bundle/direct/dstate/state.go b/bundle/direct/dstate/state.go index af2e2638eb3..4e2f8c03344 100644 --- a/bundle/direct/dstate/state.go +++ b/bundle/direct/dstate/state.go @@ -10,13 +10,17 @@ import ( "io/fs" "os" "path/filepath" + "slices" + "strings" "sync" "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/deployplan" "github.com/databricks/cli/bundle/statemgmt/resourcestate" "github.com/databricks/cli/internal/build" + "github.com/databricks/cli/libs/dms" "github.com/databricks/cli/libs/log" + "github.com/databricks/databricks-sdk-go" "github.com/google/uuid" ) @@ -28,36 +32,49 @@ const ( maxWalEntrySize = 10 * 1024 * 1024 walSuffix = ".wal" - // featureStateVersion is the schema version a future CLI will write once it - // records deployment state "feature flags" (see Header.Features). This CLI does - // not write it and records no features; it exists now only so this CLI reads - // such states correctly (see migrateState): - // - featureStateVersion with no features -> accept and leave the version as-is - // - featureStateVersion with any feature -> refuse, tell the user to upgrade - // - // A featureStateVersion state with no features is equivalent to - // currentStateVersion, but we deliberately do not flip the on-disk version down - // to currentStateVersion: a state written at featureStateVersion stays at - // featureStateVersion. This is forward-compat scaffolding so that a later release - // can start writing featureStateVersion + features without older CLIs (with this - // change) either mishandling a feature they lack or rejecting a featureless state - // outright. featureStateVersion is always 3. - featureStateVersion = 3 - - // supportedStateVersion is the highest schema version this CLI can read. It is - // normally equal to currentStateVersion — the version this CLI reads is the - // version it writes — and exceeds it only during a two-phase version bump like - // the current feature-flag scaffolding, where this CLI reads (but does not - // write) featureStateVersion. A state newer than this is rejected as too new. - supportedStateVersion = featureStateVersion + // supportedStateVersion is the highest schema version this CLI can read: the + // version it writes. A state newer than this is rejected as too new. + supportedStateVersion = currentStateVersion ) -// featuresDocURL is the single documentation page describing deployment state -// feature flags. It is shown when a state records a feature this CLI does not -// support; it is a fixed link for all features. The #state-features anchor points -// at the feature table; if it ever breaks, the user still lands on the page. +// featureRecordDeploymentHistory marks a state whose resources are also recorded with the +// deployment metadata service. Both stores are kept in step, so the marker is what tells a +// reader the two already agree. A CLI that does not know the name refuses the state rather +// than deploying over a deployment it would leave the service out of step with. +// +// The marker is sticky: once a deployment is recorded, the service holds resources that a +// CLI which is not recording must not touch. So turning recording off does not clear it, and +// deploying such a state without recording is refused (see RequiresDeploymentHistory). +const featureRecordDeploymentHistory = "record_deployment_history" + +// featuresDocURL explains the state-features mechanism in the error shown when this CLI refuses a +// state that depends on a feature it does not recognize. const featuresDocURL = "https://docs.databricks.com/aws/en/dev-tools/bundles/state-features#state-features" +// recognizedFeatures is the set of state feature flags this CLI understands. A state depending on +// any feature not listed here is refused (see checkStateFeatures), so a newer CLI's feature is not +// silently clobbered by this one. +var recognizedFeatures = map[string]struct{}{ + featureRecordDeploymentHistory: {}, +} + +// checkStateFeatures refuses a state that depends on a feature this CLI does not recognize: keeping +// the state and the service in step needs every reader to understand the features in play, so an +// unknown one belongs to a newer CLI and this one refuses rather than clobber it. +func checkStateFeatures(features map[string]struct{}) error { + var unsupported []string + for feature := range features { + if _, ok := recognizedFeatures[feature]; !ok { + unsupported = append(unsupported, feature) + } + } + if len(unsupported) == 0 { + return nil + } + slices.Sort(unsupported) + return fmt.Errorf("the deployment state requires features this CLI does not support: %s; upgrade to the latest CLI version and see %s for more information", strings.Join(unsupported, ", "), featuresDocURL) +} + // errStaleWAL is returned when the WAL serial is behind the expected serial. // The caller should delete the stale WAL and proceed normally. var errStaleWAL = errors.New("stale WAL") @@ -70,6 +87,14 @@ type DeploymentState struct { // Maps resource key to ID. Unlike Data.State, this is up to date during writes (deploys). stateIDs map[string]string + + // operationBuffer records each state write with DMS. Nil unless the bundle records deployment + // history, in which case StartRecording installs it once the version exists. + operationBuffer *dms.OperationBuffer + + // dmsClient talks to the deployment metadata service. Open builds it from the workspace + // client when the deployment records history; nil otherwise. + dmsClient *dms.Client } type Header struct { @@ -84,10 +109,9 @@ type Header struct { Serial int `json:"serial"` // Features maps each feature flag this state depends on to a (currently empty) - // value. This CLI writes no features; it only reads the field to detect a state - // that depends on features it lacks and refuse it (see migrateState). It is a - // map so a future CLI can attach per-feature data without reshaping the state. - // Empty/omitted for states that use no features. + // value. It is read to detect a state that depends on features this CLI lacks and + // refuse it (see migrateState). It is a map so a future CLI can attach per-feature + // data without reshaping the state. Empty/omitted for states that use no features. Features map[string]struct{} `json:"features,omitempty"` } @@ -111,6 +135,52 @@ type WALEntry struct { Value *ResourceEntry `json:"v,omitempty"` // nil means delete } +// StartRecording opens the operation buffer that records each subsequent state write with DMS +// under deploymentID and versionID, so what the service holds mirrors the WAL. It uses the client +// Open built, so it is called only when the deployment records history, and once the version +// exists (after approval) - which is why it is not an Open option. Returns the buffer so the +// deploy can drain it. +func (db *DeploymentState) StartRecording(ctx context.Context, deploymentID string, versionID int64) *dms.OperationBuffer { + buf := dms.StartOperationBuffer(ctx, db.dmsClient, deploymentID, versionID) + + db.mu.Lock() + defer db.mu.Unlock() + db.operationBuffer = buf + return buf +} + +// RecordFailure records that a resource did not apply, so the history says why rather than +// leaving the resource out. resourceID is the id it had before the failure. +func (db *DeploymentState) RecordFailure(resourceKey, resourceID string, cause error) { + r := db.recorder() + if r == nil { + return + } + + // The service refuses a failure that leaves a live resource described by nothing, so re-state + // what it still has. An empty resourceID means there is nothing left: a create that never + // landed, or a recreate whose delete already went through. + var recorded json.RawMessage + if entry, ok := db.GetResourceEntry(resourceKey); resourceID != "" && ok && len(entry.State) > 0 { + var err error + recorded, err = json.Marshal(RecordedState{State: entry.State, DependsOn: entry.DependsOn}) + if err != nil { + // Nothing the caller can act on, so record the failure without the state. + recorded = nil + } + } + + r.RecordFailure(resourceKey, resourceID, recorded, cause) +} + +// recorder reads the buffer under db.mu, which guards it, and returns nil when the bundle does +// not record deployment history. +func (db *DeploymentState) recorder() *dms.OperationBuffer { + db.mu.Lock() + defer db.mu.Unlock() + return db.operationBuffer +} + func NewDatabase(lineage string, serial int) Database { return Database{ Header: Header{ @@ -123,47 +193,81 @@ func NewDatabase(lineage string, serial int) Database { } } -func (db *DeploymentState) SaveState(key, newID string, state any, dependsOn []deployplan.DependsOnEntry) error { +// SaveState records the resource's state after an operation was applied to it. +func (db *DeploymentState) SaveState(ctx context.Context, key, newID string, state any, dependsOn []deployplan.DependsOnEntry) error { db.AssertOpenedForWrite() - db.mu.Lock() - defer db.mu.Unlock() - - if db.Data.State == nil { - db.Data.State = make(map[string]ResourceEntry) - } jsonMessage, err := json.Marshal(state) if err != nil { return err } - entry := ResourceEntry{ ID: newID, State: json.RawMessage(jsonMessage), DependsOn: dependsOn, } + db.mu.Lock() + if db.Data.State == nil { + db.Data.State = make(map[string]ResourceEntry) + } err = appendJSONLine(db.walFile, WALEntry{Key: key, Value: &entry}) if err == nil { db.stateIDs[key] = newID } - return err + buf := db.operationBuffer + db.mu.Unlock() + + if err != nil { + return err + } + if buf == nil { + return nil + } + + // Recorded after the WAL write, so DMS never reports state the deploy failed to persist, + // and outside db.mu because recording waits when the service is behind - waiting under the + // lock would hold up every other resource's write. + recorded, err := json.Marshal(RecordedState{State: entry.State, DependsOn: dependsOn}) + if err != nil { + return err + } + buf.RecordOperation(ctx, key, false, newID, recorded) + + return nil } -func (db *DeploymentState) DeleteState(key string) error { +// DeleteState drops the resource's state entry: the resource is gone. inProgress records the +// operation as unfinished, which is what the first half of a recreate wants - an interrupted +// deploy must not leave the resource described as finished. +func (db *DeploymentState) DeleteState(ctx context.Context, key string, inProgress bool) error { db.AssertOpenedForWrite() - db.mu.Lock() - defer db.mu.Unlock() + db.mu.Lock() if db.Data.State == nil { + db.mu.Unlock() return nil } - + // Read before the delete: DMS needs the id to say which resource went away. + deletedID := db.stateIDs[key] err := appendJSONLine(db.walFile, WALEntry{Key: key}) if err == nil { delete(db.stateIDs, key) } - return err + buf := db.operationBuffer + db.mu.Unlock() + + if err != nil { + return err + } + + // State is nil: the resource no longer exists. Recorded outside the lock for the + // same reason as SaveState. + if buf != nil { + buf.RecordOperation(ctx, key, inProgress, deletedID, nil) + } + + return nil } func (db *DeploymentState) GetResourceEntry(key string) (ResourceEntry, bool) { @@ -201,6 +305,24 @@ func (db *DeploymentState) StateCLIVersion() string { return db.Data.CLIVersion } +// RequiresDeploymentHistory reports whether the state depends on the deployment metadata +// service recording it. Deploying such a state without recording would leave the service +// holding a deployment that no longer matches, so the caller refuses instead. +func (db *DeploymentState) RequiresDeploymentHistory() bool { + db.AssertOpenedForReadOrWrite() + db.mu.Lock() + defer db.mu.Unlock() + + _, ok := db.Data.Features[featureRecordDeploymentHistory] + return ok +} + +// DmsClient returns the deployment metadata service client Open built from the workspace client, +// or nil when the deployment does not record history. +func (db *DeploymentState) DmsClient() *dms.Client { + return db.dmsClient +} + // GetOrInitLineage returns the deployment lineage, generating and storing a new // one if the state does not have one yet. It is the single place the lineage is // initialized, shared so the direct deployment engine (when it writes state, via @@ -228,9 +350,21 @@ type ( // If true, the state is opened in Write mode, which enables methods such as SaveState // but disables GetResourceEntry (since writes go strictly into WAL and not in memory). WithWrite bool + + // If true, the deployment records history with the metadata service: Open builds a DMS + // client from the workspace client, reads resources from the service, and refuses a state + // that tracks resources without the recording marker. It forces WithRecovery off, since the + // service is the source of truth and a leftover WAL is discarded rather than replayed. + WithDeploymentHistory bool ) -func (db *DeploymentState) Open(ctx context.Context, path string, withRecovery WithRecovery, withWrite WithWrite) error { +// Open reads the deployment state from disk, recovering the WAL when withRecovery is set. +// When withDeploymentHistory is set it builds a DMS client from wsClient and reads resources from +// the service instead, with dmsDeploymentID the id the service holds (empty before the first +// recorded deploy); lineage and serial still come from the file, since that is what the write path +// increments. Open only reads through the client - StartRecording installs the write path once a +// version exists. +func (db *DeploymentState) Open(ctx context.Context, path string, withRecovery WithRecovery, withWrite WithWrite, wsClient *databricks.WorkspaceClient, withDeploymentHistory WithDeploymentHistory, dmsDeploymentID string) error { db.mu.Lock() defer db.mu.Unlock() @@ -238,7 +372,7 @@ func (db *DeploymentState) Open(ctx context.Context, path string, withRecovery W panic(fmt.Sprintf("state already opened: %v, cannot open %v", db.Path, path)) } - err := db.unlockedOpen(ctx, path, withRecovery, withWrite) + err := db.unlockedOpen(ctx, path, withRecovery, withWrite, wsClient, withDeploymentHistory, dmsDeploymentID) if err != nil { // A failed open must leave the receiver closed. unlockedOpen assigns // db.Path before every fallible step, so without this the receiver stays @@ -260,8 +394,20 @@ func (db *DeploymentState) reset() { db.stateIDs = nil } -func (db *DeploymentState) unlockedOpen(ctx context.Context, path string, withRecovery WithRecovery, withWrite WithWrite) error { +func (db *DeploymentState) unlockedOpen(ctx context.Context, path string, withRecovery WithRecovery, withWrite WithWrite, wsClient *databricks.WorkspaceClient, withDeploymentHistory WithDeploymentHistory, dmsDeploymentID string) error { db.Path = path + + if withDeploymentHistory { + // The service is the source of truth for a recorded deployment, so a leftover WAL is + // discarded below rather than replayed: force recovery off regardless of what was asked. + withRecovery = false + client, err := dms.NewClient(wsClient) + if err != nil { + return err + } + db.dmsClient = client + } + data, err := os.ReadFile(db.Path) if err != nil { if errors.Is(err, fs.ErrNotExist) { @@ -281,14 +427,22 @@ func (db *DeploymentState) unlockedOpen(ctx context.Context, path string, withRe } walPath := db.Path + walSuffix - _, err = os.Stat(walPath) - switch { - case errors.Is(err, fs.ErrNotExist): - // no WAL, nothing to do - case err != nil: - return fmt.Errorf("failed to stat WAL file %s: %w", walPath, err) - default: // WAL exists - if withRecovery { + _, statErr := os.Stat(walPath) + walExists := statErr == nil + if statErr != nil && !errors.Is(statErr, fs.ErrNotExist) { + return fmt.Errorf("failed to stat WAL file %s: %w", walPath, statErr) + } + + // A recorded deployment gets its resources from the service (applyDMSState below), so the WAL - + // a local log of an uncommitted deploy - is not merged. A leftover from a crash or a declined + // deploy is removed so the next write's exclusive create of the WAL does not fail. Without + // recording, replay the WAL as usual. + if walExists { + if withDeploymentHistory { + if err := os.Remove(walPath); err != nil { + return fmt.Errorf("removing WAL file %s: %w", walPath, err) + } + } else if withRecovery { if err := db.replayWAL(ctx); err != nil { return fmt.Errorf("reading state from %s: %w", path, err) } @@ -301,6 +455,46 @@ func (db *DeploymentState) unlockedOpen(ctx context.Context, path string, withRe return fmt.Errorf("migrating state %s: %w", path, err) } + if err := checkStateFeatures(db.Data.Features); err != nil { + return err + } + + if withDeploymentHistory { + // The service is the source of truth for a recorded deployment; local state is irrelevant. + // This state was pulled from the workspace, so it reflects the remote and not just a local + // cache, and a leftover WAL was discarded above rather than replayed. It records the feature + // once recording owns it, so a recorded deployment carries no resources here (a tombstone) + // and applyDMSState below loads the service's. A state that still tracks resources without + // the feature is a deployment made before recording: treating the service as authoritative + // would create those resources a second time, so refuse it (adopting one is a followup). + if _, recorded := db.Data.Features[featureRecordDeploymentHistory]; !recorded && len(db.Data.State) > 0 { + return errors.New(`this deployment already exists and is not recorded with the deployment metadata service, so it cannot be recorded without redeploying its resources + +To record this bundle's history, start it over as a new deployment: + 1. remove experimental.record_deployment_history from your bundle configuration + 2. run "databricks bundle destroy" to delete the existing resources + 3. add experimental.record_deployment_history back and deploy again`) + } + + // Mark the state as depending on the service so this CLI keeps recording it (see + // RequiresDeploymentHistory). unlockedSave then writes the header alone. Bumping the state + // version so older CLIs refuse a recorded state is a followup. + if db.Data.Features == nil { + db.Data.Features = make(map[string]struct{}, 1) + } + db.Data.Features[featureRecordDeploymentHistory] = struct{}{} + + if dmsDeploymentID != "" { + resources, err := db.dmsClient.ListResources(ctx, dmsDeploymentID) + if err != nil { + return err + } + if err := db.applyDMSState(resources); err != nil { + return err + } + } + } + if withWrite { if err := os.MkdirAll(filepath.Dir(walPath), 0o755); err != nil { return fmt.Errorf("failed to create state directory: %w", err) @@ -405,6 +599,7 @@ func (db *DeploymentState) mergeWalIntoState(ctx context.Context) (bool, error) if header.Serial > expectedSerial { return false, fmt.Errorf("WAL serial (%d) is ahead of expected (%d), state may be corrupted", header.Serial, expectedSerial) } + newSerial = header.Serial newCLIVersion = header.CLIVersion } else { @@ -604,7 +799,7 @@ func (db *DeploymentState) ExportState(ctx context.Context) resourcestate.Export // the WAL. A torn write would therefore leave a state file that Open rejects // next to an intact WAL it never reads. func (db *DeploymentState) unlockedSave() error { - data, err := json.MarshalIndent(db.Data, "", " ") + data, err := json.MarshalIndent(db.dataToPersist(), "", " ") if err != nil { return err } @@ -640,6 +835,20 @@ func (db *DeploymentState) unlockedSave() error { return nil } +// dataToPersist returns what the state file should hold: db.Data itself, unless the deployment +// records its history, in which case it is the header alone. The service holds those resources and +// is where they are read back from, so writing them here too would be a second, never-read copy. +// Returns a copy, leaving the in-memory state - what the rest of the deploy works from - untouched. +func (db *DeploymentState) dataToPersist() Database { + data := db.Data + if _, recorded := data.Features[featureRecordDeploymentHistory]; !recorded { + return data + } + + data.State = map[string]ResourceEntry{} + return data +} + func appendJSONLine(file *os.File, obj any) error { data, err := json.Marshal(obj) if err != nil { diff --git a/bundle/direct/dstate/state_test.go b/bundle/direct/dstate/state_test.go index 34ed9ff1c5a..8ccd6277ecb 100644 --- a/bundle/direct/dstate/state_test.go +++ b/bundle/direct/dstate/state_test.go @@ -21,14 +21,14 @@ func TestOpenSaveFinalizeRoundTrip(t *testing.T) { path := filepath.Join(t.TempDir(), "state.json") var db DeploymentState - require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true))) + require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil, WithDeploymentHistory(false), "")) - require.NoError(t, db.SaveState("jobs.my_job", "123", map[string]string{"key": "val"}, nil)) + require.NoError(t, db.SaveState(t.Context(), "jobs.my_job", "123", map[string]string{"key": "val"}, nil)) mustFinalize(t, &db) // Re-open and verify persisted data. var db2 DeploymentState - require.NoError(t, db2.Open(t.Context(), path, WithRecovery(false), WithWrite(false))) + require.NoError(t, db2.Open(t.Context(), path, WithRecovery(false), WithWrite(false), nil, WithDeploymentHistory(false), "")) assert.Equal(t, 1, db2.Data.Serial) assert.Equal(t, "123", db2.GetResourceID("jobs.my_job")) mustFinalize(t, &db2) @@ -38,7 +38,7 @@ func TestFinalizeWithNoEntriesDoesNotWriteStateFile(t *testing.T) { path := filepath.Join(t.TempDir(), "state.json") var db DeploymentState - require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true))) + require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil, WithDeploymentHistory(false), "")) mustFinalize(t, &db) _, err := os.Stat(path) @@ -94,10 +94,10 @@ func TestPanicOnDoubleOpen(t *testing.T) { path := filepath.Join(t.TempDir(), "state.json") var db DeploymentState - require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true))) + require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil, WithDeploymentHistory(false), "")) assert.Panics(t, func() { - _ = db.Open(t.Context(), path, WithRecovery(true), WithWrite(true)) + _ = db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil, WithDeploymentHistory(false), "") }) mustFinalize(t, &db) } @@ -115,12 +115,12 @@ func TestCLIVersionRecordsLastWriter(t *testing.T) { require.NoError(t, os.WriteFile(path, []byte(seed), 0o600)) var db DeploymentState - require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true))) - require.NoError(t, db.SaveState("resources.jobs.my_job", "123", map[string]string{"k": "v"}, nil)) + require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil, WithDeploymentHistory(false), "")) + require.NoError(t, db.SaveState(t.Context(), "resources.jobs.my_job", "123", map[string]string{"k": "v"}, nil)) mustFinalize(t, &db) var reopened DeploymentState - require.NoError(t, reopened.Open(t.Context(), path, WithRecovery(false), WithWrite(false))) + require.NoError(t, reopened.Open(t.Context(), path, WithRecovery(false), WithWrite(false), nil, WithDeploymentHistory(false), "")) assert.Equal(t, build.GetInfo().Version, reopened.Data.CLIVersion) assert.Equal(t, 2, reopened.Data.Serial) mustFinalize(t, &reopened) @@ -142,7 +142,7 @@ func TestHeaderOnlyWALDoesNotUpdateCLIVersion(t *testing.T) { require.NoError(t, os.WriteFile(walPath, append(headerLine, '\n'), 0o600)) var recovered DeploymentState - require.NoError(t, recovered.Open(t.Context(), path, WithRecovery(true), WithWrite(false))) + require.NoError(t, recovered.Open(t.Context(), path, WithRecovery(true), WithWrite(false), nil, WithDeploymentHistory(false), "")) assert.Equal(t, "0.1.2", recovered.Data.CLIVersion, "a header-only WAL wrote no state, so the version must not move") assert.Equal(t, 1, recovered.Data.Serial) mustFinalize(t, &recovered) @@ -154,12 +154,12 @@ func TestHeaderOnlyWALRecoveryDoesNotAdvanceSerial(t *testing.T) { // Commit serial 1 with one resource. var db DeploymentState - require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true))) - require.NoError(t, db.SaveState("jobs.my_job", "123", map[string]string{}, nil)) + require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil, WithDeploymentHistory(false), "")) + require.NoError(t, db.SaveState(t.Context(), "jobs.my_job", "123", map[string]string{}, nil)) mustFinalize(t, &db) var committed DeploymentState - require.NoError(t, committed.Open(t.Context(), path, WithRecovery(false), WithWrite(false))) + require.NoError(t, committed.Open(t.Context(), path, WithRecovery(false), WithWrite(false), nil, WithDeploymentHistory(false), "")) lineage := committed.Data.Lineage require.Equal(t, 1, committed.Data.Serial) mustFinalize(t, &committed) @@ -175,60 +175,28 @@ func TestHeaderOnlyWALRecoveryDoesNotAdvanceSerial(t *testing.T) { require.NoError(t, os.WriteFile(walPath, append(headerLine, '\n'), 0o600)) var recovered DeploymentState - require.NoError(t, recovered.Open(t.Context(), path, WithRecovery(true), WithWrite(false))) + require.NoError(t, recovered.Open(t.Context(), path, WithRecovery(true), WithWrite(false), nil, WithDeploymentHistory(false), "")) assert.Equal(t, 1, recovered.Data.Serial) assert.Equal(t, "123", recovered.GetResourceID("jobs.my_job")) assert.NoFileExists(t, walPath) mustFinalize(t, &recovered) } -// TestEmptyFeatureStateAcceptedWithoutFlippingVersion pins the special case that a -// featureStateVersion state with no features is accepted as-is — the on-disk version -// is left at featureStateVersion, not flipped down to currentStateVersion — and that -// a featureStateVersion state recording any feature is refused. This is scaffolding -// for the deferred version bump, special-cased to featureStateVersion only (see the -// featureStateVersion doc comment). -// -// When the baseline is actually bumped to featureStateVersion, this special case must -// go away. This test is the forcing function: it fails once featureStateVersion is -// removed, making the author decide what the post-bump behavior should be. -func TestEmptyFeatureStateAcceptedWithoutFlippingVersion(t *testing.T) { - // The special case applies to featureStateVersion (3) only. - require.Equal(t, 2, currentStateVersion, "when currentStateVersion is bumped, remove featureStateVersion and this special case") - require.Equal(t, 3, featureStateVersion) - - empty := &Database{Header: Header{StateVersion: featureStateVersion}} - require.NoError(t, migrateState(empty)) - assert.Equal(t, featureStateVersion, empty.StateVersion, "v3 + no features keeps its on-disk version, not flipped to v2") - - // v3 that records a feature is refused: this CLI does not understand features. - withFeature := &Database{Header: Header{ - StateVersion: featureStateVersion, - Features: map[string]struct{}{"future_feature": {}}, - }} - err := migrateState(withFeature) - require.Error(t, err) - assert.Contains(t, err.Error(), "requires features this CLI does not support") - assert.Contains(t, err.Error(), "future_feature") - assert.Contains(t, err.Error(), "upgrade to the latest CLI version") - assert.Contains(t, err.Error(), featuresDocURL) -} - func TestDeleteState(t *testing.T) { path := filepath.Join(t.TempDir(), "state.json") var db DeploymentState - require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true))) - require.NoError(t, db.SaveState("jobs.my_job", "123", map[string]string{}, nil)) + require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil, WithDeploymentHistory(false), "")) + require.NoError(t, db.SaveState(t.Context(), "jobs.my_job", "123", map[string]string{}, nil)) mustFinalize(t, &db) var db2 DeploymentState - require.NoError(t, db2.Open(t.Context(), path, WithRecovery(true), WithWrite(true))) - require.NoError(t, db2.DeleteState("jobs.my_job")) + require.NoError(t, db2.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil, WithDeploymentHistory(false), "")) + require.NoError(t, db2.DeleteState(t.Context(), "jobs.my_job", false)) mustFinalize(t, &db2) var db3 DeploymentState - require.NoError(t, db3.Open(t.Context(), path, WithRecovery(false), WithWrite(false))) + require.NoError(t, db3.Open(t.Context(), path, WithRecovery(false), WithWrite(false), nil, WithDeploymentHistory(false), "")) assert.Equal(t, 2, db3.Data.Serial) assert.Empty(t, db3.GetResourceID("jobs.my_job")) mustFinalize(t, &db3) @@ -240,7 +208,7 @@ func TestGetOrInitLineageReadableBeforeWriteAndPersisted(t *testing.T) { // Fresh state opened read-only, as the deploy does before planning: no // lineage yet. var db DeploymentState - require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(false))) + require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(false), nil, WithDeploymentHistory(false), "")) require.Empty(t, db.Data.Lineage) // GetOrInitLineage initializes the lineage and makes it readable before any @@ -252,12 +220,12 @@ func TestGetOrInitLineageReadableBeforeWriteAndPersisted(t *testing.T) { // Upgrading to write reuses the same lineage (it goes into the WAL header), // and a write makes it durable. require.NoError(t, db.UpgradeToWrite()) - require.NoError(t, db.SaveState("jobs.my_job", "123", map[string]string{}, nil)) + require.NoError(t, db.SaveState(t.Context(), "jobs.my_job", "123", map[string]string{}, nil)) mustFinalize(t, &db) // Re-open: the persisted lineage matches the one read before the write. var reopened DeploymentState - require.NoError(t, reopened.Open(t.Context(), path, WithRecovery(false), WithWrite(false))) + require.NoError(t, reopened.Open(t.Context(), path, WithRecovery(false), WithWrite(false), nil, WithDeploymentHistory(false), "")) assert.Equal(t, lineage, reopened.Data.Lineage) mustFinalize(t, &reopened) } @@ -271,13 +239,13 @@ func TestOpenFailureLeavesStateClosed(t *testing.T) { require.NoError(t, os.WriteFile(path, []byte("{not json"), 0o600)) var db DeploymentState - require.Error(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true))) + require.Error(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil, WithDeploymentHistory(false), "")) assert.Empty(t, db.Path) // Once the state file is readable, the same receiver opens without panicking. seed := `{"state_version":2,"cli_version":"0.1.2","lineage":"test-lineage","serial":1,"state":{}}` require.NoError(t, os.WriteFile(path, []byte(seed), 0o600)) - require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true))) + require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil, WithDeploymentHistory(false), "")) assert.Equal(t, "test-lineage", db.Data.Lineage) mustFinalize(t, &db) } diff --git a/bundle/direct/pkg.go b/bundle/direct/pkg.go index ea7ae1e51b6..764397d5fc3 100644 --- a/bundle/direct/pkg.go +++ b/bundle/direct/pkg.go @@ -11,6 +11,7 @@ import ( "github.com/databricks/cli/bundle/direct/dresources" "github.com/databricks/cli/bundle/direct/dstate" "github.com/databricks/cli/bundle/statemgmt/resourcestate" + "github.com/databricks/cli/libs/dms" "github.com/databricks/cli/libs/structs/structvar" ) @@ -50,6 +51,13 @@ type DeploymentBundle struct { Plan *deployplan.Plan RemoteStateCache sync.Map StateCache structvar.Cache + + // DMS carries deployment-history recording across the phases; nil unless the bundle records + // history. DmsApiClient is set where the state is opened. The deployment id and version live + // in bundle.deployment.history, not here. DmsAsyncOperationClient is opened once the deploy + // phase creates the version, and Apply drains it before returning. + DmsApiClient *dms.Client + DmsAsyncOperationClient *dms.OperationBuffer } // SetRemoteState updates the remote state with type validation and marks as fresh. diff --git a/bundle/env/dms.go b/bundle/env/dms.go new file mode 100644 index 00000000000..d7258a76867 --- /dev/null +++ b/bundle/env/dms.go @@ -0,0 +1,22 @@ +package env + +import "context" + +// RecordDeploymentHistoryVariable enables recording without setting the config field. +// Exists for CLI acceptance tests so the whole bundle suite runs with recording on. +// Deliberately undocumented. +const RecordDeploymentHistoryVariable = "DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY" + +// RecordDeploymentHistoryEnv reports whether the environment turns on deployment history +// recording. Only "true" turns it on: anything else - including a typo like "TRUE" or "yes" - +// leaves the feature off rather than silently enabling it. +func RecordDeploymentHistoryEnv(ctx context.Context) bool { + value, _ := get(ctx, []string{RecordDeploymentHistoryVariable}) + return value == "true" +} + +// RecordsDeploymentHistory reports whether recording is on, from config or env var. +// Single predicate for all recording code paths; keeps them in sync. +func RecordsDeploymentHistory(ctx context.Context, configured bool) bool { + return configured || RecordDeploymentHistoryEnv(ctx) +} diff --git a/bundle/env/dms_test.go b/bundle/env/dms_test.go new file mode 100644 index 00000000000..15bfe0e63f0 --- /dev/null +++ b/bundle/env/dms_test.go @@ -0,0 +1,41 @@ +package env + +import ( + "testing" + + "github.com/databricks/cli/libs/env" + "github.com/stretchr/testify/assert" +) + +func TestRecordDeploymentHistoryEnv(t *testing.T) { + for _, tc := range []struct { + value string + want bool + }{ + {"true", true}, + {"", false}, + {"0", false}, + {"false", false}, + // Only "true" counts, so a near miss leaves recording off. + {"1", false}, + {"TRUE", false}, + {"yes", false}, + } { + ctx := env.Set(t.Context(), RecordDeploymentHistoryVariable, tc.value) + assert.Equal(t, tc.want, RecordDeploymentHistoryEnv(ctx), "value %q", tc.value) + } +} + +func TestRecordDeploymentHistoryEnvUnset(t *testing.T) { + assert.False(t, RecordDeploymentHistoryEnv(t.Context())) +} + +func TestRecordsDeploymentHistory(t *testing.T) { + // The bundle setting alone is enough, and so is the environment; the env var + // exists so the acceptance suite can record without touching every databricks.yml. + assert.True(t, RecordsDeploymentHistory(t.Context(), true)) + assert.False(t, RecordsDeploymentHistory(t.Context(), false)) + + ctx := env.Set(t.Context(), RecordDeploymentHistoryVariable, "true") + assert.True(t, RecordsDeploymentHistory(ctx, false)) +} diff --git a/bundle/migrate/build_state.go b/bundle/migrate/build_state.go index e8b382b370d..7afa577ba8c 100644 --- a/bundle/migrate/build_state.go +++ b/bundle/migrate/build_state.go @@ -205,7 +205,7 @@ func BuildStateFromTF( } } - if err := stateDB.SaveState(node, id, sv.Value, dependsOn); err != nil { + if err := stateDB.SaveState(ctx, node, id, sv.Value, dependsOn); err != nil { return warningsSeen, fmt.Errorf("%s: SaveState: %w", node, err) } } diff --git a/bundle/phases/bind.go b/bundle/phases/bind.go index 81f110f5095..eb141ea59b7 100644 --- a/bundle/phases/bind.go +++ b/bundle/phases/bind.go @@ -33,6 +33,11 @@ func Bind(ctx context.Context, b *bundle.Bundle, opts *terraform.BindOptions, en }() if engine.IsDirect() { + if b.ConfiguresDeploymentHistory(ctx) { + logdiag.LogError(ctx, errors.New("bind is not supported for a bundle that records deployment history")) + return + } + // Direct engine: import into temp state, run plan, check for changes // This follows the same pattern as terraform import groupName, ok := terraform.TerraformToGroupName[opts.ResourceType] @@ -94,7 +99,8 @@ func Bind(ctx context.Context, b *bundle.Bundle, opts *terraform.BindOptions, en } } else { // Terraform engine: use terraform import - bundle.ApplySeqContext(ctx, b, + bundle.ApplySeqContext( + ctx, b, terraform.Interpolate(), terraform.Write(), terraform.Import(opts), @@ -129,6 +135,11 @@ func Unbind(ctx context.Context, b *bundle.Bundle, bundleType, tfResourceType, r }() if engine.IsDirect() { + if b.ConfiguresDeploymentHistory(ctx) { + logdiag.LogError(ctx, errors.New("unbind is not supported for a bundle that records deployment history")) + return + } + groupName, ok := terraform.TerraformToGroupName[tfResourceType] if !ok { groupName = tfResourceType @@ -141,7 +152,8 @@ func Unbind(ctx context.Context, b *bundle.Bundle, bundleType, tfResourceType, r return } } else { - bundle.ApplySeqContext(ctx, b, + bundle.ApplySeqContext( + ctx, b, terraform.Interpolate(), terraform.Write(), terraform.Unbind(bundleType, tfResourceType, resourceKey), diff --git a/bundle/phases/deploy.go b/bundle/phases/deploy.go index f16088e4518..5fe25dd2eee 100644 --- a/bundle/phases/deploy.go +++ b/bundle/phases/deploy.go @@ -26,9 +26,11 @@ import ( "github.com/databricks/cli/bundle/statemgmt" "github.com/databricks/cli/libs/agent" "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/dms" "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/logdiag" "github.com/databricks/cli/libs/sync" + "github.com/databricks/databricks-sdk-go/service/bundledeployments" ) var deployApprovalGroups = []approvalGroup{ @@ -110,7 +112,8 @@ func deployCore(ctx context.Context, b *bundle.Bundle, plan *deployplan.Plan, st return } - bundle.ApplySeqContext(ctx, b, + bundle.ApplySeqContext( + ctx, b, statemgmt.Load(state), metadata.Compute(), metadata.Upload(), @@ -168,7 +171,8 @@ func logDeploySummary(ctx context.Context, b *bundle.Bundle, plan *deployplan.Pl // It also cleans up the artifacts directory and transforms wheel tasks. // It is called by only "bundle deploy". func uploadLibraries(ctx context.Context, b *bundle.Bundle, libs map[string][]libraries.LocationToUpdate) { - bundle.ApplySeqContext(ctx, b, + bundle.ApplySeqContext( + ctx, b, artifacts.CleanUp(), libraries.Upload(libs), ) @@ -179,12 +183,13 @@ func uploadLibraries(ctx context.Context, b *bundle.Bundle, libs map[string][]li // stateEngine is the engine the resolved state file uses; requestedEngine is // what bundle.engine / DATABRICKS_BUNDLE_ENGINE asked for and may differ (used // only by the post-deploy migration check). -func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHandler, stateEngine engine.EngineType, requestedEngine engine.EngineSetting, libs map[string][]libraries.LocationToUpdate, plan *deployplan.Plan) { +func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHandler, stateEngine engine.EngineType, requestedEngine engine.EngineSetting, libs map[string][]libraries.LocationToUpdate, plan *deployplan.Plan, dmsDeployment *bundledeployments.Deployment) { log.Info(ctx, "Phase: deploy") // Core mutators that CRUD resources and modify deployment state. These // mutators need informed consent if they are potentially destructive. - bundle.ApplySeqContext(ctx, b, + bundle.ApplySeqContext( + ctx, b, scripts.Execute(config.ScriptPreDeploy), lock.Acquire(lock.GoalDeploy), ) @@ -195,7 +200,13 @@ func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHand } // lock is acquired here + // + // The version is created only after approval; CompleteVersion is deferred before + // lock.Release and no-ops until then. defer func() { + if _, err := drainOperationsAndCompleteVersion(ctx, b, !logdiag.HasError(ctx)); err != nil { + logdiag.LogError(ctx, err) + } bundle.ApplyContext(ctx, b, lock.Release(lock.GoalDeploy)) }() @@ -209,7 +220,8 @@ func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHand // Upload all source files and built artifacts as a single immutable snapshot. // snapshot.Upload() sets workspace.snapshot_path; the variable-resolution // pass expands ${workspace.snapshot_path} placeholders written by translate_paths. - bundle.ApplySeqContext(ctx, b, + bundle.ApplySeqContext( + ctx, b, snapshot.Upload(), mutator.ResolveVariableReferencesOnlyResources("workspace"), ) @@ -246,7 +258,8 @@ func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHand } }() - bundle.ApplySeqContext(ctx, b, + bundle.ApplySeqContext( + ctx, b, deploy.StateUpdate(), deploy.StatePush(), permissions.ApplyWorkspaceRootPermissions(), @@ -259,6 +272,7 @@ func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHand } planFromFile := plan != nil + if plan == nil { // State is already open for read by process.go (for direct engine) plan = RunPlan(ctx, b, stateEngine) @@ -280,7 +294,8 @@ func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHand } if planFromFile { - // Initialize DeploymentBundle for applying the loaded plan + // Parse the loaded plan into the state cache the apply reads. A first deploy's deployment id + // is stamped later, after approval creates it (see below). err := b.DeploymentBundle.InitForApply(ctx, b.WorkspaceClient(ctx), plan) if err != nil { logdiag.LogError(ctx, err) @@ -295,17 +310,53 @@ func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHand return } - haveApproval, err := approvalForDeploy(ctx, b, plan) + haveApproval, approvalErr := approvalForDeploy(ctx, b, plan) + if !haveApproval { + // No version was created, so the deferred CompleteVersion is a no-op and the version + // number is left for the next deploy. Both the user declining and a console that + // cannot prompt land here. + if approvalErr != nil { + logdiag.LogError(ctx, approvalErr) + return + } + cmdio.LogString(ctx, "Deployment cancelled!") + return + } + + // Create the deployment now that the plan is approved, so a declined deploy leaves none behind. + // A first deploy's id did not exist at plan time - the version and any existing id were stamped + // then - so stamp the one just created into the plan the apply reads. + if b.DeploymentBundle.DmsApiClient != nil { + existingID, _ := deploymentAndNextVersion(b) + createOrUpdateDeployment(ctx, b, dmsDeployment) + if logdiag.HasError(ctx) { + return + } + if existingID == "" { + deploymentID, _ := deploymentAndNextVersion(b) + if err := b.DeploymentBundle.StampDeploymentID(deploymentID); err != nil { + logdiag.LogError(ctx, err) + return + } + } + } + + // Create the version the plan was stamped with, staging an operation for every resource + // it touches. Doing it here rather than before the prompt means a declined deploy never + // claims a version number. + staged, err := stagedOperations(plan) if err != nil { logdiag.LogError(ctx, err) return } - if haveApproval { - deployCore(ctx, b, plan, stateEngine) - } else { - cmdio.LogString(ctx, "Deployment cancelled!") + if err := startVersion(ctx, b, dms.VersionTypeDeploy, staged); err != nil { + logdiag.LogError(ctx, err) return } + deploymentID, versionID := deploymentAndNextVersion(b) + logDeploymentVersion(ctx, b, deploymentID, versionID) + + deployCore(ctx, b, plan, stateEngine) if logdiag.HasError(ctx) { return @@ -358,7 +409,8 @@ func RunPlan(ctx context.Context, b *bundle.Bundle, engine engine.EngineType) *d // b.Select is rejected for the terraform engine in ProcessBundleRet, so it is // never set here. - bundle.ApplySeqContext(ctx, b, + bundle.ApplySeqContext( + ctx, b, terraform.Interpolate(), terraform.Write(), terraform.Plan(terraform.PlanGoal("deploy")), diff --git a/bundle/phases/destroy.go b/bundle/phases/destroy.go index 201c3928e28..720f4ef7fa9 100644 --- a/bundle/phases/destroy.go +++ b/bundle/phases/destroy.go @@ -16,6 +16,7 @@ import ( "github.com/databricks/cli/bundle/deployplan" "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dms" "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/logdiag" "github.com/databricks/databricks-sdk-go/apierr" @@ -158,6 +159,21 @@ func destroyCore(ctx context.Context, b *bundle.Bundle, plan *deployplan.Plan, e return } + // Complete version before deleting remote files; the deployment node is under statePath. + completed, err := drainOperationsAndCompleteVersion(ctx, b, true) + if err != nil { + logdiag.LogError(ctx, err) + return + } + // A completed destroy's resources are gone, so its deployment record is deleted too. + if completed { + deploymentID, _ := deploymentAndNextVersion(b) + if err := b.DeploymentBundle.DmsApiClient.DeleteDeployment(ctx, deploymentID); err != nil { + logdiag.LogError(ctx, fmt.Errorf("failed to delete deployment: %w", err)) + return + } + } + bundle.ApplyContext(ctx, b, files.Delete()) if !logdiag.HasError(ctx) && b.Quiet < bundle.QuietAll { @@ -196,12 +212,26 @@ func Destroy(ctx context.Context, b *bundle.Bundle, engine engine.EngineType) { return } + // DMS recording of this destroy: the version is created after approval, so a cancelled + // destroy records nothing. Deferred before lock.Release to hold the lock; a no-op once + // destroyCore has completed the version. defer func() { + completed, err := drainOperationsAndCompleteVersion(ctx, b, !logdiag.HasError(ctx)) + if err != nil { + logdiag.LogError(ctx, err) + } else if completed { + // A completed destroy's resources are gone, so its deployment record is deleted too. + deploymentID, _ := deploymentAndNextVersion(b) + if err := b.DeploymentBundle.DmsApiClient.DeleteDeployment(ctx, deploymentID); err != nil { + logdiag.LogError(ctx, fmt.Errorf("failed to delete deployment: %w", err)) + } + } bundle.ApplyContext(ctx, b, lock.Release(lock.GoalDestroy)) }() if !engine.IsDirect() { - bundle.ApplySeqContext(ctx, b, + bundle.ApplySeqContext( + ctx, b, // We need to resolve artifact variable (how we do it in build phase) // because some of the to-be-destroyed resource might use this variable. // Not resolving might lead to terraform "Reference to undeclared resource" error @@ -253,6 +283,22 @@ func Destroy(ctx context.Context, b *bundle.Bundle, engine engine.EngineType) { return } } + // Record a destroy version now that it is approved and the state WAL is open, but only when + // the state is recorded with DMS (its marker is set). This is the single gate on the + // destroy's recording: it opens the operation buffer that destroyCore drains and completes, + // so the drain and the deployment delete follow only when a version was started here. + // Destroy never creates or updates the deployment - it is about to be deleted. + if engine.IsDirect() && b.DeploymentBundle.StateDB.RequiresDeploymentHistory() { + staged, err := stagedOperations(plan) + if err != nil { + logdiag.LogError(ctx, err) + return + } + if err := startVersion(ctx, b, dms.VersionTypeDestroy, staged); err != nil { + logdiag.LogError(ctx, err) + return + } + } destroyCore(ctx, b, plan, engine) } else { cmdio.LogString(ctx, "Destroy cancelled!") diff --git a/bundle/phases/dms.go b/bundle/phases/dms.go new file mode 100644 index 00000000000..40257fe5f5c --- /dev/null +++ b/bundle/phases/dms.go @@ -0,0 +1,249 @@ +package phases + +import ( + "context" + "fmt" + "net/url" + "strconv" + "strings" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/cli/internal/build" + "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/dms" + "github.com/databricks/cli/libs/log" + "github.com/databricks/cli/libs/logdiag" + "github.com/databricks/cli/libs/workspaceurls" + "github.com/databricks/databricks-sdk-go/service/bundledeployments" +) + +// stagedOperations lists the resources the plan will touch, for CreateVersion to stage an +// operation each. Skipped and undefined actions are left out: nothing is applied for them, so +// their operations would stay pending and the service would hold no state for them. +func stagedOperations(plan *deployplan.Plan) ([]dms.StagedOperation, error) { + actions := plan.GetActions() + staged := make([]dms.StagedOperation, 0, len(actions)) + for _, action := range actions { + if action.ActionType == deployplan.Skip || action.ActionType == deployplan.Undefined { + continue + } + actionType, err := actionToSDK(action.ActionType) + if err != nil { + return nil, fmt.Errorf("%s: %w", action.ResourceKey, err) + } + staged = append(staged, dms.StagedOperation{ + ResourceKey: action.ResourceKey, + ActionType: actionType, + }) + } + return staged, nil +} + +// actionToSDK maps a deployplan action to the DMS action type a staged operation records. +// Only actions that mutate a resource are recordable; Skip and Undefined are rejected +// rather than silently coerced. +func actionToSDK(a deployplan.ActionType) (bundledeployments.OperationActionType, error) { + switch a { + case deployplan.Create: + return bundledeployments.OperationActionTypeOperationActionTypeCreate, nil + case deployplan.Update: + return bundledeployments.OperationActionTypeOperationActionTypeUpdate, nil + case deployplan.UpdateWithID: + return bundledeployments.OperationActionTypeOperationActionTypeUpdateWithId, nil + case deployplan.Recreate: + return bundledeployments.OperationActionTypeOperationActionTypeRecreate, nil + case deployplan.Resize: + return bundledeployments.OperationActionTypeOperationActionTypeResize, nil + case deployplan.Delete: + return bundledeployments.OperationActionTypeOperationActionTypeDelete, nil + default: + return "", fmt.Errorf("cannot record operation: unsupported action %q", a) + } +} + +// deploymentAndNextVersion reads, from the history the recording set, the deployment id and the version +// this run will create (one past the deployment's most recent). Both zero when nothing is recorded +// - the bundle records no history, so version 0 marks "no version" for logDeploymentVersion. +func deploymentAndNextVersion(b *bundle.Bundle) (string, int64) { + h := b.Config.Bundle.Deployment.History + if h == nil { + return "", 0 + } + // LatestVersionID comes from the service as an integer, so this does not fail in practice. + version, err := dms.NextVersion(h.LatestVersionID) + if err != nil { + version = 1 + } + return h.DeploymentID, version +} + +// createOrUpdateDeployment creates the deployment on a first deploy, or updates the metadata this +// run changed. current is the record the service holds (nil before the first recorded deploy), +// diffed to mask the update down to what changed. Runs after approval, so a declined deploy leaves +// no deployment behind; a first deploy's new id is then stamped into the plan (StampDeploymentID). +func createOrUpdateDeployment(ctx context.Context, b *bundle.Bundle, current *bundledeployments.Deployment) { + db := &b.DeploymentBundle + metadata := deploymentMetadata(b) + deploymentID, _ := deploymentAndNextVersion(b) + switch mask := metadata.StaleFields(current); { + case deploymentID == "": + id, err := db.DmsApiClient.CreateDeployment(ctx, b.Config.Workspace.StatePath, metadata) + if err != nil { + logdiag.LogError(ctx, fmt.Errorf("failed to create deployment: %w", err)) + return + } + deploymentID = id + case mask != "": + if err := db.DmsApiClient.UpdateDeployment(ctx, deploymentID, metadata, mask); err != nil { + logdiag.LogError(ctx, fmt.Errorf("failed to update deployment: %w", err)) + return + } + } + + // A first deploy had no deployment to read at startup, so its id enters the history here. + bundle.ApplyFuncContext(ctx, b, func(_ context.Context, b *bundle.Bundle) { + if b.Config.Bundle.Deployment.History == nil { + b.Config.Bundle.Deployment.History = &config.DeploymentHistory{} + } + b.Config.Bundle.Deployment.History.DeploymentID = deploymentID + }) +} + +// startVersion claims the version the run settled on and opens the buffer that records +// each state write under it. Called after approval, so a declined deploy never claims a number. +// A no-op when the bundle does not record deployment history. +func startVersion(ctx context.Context, b *bundle.Bundle, versionType dms.VersionType, staged []dms.StagedOperation) error { + db := &b.DeploymentBundle + if db.DmsApiClient == nil { + return nil + } + deploymentID, versionID := deploymentAndNextVersion(b) + + // The version this run creates is one past the deployment's most recent, so the previous is + // one below it - empty for the first version. + previousVersionID := "" + if versionID > 1 { + previousVersionID = strconv.FormatInt(versionID-1, 10) + } + + // The server rejects this unless the version number exceeds last_version_id and + // previous_version_id matches it, which is what makes claiming the number up front + // safe: a deploy that took it in the meantime is reported, not overwritten. + var gitInfo *bundledeployments.GitInfo + if git := b.Config.Bundle.Git; git.Branch != "" || git.Commit != "" || git.OriginURL != "" { + gitInfo = &bundledeployments.GitInfo{ + Branch: git.Branch, + Commit: git.Commit, + OriginUrl: git.OriginURL, + } + } + version, err := db.DmsApiClient.CreateVersion(ctx, deploymentID, versionID, dms.CreateVersionRequest{ + CliVersion: build.GetInfo().Version, + VersionType: versionType, + PreviousVersionId: previousVersionID, + Operations: staged, + GitInfo: gitInfo, + }) + if err != nil { + return fmt.Errorf("failed to create deployment version: %w", err) + } + log.Infof(ctx, "Created deployment version: deployment=%s version=%s", deploymentID, version.VersionId) + + db.DmsAsyncOperationClient = db.StateDB.StartRecording(ctx, deploymentID, versionID) + return nil +} + +// drainOperationsAndCompleteVersion drains the buffered operations and completes the version. +// It returns whether the version completed successfully, which a destroy uses to decide whether +// to delete the deployment. Idempotent: the first call clears the buffer, so a deferred +// safety-net after an explicit completion is a no-op returning false. Also false when no version +// was created. +func drainOperationsAndCompleteVersion(ctx context.Context, b *bundle.Bundle, success bool) (bool, error) { + db := &b.DeploymentBundle + buf := db.DmsAsyncOperationClient + if buf == nil { + return false, nil + } + db.DmsAsyncOperationClient = nil + + // Drain first; a recording error fails the deploy even when the resources applied. Its error + // is left to whoever drained (apply), not returned twice. + if buf.Drain() != nil { + success = false + } + + reason := bundledeployments.VersionCompleteVersionCompleteSuccess + if !success { + reason = bundledeployments.VersionCompleteVersionCompleteFailure + } + deploymentID, versionID := deploymentAndNextVersion(b) + if err := db.DmsApiClient.CompleteVersion(ctx, deploymentID, versionID, reason); err != nil { + return false, err + } + log.Infof(ctx, "Completed deployment version: deployment=%s version=%d reason=%s", deploymentID, versionID, reason) + + return success, nil +} + +// logDeploymentVersion logs the deployment version URL. Workspace ID is omitted +// so the page stays clickable in a terminal and redirects correctly without it. +func logDeploymentVersion(ctx context.Context, b *bundle.Bundle, deploymentID string, version int64) { + if version == 0 { + return + } + + baseURL, err := url.Parse(b.WorkspaceClient(ctx).Config.CanonicalHostName()) + if err != nil { + // Only the link is lost, so report the version without it rather than failing + // a deploy over it. + log.Debugf(ctx, "Not linking to the recorded deployment: %s", err) + cmdio.LogString(ctx, fmt.Sprintf("Current Deployment Version: %s version %d", deploymentID, version)) + return + } + + cmdio.LogString(ctx, "Current Deployment Version: "+workspaceurls.DeploymentURL(*baseURL, deploymentID, version)) +} + +// deploymentMetadata describes the bundle this deploy came from and where it +// landed, mirroring what bundle/deploy/metadata computes for the metadata file. +func deploymentMetadata(b *bundle.Bundle) dms.Metadata { + p := dms.Metadata{ + DisplayName: b.Config.Bundle.Name, + TargetName: b.Config.Bundle.Target, + Mode: deploymentModeToSDK(b.Config.Bundle.Mode), + } + + ws := &bundledeployments.WorkspaceInfo{ + RootPath: b.Config.Workspace.RootPath, + FilePath: b.Config.Workspace.FilePath, + } + // In a source-linked deployment files are not copied, so resources read them + // from the sync root instead of file_path (see bundle/deploy/metadata.Compute). + if config.IsExplicitlyEnabled(b.Config.Presets.SourceLinkedDeployment) { + ws.FilePath = b.SyncRootPath + ws.SourceLinked = true + } + // Only a deploy from a Databricks Git folder has one; a local worktree does not. + // bundle_root_path is relative to it, so the service requires both or neither. + if b.WorktreeRoot != nil && strings.HasPrefix(b.WorktreeRoot.Native(), "/Workspace/") { + ws.GitFolderPath = b.WorktreeRoot.Native() + ws.BundleRootPath = b.Config.Bundle.Git.BundleRootPath + } + p.Workspace = ws + return p +} + +// deploymentModeToSDK maps the bundle target's mode to the DMS enum. An unset mode +// maps to empty, which the service reads as "not reported". +func deploymentModeToSDK(mode config.Mode) bundledeployments.DeploymentMode { + switch mode { + case config.Development: + return bundledeployments.DeploymentModeDeploymentModeDevelopment + case config.Production: + return bundledeployments.DeploymentModeDeploymentModeProduction + default: + return "" + } +} diff --git a/bundle/phases/dms_test.go b/bundle/phases/dms_test.go new file mode 100644 index 00000000000..f6854999db9 --- /dev/null +++ b/bundle/phases/dms_test.go @@ -0,0 +1,81 @@ +package phases + +import ( + "testing" + + "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/cli/libs/dms" + "github.com/databricks/databricks-sdk-go/service/bundledeployments" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestStagedOperationsCoversEveryTouchedResource(t *testing.T) { + // The version fixes its operation set, so anything the apply will write has to appear + // here. Keys go out in the service's form, without the CLI's "resources." prefix. + plan := &deployplan.Plan{Plan: map[string]*deployplan.PlanEntry{ + "resources.jobs.foo": {Action: deployplan.Create}, + "resources.pipelines.bar": {Action: deployplan.Recreate}, + "resources.schemas.baz": {Action: deployplan.Delete}, + "resources.clusters.small": {Action: deployplan.Resize}, + }} + + staged, err := stagedOperations(plan) + require.NoError(t, err) + + assert.ElementsMatch(t, []dms.StagedOperation{ + {ResourceKey: "resources.jobs.foo", ActionType: bundledeployments.OperationActionTypeOperationActionTypeCreate}, + {ResourceKey: "resources.pipelines.bar", ActionType: bundledeployments.OperationActionTypeOperationActionTypeRecreate}, + {ResourceKey: "resources.schemas.baz", ActionType: bundledeployments.OperationActionTypeOperationActionTypeDelete}, + {ResourceKey: "resources.clusters.small", ActionType: bundledeployments.OperationActionTypeOperationActionTypeResize}, + }, staged) +} + +func TestStagedOperationsLeavesOutUntouchedResources(t *testing.T) { + // A skipped resource is never applied, so staging it would leave an operation pending for + // the life of the version. Undefined is not a real action either. + plan := &deployplan.Plan{Plan: map[string]*deployplan.PlanEntry{ + "resources.jobs.touched": {Action: deployplan.Update}, + "resources.jobs.unchanged": {Action: deployplan.Skip}, + "resources.jobs.unknown": {Action: deployplan.Undefined}, + }} + + staged, err := stagedOperations(plan) + require.NoError(t, err) + + assert.Equal(t, []dms.StagedOperation{ + {ResourceKey: "resources.jobs.touched", ActionType: bundledeployments.OperationActionTypeOperationActionTypeUpdate}, + }, staged) +} + +func TestStagedOperationsEmptyPlan(t *testing.T) { + staged, err := stagedOperations(&deployplan.Plan{Plan: map[string]*deployplan.PlanEntry{}}) + require.NoError(t, err) + assert.Empty(t, staged) +} + +func TestActionToSDK(t *testing.T) { + cases := []struct { + action deployplan.ActionType + want bundledeployments.OperationActionType + }{ + {deployplan.Create, bundledeployments.OperationActionTypeOperationActionTypeCreate}, + {deployplan.Update, bundledeployments.OperationActionTypeOperationActionTypeUpdate}, + {deployplan.UpdateWithID, bundledeployments.OperationActionTypeOperationActionTypeUpdateWithId}, + {deployplan.Recreate, bundledeployments.OperationActionTypeOperationActionTypeRecreate}, + {deployplan.Resize, bundledeployments.OperationActionTypeOperationActionTypeResize}, + {deployplan.Delete, bundledeployments.OperationActionTypeOperationActionTypeDelete}, + } + for _, c := range cases { + got, err := actionToSDK(c.action) + require.NoError(t, err) + assert.Equal(t, c.want, got) + } + + // Nothing is applied for these, so stagedOperations leaves them out rather than + // mapping them; the guard is here in case a new action type arrives without a mapping. + _, err := actionToSDK(deployplan.Skip) + assert.Error(t, err) + _, err = actionToSDK(deployplan.Undefined) + assert.Error(t, err) +} diff --git a/bundle/statemgmt/direct_migration.go b/bundle/statemgmt/direct_migration.go index 7b88290930c..5d54e8702e5 100644 --- a/bundle/statemgmt/direct_migration.go +++ b/bundle/statemgmt/direct_migration.go @@ -172,7 +172,8 @@ func checkPlanOnTempState(ctx context.Context, b *bundle.Bundle, tempStatePath s }() var planBundle direct.DeploymentBundle - if err := planBundle.StateDB.Open(planCtx, tempStatePath, false, false); err != nil { + // This plan check does not record deployment history, so no DMS client or deployment id. + if err := planBundle.StateDB.Open(planCtx, tempStatePath, false, false, nil, dstate.WithDeploymentHistory(false), ""); err != nil { return fmt.Errorf("opening migrated state for plan check: %w", err) } diff --git a/cmd/bundle/generate/dashboard.go b/cmd/bundle/generate/dashboard.go index 086ec1d600a..14bc85777c1 100644 --- a/cmd/bundle/generate/dashboard.go +++ b/cmd/bundle/generate/dashboard.go @@ -404,7 +404,7 @@ func (d *dashboard) runForResource(ctx context.Context, b *bundle.Bundle) { var state statemgmt.ExportedResourcesMap if stateDesc.Engine.IsDirect() { _, localPath := b.StateFilenameDirect(ctx) - if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WithRecovery(true), dstate.WithWrite(false)); err != nil { + if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WithRecovery(true), dstate.WithWrite(false), nil, dstate.WithDeploymentHistory(false), ""); err != nil { logdiag.LogError(ctx, err) return } diff --git a/cmd/bundle/generate/genie_space.go b/cmd/bundle/generate/genie_space.go index 6d938c5e03d..ea37ff50990 100644 --- a/cmd/bundle/generate/genie_space.go +++ b/cmd/bundle/generate/genie_space.go @@ -322,7 +322,7 @@ func (g *genieSpace) runForResource(ctx context.Context, b *bundle.Bundle) { var state statemgmt.ExportedResourcesMap if stateDesc.Engine.IsDirect() { _, localPath := b.StateFilenameDirect(ctx) - if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WithRecovery(true), dstate.WithWrite(false)); err != nil { + if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WithRecovery(true), dstate.WithWrite(false), nil, dstate.WithDeploymentHistory(false), ""); err != nil { logdiag.LogError(ctx, err) return } diff --git a/cmd/bundle/utils/process.go b/cmd/bundle/utils/process.go index fa5471e8f99..1d671fbc306 100644 --- a/cmd/bundle/utils/process.go +++ b/cmd/bundle/utils/process.go @@ -4,13 +4,17 @@ import ( "context" "errors" "fmt" + "path" "path/filepath" + "strconv" "time" "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/engine" "github.com/databricks/cli/bundle/config/mutator" "github.com/databricks/cli/bundle/config/validate" + "github.com/databricks/cli/bundle/deploy/metadata" "github.com/databricks/cli/bundle/deploy/terraform" "github.com/databricks/cli/bundle/deployplan" "github.com/databricks/cli/bundle/direct" @@ -21,11 +25,15 @@ import ( "github.com/databricks/cli/internal/build" "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dms" "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/logdiag" "github.com/databricks/cli/libs/sync" "github.com/databricks/cli/libs/telemetry/protos" + "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/apierr" + "github.com/databricks/databricks-sdk-go/service/bundledeployments" "github.com/spf13/cobra" "golang.org/x/mod/semver" ) @@ -189,6 +197,11 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle return b, nil, err } + // The current deployment read from the service (nil, id "" if there is none yet). Used for the + // metadata diff and to reject a saved plan that predates the deployment's recorded version. + var dmsDeployment *bundledeployments.Deployment + var dmsDeploymentID string + shouldReadState := opts.ReadState || opts.AlwaysPull || opts.InitIDs || opts.ErrorOnEmptyState || opts.PreDeployChecks || opts.Deploy || opts.ReadPlanPath != "" if shouldReadState { @@ -232,11 +245,65 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle needDirectState := stateDesc.Engine.IsDirect() && (opts.InitIDs || opts.ErrorOnEmptyState || opts.Deploy || opts.ReadPlanPath != "" || opts.PreDeployChecks || opts.PostStateFunc != nil) if needDirectState { _, localPath := b.StateFilenameDirect(ctx) - if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WithRecovery(true), dstate.WithWrite(false)); err != nil { + + if b.ConfiguresDeploymentHistory(ctx) { + deploymentID, deployment, err := fetchDeploymentFromStatePath(ctx, b.WorkspaceClient(ctx), b.Config.Workspace.StatePath) + if err != nil { + logdiag.LogError(ctx, err) + return b, stateDesc, root.ErrAlreadyPrinted + } + + dmsDeploymentID = deploymentID + dmsDeployment = deployment + + // Stamp the deployment and the version this run records onto every job and pipeline so + // the plan carries them. version_id is always known (last recorded + 1); deployment_id + // does not exist until a first deploy creates it, so it is left off here and the deploy + // phase stamps the created id. + lastVersionID := "" + if deployment != nil { + lastVersionID = deployment.LastVersionId + } + nextVersion, verr := dms.NextVersion(lastVersionID) + if verr != nil { + logdiag.LogError(ctx, verr) + return b, stateDesc, root.ErrAlreadyPrinted + } + muts := []bundle.Mutator{metadata.AnnotateDeploymentVersion(nextVersion)} + if deploymentID != "" { + bundle.ApplyFuncContext(ctx, b, func(_ context.Context, b *bundle.Bundle) { + b.Config.Bundle.Deployment.History = &config.DeploymentHistory{ + DeploymentID: deploymentID, + LatestVersionID: deployment.LastVersionId, + } + }) + muts = append(muts, metadata.AnnotateDeployment(deploymentID)) + } + bundle.ApplySeqContext(ctx, b, muts...) + if logdiag.HasError(ctx) { + return b, stateDesc, root.ErrAlreadyPrinted + } + } + if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WithRecovery(true), dstate.WithWrite(false), b.WorkspaceClient(ctx), dstate.WithDeploymentHistory(b.ConfiguresDeploymentHistory(ctx)), dmsDeploymentID); err != nil { logdiag.LogError(ctx, err) return b, stateDesc, root.ErrAlreadyPrinted } + // Open built the DMS client from the workspace client when recording; hand it to the + // phases that create versions and deployments. + if b.ConfiguresDeploymentHistory(ctx) { + b.DeploymentBundle.DmsApiClient = b.DeploymentBundle.StateDB.DmsClient() + } + + // The service holds this deployment, so it has to keep being recorded: deploying + // without recording would leave it describing resources that have moved on. + if !b.ConfiguresDeploymentHistory(ctx) && b.DeploymentBundle.StateDB.RequiresDeploymentHistory() { + logdiag.LogError(ctx, errors.New(`unsetting experimental.record_deployment_history is not supported + +This deployment's resources are recorded with the deployment metadata service. Set experimental.record_deployment_history: true to deploy or destroy this bundle`)) + return b, stateDesc, root.ErrAlreadyPrinted + } + // Warn when the state was last written by a newer CLI than the one // running now. The state schema version is a hard gate (dstate.Open // rejects a too-new state_version), but a state can be written by a @@ -277,6 +344,7 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle return b, stateDesc, root.ErrAlreadyPrinted } } + } var plan *deployplan.Plan @@ -300,8 +368,27 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle log.Warnf(ctx, "Plan was created with CLI version %s but current version is %s", plan.CLIVersion, currentVersion) } - // Validate that the plan's lineage and serial match the current state - // This must happen before any file operations + // The plan records the DMS deployment and version it targeted. Reject it if the live + // deployment moved on - a newer version (someone deployed since, possibly from another + // machine) or a different id (deleted and recreated) - so a stale plan is never applied on + // top of newer state. This runs before the lineage/serial check and is the authoritative + // stale guard for recorded bundles, whose local state is only a tombstone (its serial does + // not catch a deploy from elsewhere). Both sides are empty when not recording, a no-op there. + remoteLastVersion := "" + if dmsDeployment != nil { + remoteLastVersion = dmsDeployment.LastVersionId + } + if plan.LastVersionId != remoteLastVersion { + logdiag.LogError(ctx, fmt.Errorf("this plan predates the deployment's current version %s; run 'bundle plan' again", remoteLastVersion)) + return b, stateDesc, root.ErrAlreadyPrinted + } + if plan.DeploymentId != dmsDeploymentID { + logdiag.LogError(ctx, errors.New("this plan targets a different deployment than the one now recorded for this bundle; run 'bundle plan' again")) + return b, stateDesc, root.ErrAlreadyPrinted + } + + // Validate that the plan's lineage and serial match the local state. This is the stale guard + // for non-recorded bundles (recorded ones are covered by the version check above). err = direct.ValidatePlanAgainstState(&b.DeploymentBundle.StateDB, plan) if err != nil { logdiag.LogError(ctx, err) @@ -373,7 +460,7 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle } t3 := time.Now() - phases.Deploy(ctx, b, outputHandler, stateDesc.Engine, requiredEngine, libs, plan) + phases.Deploy(ctx, b, outputHandler, stateDesc.Engine, requiredEngine, libs, plan, dmsDeployment) b.Metrics.ExecutionTimes = append(b.Metrics.ExecutionTimes, protos.IntMapEntry{ Key: "phases.Deploy", Value: time.Since(t3).Milliseconds(), @@ -427,6 +514,27 @@ func ResolveEngineSetting(ctx context.Context, b *bundle.Bundle) (engine.EngineS return engine.EngineSetting{Type: engine.Default, Source: engine.SourceDefault, IsDefault: true}, nil } +// Lookup and return the deployment object from ${workspace.state_path}/resources.deployment.json +func fetchDeploymentFromStatePath(ctx context.Context, w *databricks.WorkspaceClient, statePath string) (string, *bundledeployments.Deployment, error) { + nodePath := path.Join(statePath, dms.DeploymentNodeName) + + obj, err := w.Workspace.GetStatusByPath(ctx, nodePath) + if errors.Is(err, apierr.ErrNotFound) || errors.Is(err, apierr.ErrResourceDoesNotExist) { + return "", nil, nil + } + if err != nil { + return "", nil, fmt.Errorf("looking up deployment at %s: %w", nodePath, err) + } + deploymentID := strconv.FormatInt(obj.ObjectId, 10) + deployment, err := w.BundleDeployments.GetDeployment(ctx, bundledeployments.GetDeploymentRequest{ + Name: dms.DeploymentName(deploymentID), + }) + if err != nil { + return "", nil, err + } + return deploymentID, deployment, nil +} + // isNewerVersion reports whether the state's recorded CLI version is strictly // newer than the running build. Both are bare versions without a leading "v". // An empty stateVersion (state not written by any CLI yet) or an unparseable diff --git a/libs/dms/client.go b/libs/dms/client.go new file mode 100644 index 00000000000..8dbf9915399 --- /dev/null +++ b/libs/dms/client.go @@ -0,0 +1,250 @@ +package dms + +import ( + "context" + "fmt" + "net/http" + "strconv" + "strings" + + "github.com/databricks/cli/libs/auth" + "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/client" + "github.com/databricks/databricks-sdk-go/service/bundledeployments" +) + +// DeploymentNodeName is the workspace node DMS creates per deployment. Must +// match DeploymentWhsClient.DEPLOYMENT_NODE_NAME on the service side. +const DeploymentNodeName = "resources.deployment.json" + +// statePrefix is what a bundle state key carries and a DMS resource key does not: state calls a +// job "resources.jobs.foo", DMS calls it "jobs.foo". Every exported name here takes the state +// form; the prefix comes off where a request is built, and back on where a resource is read. +const statePrefix = "resources." + +// Client carries the calls the CLI makes to DMS, as methods below. Each one goes out through one +// of two halves: the generated client for the calls it can express, and hand-written requests +// for the two it cannot. +// +// TODO: Remove this and replace with the SDK. +type Client struct { + // Service is the generated client. + Service bundledeployments.BundleDeploymentsInterface + + // raw sends what the generated client cannot; see requester. + raw requester +} + +// NewClient returns a Client for the workspace w. +func NewClient(w *databricks.WorkspaceClient) (*Client, error) { + api, err := client.New(w.Config) + if err != nil { + return nil, err + } + return &Client{Service: w.BundleDeployments, raw: &rawClient{client: api}}, nil +} + +// DeploymentName and versionName are the two resource-name formats the service uses. Every +// call builds its name here, so a caller only ever passes ids. +func DeploymentName(deploymentID string) string { + return "deployments/" + deploymentID +} + +func versionName(deploymentID string, version int64) string { + return fmt.Sprintf("deployments/%s/versions/%d", deploymentID, version) +} + +// CreateDeployment registers a deployment under parentPath and returns the id the server +// assigned it, which is the id of the workspace node it creates there. +func (c *Client) CreateDeployment(ctx context.Context, parentPath string, metadata Metadata) (string, error) { + dep := metadata.deployment() + dep.InitialParentPath = parentPath + + created, err := c.Service.CreateDeployment(ctx, bundledeployments.CreateDeploymentRequest{Deployment: dep}) + if err != nil { + return "", err + } + return deploymentIDFromName(created.Name) +} + +// UpdateDeployment writes the fields mask names onto the deployment. The service ignores every +// other field, so the mask is what decides the write. +func (c *Client) UpdateDeployment(ctx context.Context, deploymentID string, metadata Metadata, mask string) error { + return c.raw.UpdateDeployment(ctx, deploymentID, metadata.deployment(), mask) +} + +// DeleteDeployment removes the deployment record, which a completed destroy does. +func (c *Client) DeleteDeployment(ctx context.Context, deploymentID string) error { + return c.Service.DeleteDeployment(ctx, bundledeployments.DeleteDeploymentRequest{ + Name: DeploymentName(deploymentID), + }) +} + +// CreateVersion claims the version and stages the operations body carries. +func (c *Client) CreateVersion(ctx context.Context, deploymentID string, version int64, body CreateVersionRequest) (*bundledeployments.Version, error) { + return c.raw.CreateVersion(ctx, deploymentID, strconv.FormatInt(version, 10), body) +} + +// CompleteVersion closes the version out, which is what stops the service expiring its lease. +func (c *Client) CompleteVersion(ctx context.Context, deploymentID string, version int64, reason bundledeployments.VersionComplete) error { + _, err := c.Service.CompleteVersion(ctx, bundledeployments.CompleteVersionRequest{ + Name: versionName(deploymentID, version), + CompletionReason: reason, + }) + return err +} + +// UpdateOperation fills in one operation the version staged, and returns the sequence id the +// next update for that resource must send. +func (c *Client) UpdateOperation(ctx context.Context, deploymentID string, version int64, stateKey, sequenceID string, update OperationUpdate) (string, error) { + return c.raw.UpdateOperation(ctx, deploymentID, version, stateKey, sequenceID, update) +} + +// deploymentIDFromName extracts the deployment ID from a DMS resource name of +// the form "deployments/{deployment_id}". +func deploymentIDFromName(name string) (string, error) { + id, ok := strings.CutPrefix(name, DeploymentName("")) + if !ok || id == "" { + return "", fmt.Errorf("unexpected deployment name %q from deployment metadata service", name) + } + return id, nil +} + +// requester sends the two requests the generated client cannot express, so a test can capture +// what the CLI puts on the wire. Both are TODO(DMS): drop them once the spec catches up. +type requester interface { + // CreateVersion is hand-written because the generated struct has no operations: the field + // is at DEVELOPMENT stage, which keeps it out of the SDK until it is promoted. + CreateVersion(ctx context.Context, deploymentID, versionID string, body CreateVersionRequest) (*bundledeployments.Version, error) + + // UpdateDeployment is hand-written because the generated client has no such call yet. + UpdateDeployment(ctx context.Context, deploymentID string, deployment bundledeployments.Deployment, mask string) error + + // UpdateOperation is hand-written because the SDK types sequence_id as an int64 while + // the service sends a JSON string, so it cannot read the response. sequenceID is the + // token the previous update for this resource returned, or 0 for the first, which is + // what staging leaves. + UpdateOperation(ctx context.Context, deploymentID string, version int64, stateKey, sequenceID string, update OperationUpdate) (next string, err error) +} + +// CreateVersionRequest is the CreateVersion request body. +type CreateVersionRequest struct { + CliVersion string `json:"cli_version"` + VersionType VersionType `json:"version_type"` + // PreviousVersionId is the deployment's most recent version, unset for a + // deployment's first version. + PreviousVersionId string `json:"previous_version_id,omitempty"` + // GitInfo records where this version's source came from. The rest of the provenance - + // display name, target, mode, workspace paths - belongs to the deployment. + GitInfo *bundledeployments.GitInfo `json:"git_info,omitempty"` + // Operations is every resource this version will touch; see StagedOperation. It sits in this + // body with the version's own fields because the request binds body: "version", and is input + // only - the response never carries it back. + Operations []StagedOperation `json:"operations,omitempty"` +} + +// StagedOperation is one resource the version will record an operation for. The service +// creates it in OPERATION_STATUS_PENDING at sequence id 0, and the CLI fills in the outcome +// with UpdateOperation as the resource is applied. +type StagedOperation struct { + // ResourceKey is the bundle state key; the request carries the form the service uses. + ResourceKey string `json:"resource_key"` + ActionType bundledeployments.OperationActionType `json:"action_type"` +} + +// operationResponse is the part of an operation response the CLI reads back. +type operationResponse struct { + // SequenceId is the concurrency token for the next update, typed as the service sends it. + SequenceId string `json:"sequence_id,omitempty"` +} + +// rawClient sends the requests the generated client cannot express. +type rawClient struct { + client *client.DatabricksClient +} + +func (r *rawClient) CreateVersion(ctx context.Context, deploymentID, versionID string, body CreateVersionRequest) (*bundledeployments.Version, error) { + staged := make([]StagedOperation, len(body.Operations)) + for i, op := range body.Operations { + op.ResourceKey = strings.TrimPrefix(op.ResourceKey, statePrefix) + staged[i] = op + } + body.Operations = staged + + var version bundledeployments.Version + path := "/api/2.0/bundle/" + DeploymentName(deploymentID) + "/versions" + err := r.client.Do(ctx, http.MethodPost, path, + auth.WorkspaceIDHeaders(r.client.Config), + map[string]any{"version_id": versionID}, + body, &version) + if err != nil { + return nil, err + } + return &version, nil +} + +// newDeploymentUpdate builds the request body for update, holding exactly the masked fields for +// the same reason newUpdateRequest does. A map, not the SDK struct, whose omitempty tags would +// drop a masked field that is empty - which is how deployment_mode is cleared when a target stops +// setting mode. +func newDeploymentUpdate(deployment bundledeployments.Deployment, mask string) map[string]any { + values := map[string]any{ + "display_name": deployment.DisplayName, + "target_name": deployment.TargetName, + "deployment_mode": deployment.DeploymentMode, + "workspace_info": deployment.WorkspaceInfo, + } + + body := map[string]any{} + for field := range strings.SplitSeq(mask, ",") { + body[field] = values[field] + } + return body +} + +func (r *rawClient) UpdateDeployment(ctx context.Context, deploymentID string, deployment bundledeployments.Deployment, mask string) error { + path := "/api/2.0/bundle/" + DeploymentName(deploymentID) + return r.client.Do(ctx, http.MethodPatch, path, + auth.WorkspaceIDHeaders(r.client.Config), + map[string]any{"update_mask": mask}, + newDeploymentUpdate(deployment, mask), nil) +} + +// newUpdateRequest builds the request body for update. A field is in the body when the mask +// names it and absent otherwise, which is what the service requires: it rejects an update +// whose mask names a field the body leaves out, and an empty value is how a field is cleared - +// no state means the resource is gone, no error_message means an earlier failure is resolved. +// A map, not a struct, so presence cannot drift from the mask through an omitempty tag. +func newUpdateRequest(update OperationUpdate, sequenceID string) map[string]any { + body := map[string]any{"sequence_id": sequenceID} + // A masked state with no value is how the service is told the resource is gone, so a nil + // state is left out rather than sent empty. + if update.Fields.Has(FieldState) && update.State != nil { + body["state"] = string(update.State) + } + if update.Fields.Has(FieldResourceID) { + body["resource_id"] = update.ResourceID + } + if update.Fields.Has(FieldErrorMessage) { + body["error_message"] = update.ErrorMessage + } + if update.Fields.Has(FieldStatus) { + body["status"] = update.Status + } + return body +} + +func (r *rawClient) UpdateOperation(ctx context.Context, deploymentID string, version int64, stateKey, sequenceID string, update OperationUpdate) (string, error) { + body := newUpdateRequest(update, sequenceID) + + var result operationResponse + path := "/api/2.0/bundle/" + versionName(deploymentID, version) + "/operations/" + strings.TrimPrefix(stateKey, statePrefix) + err := r.client.Do(ctx, http.MethodPatch, path, + auth.WorkspaceIDHeaders(r.client.Config), + map[string]any{"update_mask": update.Fields.Mask()}, + body, &result) + if err != nil { + return "", err + } + return result.SequenceId, nil +} diff --git a/libs/dms/client_test.go b/libs/dms/client_test.go new file mode 100644 index 00000000000..867655bca4e --- /dev/null +++ b/libs/dms/client_test.go @@ -0,0 +1,67 @@ +package dms + +import ( + "encoding/json" + "errors" + "testing" + + "github.com/databricks/databricks-sdk-go/service/bundledeployments" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestClientNamesEveryResourceTheSameWay(t *testing.T) { + // One format each, so a call only ever passes ids. + assert.Equal(t, "deployments/dep-1", DeploymentName("dep-1")) + assert.Equal(t, "deployments/dep-1/versions/2", versionName("dep-1", 2)) +} + +func TestDeploymentIDFromName(t *testing.T) { + id, err := deploymentIDFromName("deployments/abc-123") + require.NoError(t, err) + assert.Equal(t, "abc-123", id) + + _, err = deploymentIDFromName("abc-123") + assert.Error(t, err) + + _, err = deploymentIDFromName("deployments/") + assert.Error(t, err) +} + +func TestUpdateRequestSendsExactlyTheMaskedFields(t *testing.T) { + // The service rejects an update whose mask names a field the body leaves out, and treats a + // field it does carry as written - so the body has to hold every masked field and nothing + // else, empty values included. Both masks the CLI builds are asserted on the wire by + // acceptance/bundle/dms; this pins the rule they both rely on. + update := OperationUpdate{ + Fields: DescribesResource, + State: json.RawMessage(`{"state":{"name":"foo"}}`), + ResourceID: "job-1", + Status: bundledeployments.OperationStatusOperationStatusSucceeded, + } + + // A successful write reports no error, and the empty value is what clears an earlier one. + assert.Equal(t, map[string]any{ + "state": `{"state":{"name":"foo"}}`, + "resource_id": "job-1", + "error_message": "", + "status": bundledeployments.OperationStatusOperationStatusSucceeded, + "sequence_id": "3", + }, newUpdateRequest(update, "3")) + + // A failure keeps the recorded state, so state is absent rather than empty: naming it + // would clear what the resource last recorded. + assert.Equal(t, map[string]any{ + "error_message": "boom", + "status": bundledeployments.OperationStatusOperationStatusFailed, + "sequence_id": "3", + }, newUpdateRequest(NewFailureUpdate("job-1", nil, errors.New("boom")), "3")) + + // The deployment's own fields follow the same rule. Clearing deployment_mode - a target that + // stops setting mode - sends it empty, which the SDK struct's omitempty would have dropped. + deployment := Metadata{DisplayName: "b", TargetName: "t"}.deployment() + assert.Equal(t, map[string]any{ + "target_name": "t", + "deployment_mode": bundledeployments.DeploymentMode(""), + }, newDeploymentUpdate(deployment, "target_name,deployment_mode")) +} diff --git a/libs/dms/fields.go b/libs/dms/fields.go new file mode 100644 index 00000000000..05d39b387d7 --- /dev/null +++ b/libs/dms/fields.go @@ -0,0 +1,59 @@ +package dms + +import "strings" + +// Fields is the set of operation fields an update writes, sent as its update mask. The +// service rejects any other path, so this is the whole vocabulary. +type Fields uint8 + +const ( + FieldState Fields = 1 << iota + FieldErrorMessage + FieldResourceID + FieldStatus +) + +// DescribesResource is what a write that says how the resource looks claims: every field +// an update may change. +const DescribesResource = FieldState | FieldErrorMessage | FieldResourceID | FieldStatus + +// ClearsState is what a write that leaves no resource claims: a delete, and the delete half of +// a recreate. State is named so the service reads the absent value as a clear, which is what +// drops the resource from the deployment. It cannot name resource_id either: the service counts +// only a state it was given as recording one, so an id alongside a cleared state is refused. +const ClearsState = FieldState | FieldErrorMessage | FieldStatus + +// KeepsState is what a failure claims: mark it failed and leave state alone. State means +// the resource is as it was written; no state means a delete went through and nothing +// replaced it, so the resource really is gone and the deployment should say so. +// +// It cannot name resource_id: the service requires state in any mask that names the id, and +// naming state here would overwrite what the resource last recorded. +const KeepsState = FieldErrorMessage | FieldStatus + +// wireNames pairs each field with its name on the wire, in the order a mask lists them. +var wireNames = []struct { + field Fields + name string +}{ + {FieldState, "state"}, + {FieldErrorMessage, "error_message"}, + {FieldResourceID, "resource_id"}, + {FieldStatus, "status"}, +} + +// Has reports whether f contains every field in other. +func (f Fields) Has(other Fields) bool { + return f&other == other +} + +// Mask renders f as the update_mask the service expects, always in the same order. +func (f Fields) Mask() string { + names := make([]string, 0, len(wireNames)) + for _, w := range wireNames { + if f.Has(w.field) { + names = append(names, w.name) + } + } + return strings.Join(names, ",") +} diff --git a/libs/dms/fields_test.go b/libs/dms/fields_test.go new file mode 100644 index 00000000000..198a55efca8 --- /dev/null +++ b/libs/dms/fields_test.go @@ -0,0 +1,23 @@ +package dms + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestFieldsMask(t *testing.T) { + // The order is fixed, so the same set always sends the same mask. + assert.Equal(t, "state,error_message,resource_id,status", DescribesResource.Mask()) + assert.Equal(t, "error_message,status", KeepsState.Mask()) + assert.Equal(t, "state", FieldState.Mask()) + assert.Empty(t, Fields(0).Mask()) +} + +func TestFieldsHas(t *testing.T) { + assert.True(t, DescribesResource.Has(FieldState)) + assert.False(t, KeepsState.Has(FieldState)) + // Has asks for every field, not any of them. + assert.True(t, DescribesResource.Has(FieldState|FieldStatus)) + assert.False(t, KeepsState.Has(FieldState|FieldStatus)) +} diff --git a/libs/dms/metadata.go b/libs/dms/metadata.go new file mode 100644 index 00000000000..d63154a4790 --- /dev/null +++ b/libs/dms/metadata.go @@ -0,0 +1,95 @@ +package dms + +import ( + "fmt" + "reflect" + "strconv" + "strings" + + "github.com/databricks/databricks-sdk-go/service/bundledeployments" +) + +// VersionType identifies the kind of deployment a version records. +type VersionType = bundledeployments.VersionType + +const ( + VersionTypeDeploy VersionType = bundledeployments.VersionTypeVersionTypeDeploy + VersionTypeDestroy VersionType = bundledeployments.VersionTypeVersionTypeDestroy +) + +// Metadata is what a version records about the bundle, its source and where it +// landed. The service copies these onto the deployment, so they describe it as of +// its most recent version. +type Metadata struct { + // DisplayName is the bundle's name, which the deployment is listed under. + DisplayName string + // TargetName is the bundle target that was deployed. + TargetName string + // Mode is the bundle target's mode, empty when the target sets none. + Mode bundledeployments.DeploymentMode + Workspace *bundledeployments.WorkspaceInfo +} + +// deploymentFields are the deployment's own metadata, in the order a mask lists them. Git is not +// among them: the service derives the deployment's from the version that carried it. +var deploymentFields = []string{"display_name", "target_name", "deployment_mode", "workspace_info"} + +// sameWorkspaceInfo compares the paths alone. The SDK records which fields a response carried in +// ForceSendFields, so a record read back never deep-equals one built here, and comparing the +// structs whole would report every run as a change. +func sameWorkspaceInfo(want, current *bundledeployments.WorkspaceInfo) bool { + if want == nil || current == nil { + return want == nil && current == nil + } + + a, b := *want, *current + a.ForceSendFields, b.ForceSendFields = nil, nil + return reflect.DeepEqual(a, b) +} + +// deployment renders the metadata the deployment owns. +func (m Metadata) deployment() bundledeployments.Deployment { + return bundledeployments.Deployment{ + DisplayName: m.DisplayName, + TargetName: m.TargetName, + DeploymentMode: m.Mode, + WorkspaceInfo: m.Workspace, + } +} + +// StaleFields returns the mask that brings current up to m, empty when the deployment already +// says what this run would say. current is nil before the first recorded deploy. +func (m Metadata) StaleFields(current *bundledeployments.Deployment) string { + if current == nil { + return strings.Join(deploymentFields, ",") + } + + want := m.deployment() + var stale []string + if want.DisplayName != current.DisplayName { + stale = append(stale, "display_name") + } + if want.TargetName != current.TargetName { + stale = append(stale, "target_name") + } + if want.DeploymentMode != current.DeploymentMode { + stale = append(stale, "deployment_mode") + } + if !sameWorkspaceInfo(want.WorkspaceInfo, current.WorkspaceInfo) { + stale = append(stale, "workspace_info") + } + return strings.Join(stale, ",") +} + +// NextVersion is the version number this run will create, given the deployment's most recent +// one. lastVersionID is empty before the deployment has any version. +func NextVersion(lastVersionID string) (int64, error) { + if lastVersionID == "" { + return 1, nil + } + last, err := strconv.ParseInt(lastVersionID, 10, 64) + if err != nil { + return 0, fmt.Errorf("failed to parse last_version_id %q: %w", lastVersionID, err) + } + return last + 1, nil +} diff --git a/libs/dms/metadata_test.go b/libs/dms/metadata_test.go new file mode 100644 index 00000000000..e3446466407 --- /dev/null +++ b/libs/dms/metadata_test.go @@ -0,0 +1,25 @@ +package dms + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestNextVersion(t *testing.T) { + for _, tt := range []struct { + last string + want int64 + }{ + {"", 1}, + {"4", 5}, + } { + got, err := NextVersion(tt.last) + require.NoError(t, err) + assert.Equal(t, tt.want, got) + } + + _, err := NextVersion("not-a-number") + assert.ErrorContains(t, err, "last_version_id") +} diff --git a/libs/dms/operation.go b/libs/dms/operation.go new file mode 100644 index 00000000000..16e1c1a3588 --- /dev/null +++ b/libs/dms/operation.go @@ -0,0 +1,139 @@ +package dms + +import ( + "encoding/json" + "fmt" + "unicode/utf8" + + "github.com/databricks/cli/libs/diag" + "github.com/databricks/databricks-sdk-go/service/bundledeployments" +) + +// maxStateSize is the largest serialized state DMS accepts per operation. More than this +// and the resource cannot be recorded at all, so the deploy fails rather than leaving the +// service holding a resource with no state. +const maxStateSize = 64 * 1024 + +// maxErrorMessageSize is how much of a failure's message the service stores. +const maxErrorMessageSize = 16 * 1024 + +// StatusPending is what the service calls an operation it has recorded but not applied, which +// is where a recreate sits between deleting the old resource and creating the new one. Not in +// the SDK: the enum ships server-first while it is staged DEVELOPMENT, so the generated client +// carries only the terminal two. +const StatusPending bundledeployments.OperationStatus = "OPERATION_STATUS_PENDING" + +// OperationUpdate is one write to an operation the version staged: the fields it claims +// and their values. It is built where the outcome is known, so a malformed one fails the +// resource that produced it rather than the upload at the end of apply. +type OperationUpdate struct { + // Fields is the mask to send. It is taken literally: a field named here is written, + // one left out keeps the value it had. + Fields Fields + + // State is the serialized state after the operation, and nil for a delete. + State json.RawMessage + + ResourceID string + Status bundledeployments.OperationStatus + ErrorMessage string +} + +// NewStateUpdate describes how the resource looks now: state is the serialized envelope the +// state DB just persisted, and nil for a delete. inProgress marks a write that is half of a +// larger change - a recreate's delete - so an interrupted deploy does not report it finished. +func NewStateUpdate(resourceID string, state json.RawMessage, inProgress bool) (OperationUpdate, error) { + if len(state) > maxStateSize { + return OperationUpdate{}, fmt.Errorf("serialized state is %d bytes, which exceeds the %d byte limit for recording deployment history", len(state), maxStateSize) + } + + status := bundledeployments.OperationStatusOperationStatusSucceeded + if inProgress { + status = StatusPending + } + + // No state means nothing is left to describe, so the write clears it rather than recording + // it as empty: an empty value is a state the service refuses on a delete that succeeded, + // while an absent one is how the resource stops being listed at all. + if state == nil { + return OperationUpdate{ + Fields: ClearsState, + Status: status, + }, nil + } + + return OperationUpdate{ + Fields: DescribesResource, + State: state, + ResourceID: resourceID, + Status: status, + }, nil +} + +// failureMessage summarizes cause for the history, cut to what the service stores. +func failureMessage(cause error) string { + // Summarized, not cause.Error(): for an API failure that adds the status and error + // code, which is often the most actionable part of the history. + message := diag.FormatAPIErrorSummary(cause) + if len(message) > maxErrorMessageSize { + message = message[:maxErrorMessageSize] + // The cut can land inside a rune, and the service stores a string. Drop the partial + // one: at most UTFMax-1 bytes of it can be left, so a message that was already + // invalid loses those bytes rather than being stripped away entirely. + for range utf8.UTFMax - 1 { + if utf8.ValidString(message) { + break + } + message = message[:len(message)-1] + } + } + return message +} + +// failureUpdate marks an operation failed and carries the message, with no state of its own: it +// claims KeepsState so a Merge onto the resource's last write keeps that write's state and id. +// OperationBuffer.RecordFailure uses it when a write already recorded the resource's state. +func failureUpdate(cause error) OperationUpdate { + return OperationUpdate{ + Fields: KeepsState, + Status: bundledeployments.OperationStatusOperationStatusFailed, + ErrorMessage: failureMessage(cause), + } +} + +// NewFailureUpdate records that an operation did not apply, so the history says why a resource +// failed rather than leaving it pending. state is what the resource still has, which the service +// requires of a failure on a live one; nil when nothing is left to describe. +func NewFailureUpdate(resourceID string, state json.RawMessage, cause error) OperationUpdate { + u := failureUpdate(cause) + if state != nil { + // The id travels with the state, which the service requires of any mask naming one. + u.Fields = DescribesResource + u.State = state + u.ResourceID = resourceID + } + return u +} + +// Merge folds a later update into u, for a resource written twice before either upload ran. +// Each field comes from whichever update claimed it, newer winning when both did, and the +// mask is the union. What an update claims is decided where it is built, not here. +func (u OperationUpdate) Merge(newer OperationUpdate) OperationUpdate { + merged := u + merged.Fields = u.Fields | newer.Fields + + if newer.Fields.Has(FieldState) { + merged.State = newer.State + } + if newer.Fields.Has(FieldResourceID) { + merged.ResourceID = newer.ResourceID + } + if newer.Fields.Has(FieldErrorMessage) { + merged.ErrorMessage = newer.ErrorMessage + } + if newer.Fields.Has(FieldStatus) { + merged.Status = newer.Status + } + + return merged +} diff --git a/libs/dms/operation_buffer.go b/libs/dms/operation_buffer.go new file mode 100644 index 00000000000..ac5cab63dd8 --- /dev/null +++ b/libs/dms/operation_buffer.go @@ -0,0 +1,197 @@ +package dms + +import ( + "context" + "encoding/json" + "fmt" + "sync" +) + +// bufferedOperations caps how far ahead of the service a deploy may get; DMS is what the next +// plan reads. +const bufferedOperations = 10 + +// stagedSequenceID is what version creation leaves on a staged operation, so a resource's first +// update sends it as the precondition. +const stagedSequenceID = "0" + +// OperationBuffer records each state write with DMS for one deployment version, off the apply +// path: writes are queued and sent on one background goroutine. It exists only while a bundle +// records deployment history; callers hold a nil buffer otherwise and must not call it. +type OperationBuffer struct { + client *Client + deploymentID string + versionNum int64 + + // The buffer. queue holds bundle state keys, and pending the newest update per key, so a + // second write for a resource replaces the first. + queue chan string + done chan struct{} + stopQueue func() + + // sequenceIDs holds the token the last update for a resource returned. A resource absent + // from it has only what staging left, so its first update sends that. Unguarded: run is the + // only goroutine that writes, one update at a time. + sequenceIDs map[string]string + + // mu guards the fields below. + mu sync.Mutex + + // pending holds the newest update per resource key, absent once the writer takes it. + pending map[string]OperationUpdate + + // latestState is the newest state-bearing update per resource key, kept for the buffer's + // whole life - unlike pending, which the writer removes on take. RecordFailure merges its + // status and message onto this so a failure keeps the state the last write recorded (which + // the service may already hold), not the pre-deploy state, which would overwrite it. + latestState map[string]OperationUpdate + + err error +} + +// StartOperationBuffer opens the buffer for the version the caller just created. The version +// must already exist: operations record under it, and nothing here creates it. +func StartOperationBuffer(ctx context.Context, client *Client, deploymentID string, versionNum int64) *OperationBuffer { + b := &OperationBuffer{ + client: client, + deploymentID: deploymentID, + versionNum: versionNum, + queue: make(chan string, bufferedOperations), + done: make(chan struct{}), + pending: make(map[string]OperationUpdate), + latestState: make(map[string]OperationUpdate), + sequenceIDs: make(map[string]string), + } + b.stopQueue = sync.OnceFunc(func() { close(b.queue) }) + go b.run(ctx) + return b +} + +// RecordOperation records a state write. state is the serialized envelope, nil for a delete. +// An earlier failure does not stop it: keep recording, best effort. +func (b *OperationBuffer) RecordOperation(ctx context.Context, resourceKey string, inProgress bool, resourceID string, state json.RawMessage) { + update, err := NewStateUpdate(resourceID, state, inProgress) + if err != nil { + b.setErr(fmt.Errorf("recording operation for %s: %w", resourceKey, err)) + return + } + + // Remember the state this write recorded, kept past the writer taking it, so a later failure + // for this resource keeps this state rather than the pre-deploy one (see RecordFailure). + b.mu.Lock() + b.latestState[resourceKey] = update + b.mu.Unlock() + + b.record(resourceKey, update) +} + +// RecordFailure records that a resource did not apply, so the history says why rather than +// leaving the resource out. When a write already recorded this resource's state, the failure +// keeps that state - which the service may already hold - and changes only status and message; +// recording the pre-deploy state instead would overwrite the newer state, and after a rename +// pair the new id with the old state. state is the fallback, used only when no write recorded +// anything: the operation failed before it saved state. +func (b *OperationBuffer) RecordFailure(resourceKey, resourceID string, state json.RawMessage, cause error) { + b.mu.Lock() + latest, recorded := b.latestState[resourceKey] + b.mu.Unlock() + + if recorded { + b.record(resourceKey, latest.Merge(failureUpdate(cause))) + return + } + + b.record(resourceKey, NewFailureUpdate(resourceID, state, cause)) +} + +// record makes update the one waiting for resourceKey, waiting itself while the queue is +// full. Recording after Drain panics, so every caller must return before Drain. +func (b *OperationBuffer) record(resourceKey string, update OperationUpdate) { + b.mu.Lock() + waiting, queued := b.pending[resourceKey] + if queued { + update = waiting.Merge(update) + } + b.pending[resourceKey] = update + b.mu.Unlock() + + // when already queued: the writer reads the map when it gets to the key, + // so it picks up what was just stored. + // + // Only when the resource is not already queued, enqueue it. + if !queued { + b.queue <- resourceKey + } +} + +// take claims the update waiting for resourceKey. +func (b *OperationBuffer) take(resourceKey string) (OperationUpdate, bool) { + b.mu.Lock() + defer b.mu.Unlock() + + update, ok := b.pending[resourceKey] + delete(b.pending, resourceKey) + return update, ok +} + +func (b *OperationBuffer) run(ctx context.Context) { + defer close(b.done) + + for resourceKey := range b.queue { + update, ok := b.take(resourceKey) + if !ok { + // Unreachable: a key is queued only when nothing was waiting for it. Guard so + // a stray key could never write a zero-valued update. + continue + } + + // Keep going after a failure, so one bad write does not drop everything behind it. + if err := b.write(ctx, resourceKey, update); err != nil { + b.setErr(fmt.Errorf("recording operation for %s: %w", resourceKey, err)) + } + } +} + +// write sends one update, at the sequence id the resource is at. +func (b *OperationBuffer) write(ctx context.Context, key string, update OperationUpdate) error { + sequenceID, written := b.sequenceIDs[key] + if !written { + sequenceID = stagedSequenceID + } + + next, err := b.client.UpdateOperation(ctx, b.deploymentID, b.versionNum, key, sequenceID, update) + if err != nil { + return err + } + + // The next write for this resource echoes the sequence id this one earned. + b.sequenceIDs[key] = next + + return nil +} + +// Drain waits for everything buffered to reach the service and returns the first write error, +// which fails the deploy: DMS is the source of truth for what exists. Safe to call twice. +func (b *OperationBuffer) Drain() error { + b.stopQueue() + <-b.done + return b.Err() +} + +// setErr keeps the first error; one failure is enough to fail the deploy. +func (b *OperationBuffer) setErr(err error) { + b.mu.Lock() + defer b.mu.Unlock() + + if b.err == nil { + b.err = err + } +} + +// Err returns the first recording error, or nil. +func (b *OperationBuffer) Err() error { + b.mu.Lock() + defer b.mu.Unlock() + + return b.err +} diff --git a/libs/dms/operation_buffer_test.go b/libs/dms/operation_buffer_test.go new file mode 100644 index 00000000000..b4fba8ce124 --- /dev/null +++ b/libs/dms/operation_buffer_test.go @@ -0,0 +1,116 @@ +package dms + +import ( + "encoding/json" + "errors" + "strings" + "sync" + "testing" + + "github.com/databricks/databricks-sdk-go/service/bundledeployments" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// What the service ends up holding - the version a deploy claims, the operations it stages, the +// completion it reports - is asserted end to end by acceptance/bundle/dms. What is left here is +// what a deploy cannot reach: the queue's coalescing and the size limit. No test here goes +// through the API - the queue is driven directly, so nothing waits on a request. + +// queued builds a buffer whose queue nothing drains, so a test drives record and take itself. +func queued() *OperationBuffer { + return &OperationBuffer{ + queue: make(chan string, bufferedOperations), + pending: make(map[string]OperationUpdate), + latestState: make(map[string]OperationUpdate), + } +} + +func stateUpdate(t *testing.T, name string) OperationUpdate { + t.Helper() + update, err := NewStateUpdate("id-1", json.RawMessage(`{"state":{"name":"`+name+`"}}`), false) + require.NoError(t, err) + return update +} + +func TestBufferCoalescesWhileAKeyIsPending(t *testing.T) { + s := queued() + + // Two writes for one resource with nothing draining. They carry the resource's full + // state, so only the newest needs to go: one slot in the queue, not two. + s.record("resources.jobs.foo", stateUpdate(t, "v1")) + s.record("resources.jobs.foo", stateUpdate(t, "v2")) + + assert.Len(t, s.queue, 1) + update, ok := s.take("resources.jobs.foo") + require.True(t, ok) + assert.JSONEq(t, `{"state":{"name":"v2"}}`, string(update.State)) + + // Taken means a request has it, and an in-flight request cannot be recalled, so the next + // write gets its own slot rather than joining it. + s.record("resources.jobs.foo", stateUpdate(t, "v3")) + assert.Len(t, s.queue, 2) + update, ok = s.take("resources.jobs.foo") + require.True(t, ok) + assert.JSONEq(t, `{"state":{"name":"v3"}}`, string(update.State)) +} + +func TestBufferFailsOnOversizedState(t *testing.T) { + // The service will not take a state this large, so the resource cannot be recorded. + // Failing here says so, where reporting nothing would leave DMS without the resource + // and the next plan would create it again. + s := queued() + + s.RecordOperation(t.Context(), "resources.jobs.foo", false, "id-1", json.RawMessage(strings.Repeat("x", maxStateSize+1))) + + assert.ErrorContains(t, s.Err(), "exceeds the 65536 byte limit") + assert.Empty(t, s.queue) + assert.Empty(t, s.pending) +} + +func TestDrainIsIdempotent(t *testing.T) { + // Nothing recorded, so no transport is needed: this is the second drain, which must not + // panic on an already closed queue. + s := queued() + s.done = make(chan struct{}) + s.stopQueue = sync.OnceFunc(func() { close(s.queue) }) + go s.run(t.Context()) + + require.NoError(t, s.Drain()) + require.NoError(t, s.Drain()) +} + +func TestRecordFailureKeepsTheStateTheLastWriteRecorded(t *testing.T) { + s := queued() + + // A write records the resource's post-apply state - a rename, so a new id and new state - and + // the writer sends it (take), leaving nothing pending. + s.RecordOperation(t.Context(), "resources.jobs.foo", false, "id-new", json.RawMessage(`{"state":{"name":"v2"}}`)) + _, ok := s.take("resources.jobs.foo") + require.True(t, ok) + require.Empty(t, s.pending) + + // The post-update wait then fails. The failure carries the pre-deploy state (old id, old + // state), which it must not use: the write already recorded v2, which the service may hold. + s.RecordFailure("resources.jobs.foo", "id-old", json.RawMessage(`{"state":{"name":"v1"}}`), errors.New("wait timed out")) + + update, ok := s.take("resources.jobs.foo") + require.True(t, ok) + assert.JSONEq(t, `{"state":{"name":"v2"}}`, string(update.State), "keeps the state the write recorded") + assert.Equal(t, "id-new", update.ResourceID, "keeps the id the write recorded, not the pre-deploy id") + assert.Equal(t, bundledeployments.OperationStatusOperationStatusFailed, update.Status) + assert.Contains(t, update.ErrorMessage, "wait timed out") +} + +func TestRecordFailureUsesTheGivenStateWhenNoWriteRecorded(t *testing.T) { + // No write recorded anything - the operation failed before it saved state, e.g. a create that + // never landed - so the failure falls back to the state passed in, what the resource still has. + s := queued() + + s.RecordFailure("resources.jobs.foo", "id-1", json.RawMessage(`{"state":{"name":"v1"}}`), errors.New("create failed")) + + update, ok := s.take("resources.jobs.foo") + require.True(t, ok) + assert.JSONEq(t, `{"state":{"name":"v1"}}`, string(update.State)) + assert.Equal(t, bundledeployments.OperationStatusOperationStatusFailed, update.Status) +} diff --git a/libs/dms/operation_test.go b/libs/dms/operation_test.go new file mode 100644 index 00000000000..2bae9a81144 --- /dev/null +++ b/libs/dms/operation_test.go @@ -0,0 +1,83 @@ +package dms + +import ( + "encoding/json" + "errors" + "strings" + "testing" + "unicode/utf8" + + "github.com/databricks/databricks-sdk-go/service/bundledeployments" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// What these updates put on the wire - the mask, the status, the state a write carries and a +// failure leaves alone - is asserted by acceptance/bundle/dms. What is left here are the limits +// and the merge, which a deploy cannot reach. + +func TestNewFailureUpdateTruncatesLongError(t *testing.T) { + // Truncated rather than rejected: a message over the limit would make recording + // fail and hide the error it is reporting. + update := NewFailureUpdate("job-123", nil, errors.New(strings.Repeat("x", maxErrorMessageSize+100))) + + assert.Len(t, update.ErrorMessage, maxErrorMessageSize) +} + +func TestNewFailureUpdatePreservesUTF8OnTruncation(t *testing.T) { + // The cut lands one byte into the emoji, so a byte-wise truncation would leave a partial + // rune behind and the service stores state and messages as strings. + msg := strings.Repeat("a", maxErrorMessageSize-1) + "❌" + "x" + + update := NewFailureUpdate("job-123", nil, errors.New(msg)) + + assert.True(t, utf8.ValidString(update.ErrorMessage)) + // The whole emoji went, so the message is shorter than the limit rather than exactly it. + assert.Equal(t, strings.Repeat("a", maxErrorMessageSize-1), update.ErrorMessage) +} + +func TestMergeLetsAWriteSupersedeAFailure(t *testing.T) { + // A failure is not the last word. A retry that writes state wins whole - state, id and + // mask - and the mask names error_message so the recorded failure is cleared. The service + // rejects a succeeded operation that still carries an error. + failed := NewFailureUpdate("id-old", nil, errors.New("boom")) + retried, err := NewStateUpdate("id-new", json.RawMessage(`{"state":{"name":"after"}}`), false) + require.NoError(t, err) + + merged := failed.Merge(retried) + + assert.Equal(t, bundledeployments.OperationStatusOperationStatusSucceeded, merged.Status) + assert.Empty(t, merged.ErrorMessage) + assert.Equal(t, "id-new", merged.ResourceID) + assert.JSONEq(t, `{"state":{"name":"after"}}`, string(merged.State)) + assert.Equal(t, DescribesResource, merged.Fields) +} + +func TestMergeKeepsTheWritesStateAndMask(t *testing.T) { + // A failure claims only status and error_message, so the write's state, id and mask + // survive: the resource stays listed as it was written, now marked failed. + write, err := NewStateUpdate("id-new", json.RawMessage(`{"state":{"name":"before"}}`), false) + require.NoError(t, err) + failed := NewFailureUpdate("id-old", nil, errors.New("boom")) + + merged := write.Merge(failed) + + assert.Equal(t, bundledeployments.OperationStatusOperationStatusFailed, merged.Status) + assert.Equal(t, "boom", merged.ErrorMessage) + assert.Equal(t, "id-new", merged.ResourceID) + assert.JSONEq(t, `{"state":{"name":"before"}}`, string(merged.State)) + assert.Equal(t, DescribesResource, merged.Fields) +} + +func TestMergeLetsADeleteClearTheState(t *testing.T) { + // A delete legitimately carries no state, and merging must let it through: the resource + // is gone, and keeping the state it replaces would leave it listed. + write, err := NewStateUpdate("id-1", json.RawMessage(`{"state":{"name":"before"}}`), false) + require.NoError(t, err) + deleted, err := NewStateUpdate("id-1", nil, false) + require.NoError(t, err) + + merged := write.Merge(deleted) + + assert.Nil(t, merged.State) +} diff --git a/libs/dms/resources.go b/libs/dms/resources.go new file mode 100644 index 00000000000..565571415f6 --- /dev/null +++ b/libs/dms/resources.go @@ -0,0 +1,41 @@ +package dms + +import ( + "context" + "fmt" + + "github.com/databricks/databricks-sdk-go/service/bundledeployments" +) + +// Resource is what DMS holds for one resource of a deployment, as of the last operation +// that recorded it. +type Resource struct { + // Key is the bundle state key, as everything outside this package spells it. + Key string + ID string + + // State is the state the last operation recorded, as the opaque string the service + // stores, and empty when no operation recorded one. + State string +} + +// ListResources returns every resource DMS holds for the deployment. +func (c *Client) ListResources(ctx context.Context, deploymentID string) ([]Resource, error) { + it := c.Service.ListResources(ctx, bundledeployments.ListResourcesRequest{ + Parent: DeploymentName(deploymentID), + }) + + var out []Resource + for it.HasNext(ctx) { + res, err := it.Next(ctx) + if err != nil { + return nil, fmt.Errorf("listing resources from deployment metadata service: %w", err) + } + out = append(out, Resource{ + Key: statePrefix + res.ResourceKey, + ID: res.ResourceId, + State: res.State, + }) + } + return out, nil +} diff --git a/libs/testserver/bundledeployments.go b/libs/testserver/bundledeployments.go new file mode 100644 index 00000000000..3545036297a --- /dev/null +++ b/libs/testserver/bundledeployments.go @@ -0,0 +1,615 @@ +package testserver + +import ( + "bytes" + "encoding/json" + "fmt" + "path" + "slices" + "strconv" + "strings" + + "github.com/databricks/databricks-sdk-go/service/bundledeployments" + "github.com/databricks/databricks-sdk-go/service/workspace" +) + +// Handlers for the Deployment Metadata Service (DMS) API under /api/2.0/bundle. + +// maxOperationsPerVersion mirrors the service's compiled-in default. A bundle past it cannot +// be recorded, since the operation set is fixed when the version is created. +const maxOperationsPerVersion = 800 + +// operationStatusPending is what CreateVersion leaves on a staged operation. Declared here +// because the SDK enum is generated from the OpenAPI spec, which trails the service proto. +const operationStatusPending bundledeployments.OperationStatus = "OPERATION_STATUS_PENDING" + +// State is kept in FakeWorkspace.DmsDeployments, keyed by deployment ID. + +// dmsDeploymentNodeName is the workspace node name the service uses for deployments. +// It must match DEPLOYMENT_NODE_NAME on the service side (DeploymentWhsClient). +const dmsDeploymentNodeName = "resources.deployment.json" + +// dmsUpdatableOperationFields are the update_mask paths UpdateOperation accepts. Any +// other path is rejected, and action_type in particular is fixed when the operation is +// created. +var dmsUpdatableOperationFields = []string{"state", "error_message", "resource_id", "status"} + +// DmsDeployment holds a deployment record together with the versions and +// resources recorded under it, so the read APIs (ListVersions/ListResources) +// can serve back what deploys wrote. +type DmsDeployment struct { + Deployment bundledeployments.Deployment + Versions map[string]*bundledeployments.Version + // resources is the latest resource state per resource key, updated as + // operations are recorded. + Resources map[string]bundledeployments.Resource + // operations holds the recorded operations by resource name. The service keeps + // one per resource per version, so a resource written twice in a version updates + // its operation rather than adding another. + Operations map[string]*bundledeployments.Operation + // lastSuccessfulVersionID is the highest version completed successfully. + // The read path treats a non-empty value as "DMS owns the state"; + // the SDK Deployment struct does not yet carry this field. + LastSuccessfulVersionID string +} + +func (s *FakeWorkspace) CreateDeployment(req Request) Response { + var dep bundledeployments.Deployment + if err := json.Unmarshal(req.Body, &dep); err != nil { + return Response{StatusCode: 400, Body: map[string]string{"message": err.Error()}} + } + if dep.InitialParentPath == "" { + return Response{ + StatusCode: 400, + Body: map[string]string{"error_code": "INVALID_PARAMETER_VALUE", "message": "initial_parent_path is required"}, + } + } + + defer s.LockUnlock()() + + // The service registers the deployment as a workspace node under + // initial_parent_path and uses that node's ID as the deployment ID, so a + // get-status on the node path is how clients look the deployment back up. + nodePath := path.Join(dep.InitialParentPath, dmsDeploymentNodeName) + if resp, ok := s.requireParentDirectory(nodePath); !ok { + return resp + } + objectID := nextID() + s.files[nodePath] = FileEntry{ + Info: workspace.ObjectInfo{ + // The service registers its own node type, which get-status reports back; + // a plain FILE is what the fake used to claim and the real API never returns. + ObjectType: "BUNDLE_DEPLOYMENT", + Path: nodePath, + ObjectId: objectID, + }, + } + + // The record carries no version yet; last_version_id stays empty until + // the first CreateVersion. A failed registration leaves a record with no versions. + deploymentID := strconv.FormatInt(objectID, 10) + s.DmsDeploymentNodes[deploymentID] = nodePath + + if resp, ok := checkWorkspaceInfo(dep.WorkspaceInfo); !ok { + return resp + } + + dep.Name = "deployments/" + deploymentID + dep.Status = bundledeployments.DeploymentStatusDeploymentStatusActive + s.DmsDeployments[deploymentID] = &DmsDeployment{ + Deployment: dep, + Versions: map[string]*bundledeployments.Version{}, + Resources: map[string]bundledeployments.Resource{}, + Operations: map[string]*bundledeployments.Operation{}, + } + return Response{Body: dep} +} + +func (s *FakeWorkspace) GetDeployment(deploymentID string) Response { + defer s.LockUnlock()() + + d, ok := s.DmsDeployments[deploymentID] + if !ok { + return dmsNotFound("deployment " + deploymentID) + } + + body, err := deploymentBody(d) + if err != nil { + return Response{StatusCode: 500, Body: map[string]string{"message": err.Error()}} + } + return Response{Body: body} +} + +// deploymentBody renders a deployment with last_successful_version_id, which +// the SDK struct doesn't yet carry. Embedding in a wrapper won't work because +// Deployment.MarshalJSON silently drops sibling fields. +func deploymentBody(d *DmsDeployment) (map[string]any, error) { + raw, err := json.Marshal(d.Deployment) + if err != nil { + return nil, err + } + + var body map[string]any + dec := json.NewDecoder(bytes.NewReader(raw)) + dec.UseNumber() + if err := dec.Decode(&body); err != nil { + return nil, err + } + + if d.LastSuccessfulVersionID != "" { + body["last_successful_version_id"] = d.LastSuccessfulVersionID + } + return body, nil +} + +// dmsUpdatableDeploymentFields are the update_mask paths UpdateDeployment accepts. +var dmsUpdatableDeploymentFields = []string{"display_name", "target_name", "deployment_mode", "workspace_info"} + +// checkWorkspaceInfo rejects a bundle_root_path without the git_folder_path it is relative to, +// which is what the service does. +func checkWorkspaceInfo(ws *bundledeployments.WorkspaceInfo) (Response, bool) { + if ws != nil && (ws.GitFolderPath == "") != (ws.BundleRootPath == "") { + return dmsInvalidArgument("workspace_info.git_folder_path and workspace_info.bundle_root_path must be set together"), false + } + return Response{}, true +} + +func (s *FakeWorkspace) UpdateDeployment(req Request, deploymentID string) Response { + var raw map[string]json.RawMessage + if err := json.Unmarshal(req.Body, &raw); err != nil { + return Response{StatusCode: 400, Body: map[string]string{"message": err.Error()}} + } + var dep bundledeployments.Deployment + if err := json.Unmarshal(req.Body, &dep); err != nil { + return Response{StatusCode: 400, Body: map[string]string{"message": err.Error()}} + } + + updateMask := req.URL.Query().Get("update_mask") + if updateMask == "" { + return dmsInvalidArgument("update_mask is required") + } + update := map[string]bool{} + for path := range strings.SplitSeq(updateMask, ",") { + path = strings.TrimSpace(path) + if !slices.Contains(dmsUpdatableDeploymentFields, path) { + return dmsInvalidArgument("update_mask path " + path + " is not updatable") + } + if _, sent := raw[path]; !sent { + // An empty value is how a field is cleared, so the field has to be there to say so. + return dmsInvalidArgument(path + " is required when '" + path + "' is in update_mask (an empty value clears it)") + } + update[path] = true + } + + if resp, ok := checkWorkspaceInfo(dep.WorkspaceInfo); !ok { + return resp + } + + defer s.LockUnlock()() + + d, ok := s.DmsDeployments[deploymentID] + if !ok { + return dmsNotFound("deployment " + deploymentID) + } + + // Only the masked paths are written; every other field of the body is ignored. + if update["display_name"] { + d.Deployment.DisplayName = dep.DisplayName + } + if update["target_name"] { + d.Deployment.TargetName = dep.TargetName + } + if update["deployment_mode"] { + d.Deployment.DeploymentMode = dep.DeploymentMode + } + if update["workspace_info"] { + d.Deployment.WorkspaceInfo = dep.WorkspaceInfo + } + + body, err := deploymentBody(d) + if err != nil { + return Response{StatusCode: 500, Body: map[string]string{"message": err.Error()}} + } + return Response{Body: body} +} + +func (s *FakeWorkspace) DeleteDeployment(deploymentID string) Response { + defer s.LockUnlock()() + + // The service trashes the deployment's workspace node, so a later get-status + // on the node path reports the deployment as absent. + if nodePath, ok := s.DmsDeploymentNodes[deploymentID]; ok { + delete(s.files, nodePath) + } + delete(s.DmsDeploymentNodes, deploymentID) + delete(s.DmsDeployments, deploymentID) + return Response{Body: map[string]any{}} +} + +func (s *FakeWorkspace) CreateVersion(req Request, deploymentID string) Response { + versionID := req.URL.Query().Get("version_id") + + var version bundledeployments.Version + if err := json.Unmarshal(req.Body, &version); err != nil { + return Response{StatusCode: 400, Body: map[string]string{"message": err.Error()}} + } + + // operations rides in the version body like every other field, but is absent from the + // generated struct while the field is at DEVELOPMENT stage, so it is read separately. + // Embedding Version instead would promote its own UnmarshalJSON and silently drop this + // field. It is input only, so it is read here and never returned. + var staged struct { + Operations []struct { + ResourceKey string `json:"resource_key"` + ActionType bundledeployments.OperationActionType `json:"action_type"` + } `json:"operations"` + } + if err := json.Unmarshal(req.Body, &staged); err != nil { + return Response{StatusCode: 400, Body: map[string]string{"message": err.Error()}} + } + + defer s.LockUnlock()() + + d, ok := s.DmsDeployments[deploymentID] + if !ok { + return dmsNotFound("deployment " + deploymentID) + } + + // version_id must be numerically greater than the most recent version, + // and previous_version_id must name that version to detect concurrent deploys. + next, err := strconv.ParseInt(versionID, 10, 64) + if err != nil || next < 1 { + return dmsInvalidArgument("version_id must be a positive integer, got " + versionID) + } + var last int64 + if d.Deployment.LastVersionId != "" { + last, _ = strconv.ParseInt(d.Deployment.LastVersionId, 10, 64) + } + if next <= last { + return dmsInvalidArgument("version_id " + versionID + " must be greater than the most recent version " + d.Deployment.LastVersionId) + } + if version.PreviousVersionId != d.Deployment.LastVersionId { + return dmsAborted("previous_version_id is outdated; the deployment's most recent version is " + d.Deployment.LastVersionId) + } + + // Note: deployment lock not modelled. Tests kill the CLI mid-apply, leaving + // the version in-progress forever, whereas the real service lets the lease expire. + + // A version records its whole operation set up front; there is no API to add one later. + if len(staged.Operations) > maxOperationsPerVersion { + return Response{ + StatusCode: 429, + Body: map[string]string{ + "error_code": "RESOURCE_EXHAUSTED", + "message": fmt.Sprintf("a version may stage at most %d operations, got %d", maxOperationsPerVersion, len(staged.Operations)), + }, + } + } + seen := make(map[string]bool, len(staged.Operations)) + for _, staged := range staged.Operations { + switch { + case staged.ResourceKey == "": + return dmsInvalidArgument("operations.resource_key is required") + case !strings.Contains(staged.ResourceKey, "."): + return dmsInvalidArgument("operations.resource_key must have a known resource type prefix (e.g. 'jobs.', 'pipelines.'): " + staged.ResourceKey) + case staged.ActionType == "": + return dmsInvalidArgument("operations.action_type is required and must not be UNSPECIFIED for resource " + staged.ResourceKey) + case seen[staged.ResourceKey]: + return dmsInvalidArgument("operations must have distinct resource_keys; duplicate: " + staged.ResourceKey) + } + seen[staged.ResourceKey] = true + } + + d.Deployment.LastVersionId = versionID + version.Name = "deployments/" + deploymentID + "/versions/" + versionID + version.VersionId = versionID + version.Status = bundledeployments.VersionStatusVersionStatusInProgress + d.Versions[versionID] = &version + + // The deployment's git provenance is derived from the version that carried it; the rest of + // its metadata is written through CreateDeployment and UpdateDeployment. + d.Deployment.GitInfo = version.GitInfo + + // Each staged operation starts pending at sequence 0, and the CLI fills in its outcome + // with UpdateOperation as the resource is applied. + for _, staged := range staged.Operations { + opName := "deployments/" + deploymentID + "/versions/" + versionID + "/operations/" + staged.ResourceKey + d.Operations[opName] = &bundledeployments.Operation{ + Name: opName, + ResourceKey: staged.ResourceKey, + ActionType: staged.ActionType, + Status: operationStatusPending, + SequenceId: 0, + } + } + + return Response{Body: version} +} + +func (s *FakeWorkspace) CompleteVersion(req Request, deploymentID, versionID string) Response { + var completeReq bundledeployments.CompleteVersionRequest + if err := json.Unmarshal(req.Body, &completeReq); err != nil { + return Response{StatusCode: 400, Body: map[string]string{"message": err.Error()}} + } + + defer s.LockUnlock()() + + d, ok := s.DmsDeployments[deploymentID] + if !ok { + return dmsNotFound("deployment " + deploymentID) + } + v, ok := d.Versions[versionID] + if !ok { + return dmsNotFound("version " + versionID) + } + + v.Status = bundledeployments.VersionStatusVersionStatusCompleted + v.CompletionReason = completeReq.CompletionReason + if completeReq.CompletionReason == bundledeployments.VersionCompleteVersionCompleteSuccess { + d.LastSuccessfulVersionID = versionID + } + return Response{Body: *v} +} + +func (s *FakeWorkspace) Heartbeat() Response { + return Response{Body: bundledeployments.HeartbeatResponse{}} +} + +// operationBody renders an operation the way the service does: sequence_id as a +// JSON string, which the SDK struct cannot express (it types the field int64). +func operationBody(op *bundledeployments.Operation) (map[string]any, error) { + raw, err := json.Marshal(op) + if err != nil { + return nil, err + } + + var body map[string]any + dec := json.NewDecoder(bytes.NewReader(raw)) + dec.UseNumber() + if err := dec.Decode(&body); err != nil { + return nil, err + } + + body["sequence_id"] = strconv.FormatInt(op.SequenceId, 10) + return body, nil +} + +// UpdateOperation applies a later write for a resource already recorded in this +// version. sequence_id is the concurrency precondition and increments on success. +func (s *FakeWorkspace) UpdateOperation(req Request, deploymentID, versionID, resourceKey string) Response { + // sequence_id arrives as a string, which the SDK struct cannot hold (it types the + // field int64), so read the body twice: once for the typed fields with that key + // removed, and once for the precondition alone. + var raw map[string]json.RawMessage + if err := json.Unmarshal(req.Body, &raw); err != nil { + return Response{StatusCode: 400, Body: map[string]string{"message": err.Error()}} + } + var precondition struct { + SequenceId string `json:"sequence_id"` + } + if err := json.Unmarshal(req.Body, &precondition); err != nil { + return Response{StatusCode: 400, Body: map[string]string{"message": err.Error()}} + } + delete(raw, "sequence_id") + typedBody, err := json.Marshal(raw) + if err != nil { + return Response{StatusCode: 500, Body: map[string]string{"message": err.Error()}} + } + var op bundledeployments.Operation + if err := json.Unmarshal(typedBody, &op); err != nil { + return Response{StatusCode: 400, Body: map[string]string{"message": err.Error()}} + } + + updateMask := req.URL.Query().Get("update_mask") + if updateMask == "" { + return dmsInvalidArgument("update_mask is required") + } + // Only masked paths are written; other fields keep their values. + // Omitting state keeps the already-recorded state. + update := map[string]bool{} + for path := range strings.SplitSeq(updateMask, ",") { + path = strings.TrimSpace(path) + if !slices.Contains(dmsUpdatableOperationFields, path) { + return dmsInvalidArgument("update_mask path " + path + " is not updatable") + } + // state is the exception: masked with no value is how the service is told the resource + // is gone, which is what stops it being listed. + if _, sent := raw[path]; !sent && path != "state" { + // An empty value is how a field is cleared, so the field has to be there to say so. + return dmsInvalidArgument(path + " is required when '" + path + "' is in update_mask (an empty value clears it)") + } + update[path] = true + } + + // The service parses the status as a proto enum, so a value outside the enum arrives as + // UNSPECIFIED and is refused. Checked here too: a status the API does not have otherwise + // passes every local test and fails only against a real workspace. + if update["status"] { + switch op.Status { + case bundledeployments.OperationStatusOperationStatusSucceeded, + bundledeployments.OperationStatusOperationStatusFailed, + operationStatusPending: + default: + return dmsInvalidArgument("status is required and must not be UNSPECIFIED when 'status' is in update_mask") + } + } + + // The service counts only a state it was given as recording one, so these ask whether a + // value arrived rather than whether the mask names the field. + _, stateSent := raw["state"] + recordsState := update["state"] && stateSent + + // A resource_id can be written but not cleared, and only alongside the state it belongs to. + if update["resource_id"] { + if op.ResourceId == "" { + return dmsInvalidArgument("resource_id is required when 'resource_id' is in update_mask") + } + if !recordsState { + return dmsInvalidArgument("state must be in update_mask when 'resource_id' is") + } + } + + defer s.LockUnlock()() + + d, ok := s.DmsDeployments[deploymentID] + if !ok { + return dmsNotFound("deployment " + deploymentID) + } + + opName := "deployments/" + deploymentID + "/versions/" + versionID + "/operations/" + resourceKey + existing, ok := d.Operations[opName] + if !ok { + return dmsNotFound("operation " + opName) + } + if precondition.SequenceId != strconv.FormatInt(existing.SequenceId, 10) { + return dmsAborted("sequence_id is outdated; the operation is at " + strconv.FormatInt(existing.SequenceId, 10)) + } + + // Invariants check the operation after the update, not the request. + // The mask leaves unspecified fields unchanged. + after := *existing + if update["state"] { + after.State = op.State + } + if update["error_message"] { + after.ErrorMessage = op.ErrorMessage + } + if update["resource_id"] { + after.ResourceId = op.ResourceId + } + if update["status"] { + after.Status = op.Status + } + + failed := after.Status == bundledeployments.OperationStatusOperationStatusFailed + if !failed && after.ErrorMessage != "" { + return dmsInvalidArgument("error_message is only allowed when status is OPERATION_STATUS_FAILED") + } + + // A failure may leave no state only where nothing is left to describe: a create that never + // landed, or a recreate whose delete already went through. + if failed && after.State == "" && leavesLiveResource(after.ActionType) { + return dmsInvalidArgument("state is required for a " + string(after.ActionType) + + " operation, because it acts on a resource that already exists and cannot destroy it, even when it fails") + } + + // An operation with state must identify its resource via resource_id, + // even for failed operations reporting prior state. + if after.State != "" && after.ResourceId == "" { + return dmsInvalidArgument("resource_id is required for an operation that records state") + } + + // Only the masked fields change; action_type and resource_key stay as created. + if update["state"] { + existing.State = op.State + } + if update["error_message"] { + existing.ErrorMessage = op.ErrorMessage + } + if update["resource_id"] { + existing.ResourceId = op.ResourceId + } + if update["status"] { + existing.Status = op.Status + } + existing.SequenceId++ + + body, err := operationBody(existing) + if err != nil { + return Response{StatusCode: 500, Body: map[string]string{"message": err.Error()}} + } + + // Only an update that names state moves the resource: naming it with no value clears + // it and removes the resource, and an update that leaves it out - a failure reporting + // its outcome - must not disturb what the deployment already holds. Every version + // stages its operations without state, so re-deriving this from the operation + // regardless of the mask would drop a resource whose deploy failed before writing. + if update["state"] { + if existing.State == "" { + delete(d.Resources, resourceKey) + } else { + d.Resources[resourceKey] = bundledeployments.Resource{ + Name: "deployments/" + deploymentID + "/resources/" + resourceKey, + ResourceKey: resourceKey, + ResourceId: existing.ResourceId, + ResourceType: existing.ResourceType, + LastActionType: existing.ActionType, + LastVersionId: versionID, + State: existing.State, + } + } + } else if resource, projected := d.Resources[resourceKey]; projected && update["resource_id"] { + // resource_id is mirrored too, for a resource the deployment still holds. + resource.ResourceId = existing.ResourceId + d.Resources[resourceKey] = resource + } + + return Response{Body: body} +} + +func (s *FakeWorkspace) ListResources(deploymentID string) Response { + defer s.LockUnlock()() + + d, ok := s.DmsDeployments[deploymentID] + if !ok { + return dmsNotFound("deployment " + deploymentID) + } + + // Sort by resource key so the response order is deterministic. + keys := make([]string, 0, len(d.Resources)) + for key := range d.Resources { + keys = append(keys, key) + } + slices.Sort(keys) + + resources := make([]bundledeployments.Resource, 0, len(keys)) + for _, key := range keys { + resources = append(resources, d.Resources[key]) + } + return Response{Body: bundledeployments.ListResourcesResponse{Resources: resources}} +} + +// leavesLiveResource reports whether a failed operation of this type leaves a resource behind +// that it still has to describe. A create leaves nothing, and a recreate already deleted. +func leavesLiveResource(action bundledeployments.OperationActionType) bool { + switch action { + case bundledeployments.OperationActionTypeOperationActionTypeCreate, + bundledeployments.OperationActionTypeOperationActionTypeRecreate: + return false + default: + return true + } +} + +// dmsNotFound returns the RESOURCE_DOES_NOT_EXIST error shape the DMS API uses, +// which the SDK maps to apierr.ErrNotFound. +func dmsNotFound(what string) Response { + return Response{ + StatusCode: 404, + // Content-Type is required for the SDK to parse the body into a typed error, + // which is what callers match against apierr.ErrResourceDoesNotExist. + Headers: map[string][]string{"Content-Type": {"application/json"}}, + Body: map[string]string{ + "error_code": "RESOURCE_DOES_NOT_EXIST", + "message": what + " does not exist", + }, + } +} + +// dmsAborted returns the 409 ABORTED error the server uses for the version +// optimistic-concurrency check. +func dmsInvalidArgument(message string) Response { + return Response{ + StatusCode: 400, + Headers: map[string][]string{"Content-Type": {"application/json"}}, + Body: map[string]string{"error_code": "INVALID_PARAMETER_VALUE", "message": message}, + } +} + +func dmsAborted(message string) Response { + return Response{ + StatusCode: 409, + Headers: map[string][]string{"Content-Type": {"application/json"}}, + Body: map[string]string{"error_code": "ABORTED", "message": message}, + } +} diff --git a/libs/testserver/fake_workspace.go b/libs/testserver/fake_workspace.go index 2f077b1b631..b544c17538e 100644 --- a/libs/testserver/fake_workspace.go +++ b/libs/testserver/fake_workspace.go @@ -248,6 +248,15 @@ type FakeWorkspace struct { // clusterVenvs caches Python venvs per existing cluster ID, // matching cloud behavior where libraries are cached on running clusters. clusterVenvs map[string]*clusterEnv + + // DmsDeployments holds Deployment Metadata Service (DMS) records, keyed by + // deployment ID. Each record carries its versions and latest resource state. + DmsDeployments map[string]*DmsDeployment + + // DmsDeploymentNodes maps deployment ID to the workspace node CreateDeployment made for + // it. An ID appears here before DmsDeployments has a record, which its first version + // creates, so the node is what makes the ID valid in between. + DmsDeploymentNodes map[string]string } func (s *FakeWorkspace) LockUnlock() func() { @@ -496,6 +505,8 @@ func NewFakeWorkspace(url, token string) *FakeWorkspace { postgresImplicitBranches: map[string]bool{}, postgresImplicitEndpoints: map[string]bool{}, clusterVenvs: map[string]*clusterEnv{}, + DmsDeployments: map[string]*DmsDeployment{}, + DmsDeploymentNodes: map[string]string{}, Alerts: map[string]sql.AlertV2{}, Experiments: map[string]ml.GetExperimentResponse{}, ModelRegistryModels: map[string]ml.Model{}, diff --git a/libs/testserver/handlers.go b/libs/testserver/handlers.go index 8c44d97eae7..33a742cc2ce 100644 --- a/libs/testserver/handlers.go +++ b/libs/testserver/handlers.go @@ -279,6 +279,35 @@ func AddDefaultHandlers(server *Server) { return req.Workspace.JobsCreate(req) }) + // Deployment Metadata Service (DMS) endpoints. + server.Handle("POST", "/api/2.0/bundle/deployments", func(req Request) any { + return req.Workspace.CreateDeployment(req) + }) + server.Handle("GET", "/api/2.0/bundle/deployments/{deployment_id}", func(req Request) any { + return req.Workspace.GetDeployment(req.Vars["deployment_id"]) + }) + server.Handle("PATCH", "/api/2.0/bundle/deployments/{deployment_id}", func(req Request) any { + return req.Workspace.UpdateDeployment(req, req.Vars["deployment_id"]) + }) + server.Handle("DELETE", "/api/2.0/bundle/deployments/{deployment_id}", func(req Request) any { + return req.Workspace.DeleteDeployment(req.Vars["deployment_id"]) + }) + server.Handle("POST", "/api/2.0/bundle/deployments/{deployment_id}/versions", func(req Request) any { + return req.Workspace.CreateVersion(req, req.Vars["deployment_id"]) + }) + server.Handle("POST", "/api/2.0/bundle/deployments/{deployment_id}/versions/{version_id}/complete", func(req Request) any { + return req.Workspace.CompleteVersion(req, req.Vars["deployment_id"], req.Vars["version_id"]) + }) + server.Handle("POST", "/api/2.0/bundle/deployments/{deployment_id}/versions/{version_id}/heartbeat", func(req Request) any { + return req.Workspace.Heartbeat() + }) + server.Handle("PATCH", "/api/2.0/bundle/deployments/{deployment_id}/versions/{version_id}/operations/{resource_key}", func(req Request) any { + return req.Workspace.UpdateOperation(req, req.Vars["deployment_id"], req.Vars["version_id"], req.Vars["resource_key"]) + }) + server.Handle("GET", "/api/2.0/bundle/deployments/{deployment_id}/resources", func(req Request) any { + return req.Workspace.ListResources(req.Vars["deployment_id"]) + }) + server.Handle("POST", "/api/2.2/jobs/delete", func(req Request) any { var request jobs.DeleteJob if err := json.Unmarshal(req.Body, &request); err != nil { diff --git a/libs/workspaceurls/urls.go b/libs/workspaceurls/urls.go index 61c3f271468..0b6a80fd71b 100644 --- a/libs/workspaceurls/urls.go +++ b/libs/workspaceurls/urls.go @@ -4,6 +4,7 @@ import ( "fmt" "net/url" "slices" + "strconv" "strings" ) @@ -71,6 +72,22 @@ func ResourceTypes() []string { return names } +// DeploymentURL returns the workspace URL for a bundle deployment: +// /deployments/?version=. Version pins the page to the deploy that produced it. +func DeploymentURL(baseURL url.URL, deploymentID string, version int64) string { + if deploymentID == "" { + return "" + } + + baseURL.Path = "deployments/" + deploymentID + if version > 0 { + values := baseURL.Query() + values.Set("version", strconv.FormatInt(version, 10)) + baseURL.RawQuery = values.Encode() + } + return baseURL.String() +} + // JobRunPath returns the modern workspace path for a job run, of the form // // jobs//runs/ diff --git a/libs/workspaceurls/urls_test.go b/libs/workspaceurls/urls_test.go index a398167e027..b92df037c84 100644 --- a/libs/workspaceurls/urls_test.go +++ b/libs/workspaceurls/urls_test.go @@ -141,6 +141,55 @@ func TestResourceURL(t *testing.T) { } } +func TestDeploymentURL(t *testing.T) { + tests := []struct { + name string + deploymentID string + version int64 + base url.URL + expected string + }{ + { + name: "id and version", + deploymentID: "996980114684409", + version: 2, + base: url.URL{Scheme: "https", Host: "host.com"}, + expected: "https://host.com/deployments/996980114684409?version=2", + }, + { + // The version is only known once CreateVersion has run, so link to the + // deployment itself rather than emitting version=0. + name: "zero version omits the query", + deploymentID: "996980114684409", + version: 0, + base: url.URL{Scheme: "https", Host: "host.com"}, + expected: "https://host.com/deployments/996980114684409", + }, + { + // A base URL carrying ?w= keeps it, so a vanity or legacy + // host still addresses the right workspace. + name: "preserves an existing query", + deploymentID: "42", + version: 7, + base: url.URL{Scheme: "https", Host: "host.com", RawQuery: "w=123"}, + expected: "https://host.com/deployments/42?version=7&w=123", + }, + { + name: "empty id returns empty", + deploymentID: "", + version: 1, + base: url.URL{Scheme: "https", Host: "host.com"}, + expected: "", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.expected, DeploymentURL(tt.base, tt.deploymentID, tt.version)) + }) + } +} + func TestHasWorkspaceIDInHostname(t *testing.T) { tests := []struct { name string From a9ce3a277f55e3c96729cb2a2141acf4f1d4c828 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Fri, 4 Sep 2026 02:02:34 +0000 Subject: [PATCH 02/46] acceptance/dms: list-resources shows empty tombstone for no-resources Co-authored-by: Isaac --- acceptance/bundle/dms/no-resources/output.txt | 6 ++++++ acceptance/bundle/dms/no-resources/script | 6 ++++++ 2 files changed, 12 insertions(+) diff --git a/acceptance/bundle/dms/no-resources/output.txt b/acceptance/bundle/dms/no-resources/output.txt index a4f993a04e5..5c80f26a912 100644 --- a/acceptance/bundle/dms/no-resources/output.txt +++ b/acceptance/bundle/dms/no-resources/output.txt @@ -78,3 +78,9 @@ Resources: 0 created, 0 changed, 0 deleted, 0 unchanged "completion_reason": "VERSION_COMPLETE_SUCCESS" } } + +=== The service lists no resources for the deployment: nothing was recorded beyond the deployment itself +>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[NUMID] -o json +[ + +] diff --git a/acceptance/bundle/dms/no-resources/script b/acceptance/bundle/dms/no-resources/script index 13924f3664f..81444e00ccf 100644 --- a/acceptance/bundle/dms/no-resources/script +++ b/acceptance/bundle/dms/no-resources/script @@ -8,3 +8,9 @@ trace MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_U title "Redeploy: the deployment is resolved from that node, so no second deployment is created" trace $CLI bundle deploy trace print_requests.py --dms //api/2.0/bundle --get + +title "The service lists no resources for the deployment: nothing was recorded beyond the deployment itself" +deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-no-resources-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') +trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json +# Discard the get-status + list-resources reads above so out.requests.txt is not a golden. +rm -f out.requests.txt From 4abbea01168c1a6a56a5696c26374b02605646e9 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Fri, 4 Sep 2026 02:12:56 +0000 Subject: [PATCH 03/46] acceptance/dms: assert declined deploy leaves recorded resources unchanged Co-authored-by: Isaac --- acceptance/bundle/dms/declined-deploy/output.txt | 10 ++++++++++ acceptance/bundle/dms/declined-deploy/script | 4 ++++ 2 files changed, 14 insertions(+) diff --git a/acceptance/bundle/dms/declined-deploy/output.txt b/acceptance/bundle/dms/declined-deploy/output.txt index d09e8d18743..faab3106201 100644 --- a/acceptance/bundle/dms/declined-deploy/output.txt +++ b/acceptance/bundle/dms/declined-deploy/output.txt @@ -76,6 +76,16 @@ Files: 4 uploaded, 0 deleted === Nothing was recorded for the declined deploy - no version, so none to abort >>> print_requests.py --dms //api/2.0/bundle +=== The recorded resources are unchanged by the decline: the schema is still at version 1 +>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[NUMID] -o json +[ + { + "last_action_type": "OPERATION_ACTION_TYPE_CREATE", + "last_version_id": "1", + "resource_key": "schemas.foo" + } +] + >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: delete resources.schemas.foo diff --git a/acceptance/bundle/dms/declined-deploy/script b/acceptance/bundle/dms/declined-deploy/script index e67ff786b19..c8aa0a40301 100644 --- a/acceptance/bundle/dms/declined-deploy/script +++ b/acceptance/bundle/dms/declined-deploy/script @@ -18,5 +18,9 @@ title "Nothing was recorded for the declined deploy - no version, so none to abo # history has no entry that reads like a deploy which failed or did nothing. trace print_requests.py --dms //api/2.0/bundle +title "The recorded resources are unchanged by the decline: the schema is still at version 1" +deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-declined-deploy-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') +trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' + trace $CLI bundle destroy --auto-approve rm -f out.requests.txt From 2a56b8be41f15f66fc3cd3ec8740776457553aec Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Fri, 4 Sep 2026 02:16:39 +0000 Subject: [PATCH 04/46] acceptance/dms: list-resources coverage for depends-on and emptied-resource Co-authored-by: Isaac --- acceptance/bundle/dms/depends-on/output.txt | 15 +++++++++++++++ acceptance/bundle/dms/depends-on/script | 4 ++++ acceptance/bundle/dms/emptied-resource/output.txt | 10 ++++++++++ acceptance/bundle/dms/emptied-resource/script | 4 ++++ 4 files changed, 33 insertions(+) diff --git a/acceptance/bundle/dms/depends-on/output.txt b/acceptance/bundle/dms/depends-on/output.txt index 4807497d63c..b36e5cfb154 100644 --- a/acceptance/bundle/dms/depends-on/output.txt +++ b/acceptance/bundle/dms/depends-on/output.txt @@ -37,6 +37,21 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged } } +=== The service lists both jobs, each recorded at version 1 +>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[NUMID] -o json +[ + { + "last_action_type": "OPERATION_ACTION_TYPE_CREATE", + "last_version_id": "1", + "resource_key": "jobs.child" + }, + { + "last_action_type": "OPERATION_ACTION_TYPE_CREATE", + "last_version_id": "1", + "resource_key": "jobs.parent" + } +] + >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: delete resources.jobs.child diff --git a/acceptance/bundle/dms/depends-on/script b/acceptance/bundle/dms/depends-on/script index 7f5576ce72b..dd168459b9a 100644 --- a/acceptance/bundle/dms/depends-on/script +++ b/acceptance/bundle/dms/depends-on/script @@ -4,5 +4,9 @@ title "Deploy a job that references another: depends_on is recorded alongside th trace $CLI bundle deploy trace print_requests.py --dms //versions/1/operations +title "The service lists both jobs, each recorded at version 1" +deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-depends-on-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') +trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' + trace $CLI bundle destroy --auto-approve rm -f out.requests.txt diff --git a/acceptance/bundle/dms/emptied-resource/output.txt b/acceptance/bundle/dms/emptied-resource/output.txt index 483b3d8249d..7d0ad36a596 100644 --- a/acceptance/bundle/dms/emptied-resource/output.txt +++ b/acceptance/bundle/dms/emptied-resource/output.txt @@ -130,6 +130,16 @@ Resources: 0 created, 1 changed, 0 deleted, 1 unchanged >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged +=== The grants node emptied out, so the service no longer lists it - only the schema remains +>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[NUMID] -o json +[ + { + "last_action_type": "OPERATION_ACTION_TYPE_CREATE", + "last_version_id": "1", + "resource_key": "schemas.foo" + } +] + >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: delete resources.schemas.foo diff --git a/acceptance/bundle/dms/emptied-resource/script b/acceptance/bundle/dms/emptied-resource/script index b38be294ce1..1e0606a31c0 100644 --- a/acceptance/bundle/dms/emptied-resource/script +++ b/acceptance/bundle/dms/emptied-resource/script @@ -16,5 +16,9 @@ trace print_requests.py --dms //api/2.0/bundle title "Plan again: reading state back from the service works and reports no work" trace $CLI bundle plan | contains.py "Plan: 0 to add, 0 to change, 0 to delete" "!unexpected end of JSON input" +title "The grants node emptied out, so the service no longer lists it - only the schema remains" +deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-emptied-resource-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') +trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' + trace $CLI bundle destroy --auto-approve rm -f out.requests.txt From af3ffde6270668d8d73e0340a90a08e7126c0f9a Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Fri, 4 Sep 2026 02:21:37 +0000 Subject: [PATCH 05/46] acceptance/dms: inline bundle summary into no-drift + declined-deploy, drop dedicated summary test Co-authored-by: Isaac --- .../bundle/dms/declined-deploy/output.txt | 4 ++ acceptance/bundle/dms/declined-deploy/script | 2 + acceptance/bundle/dms/no-drift/output.txt | 12 ++++++ acceptance/bundle/dms/no-drift/script | 2 + .../bundle/dms/summary/databricks.yml.tmpl | 9 ----- acceptance/bundle/dms/summary/out.test.toml | 3 -- acceptance/bundle/dms/summary/output.txt | 37 ------------------- acceptance/bundle/dms/summary/script | 17 --------- 8 files changed, 20 insertions(+), 66 deletions(-) delete mode 100644 acceptance/bundle/dms/summary/databricks.yml.tmpl delete mode 100644 acceptance/bundle/dms/summary/out.test.toml delete mode 100644 acceptance/bundle/dms/summary/output.txt delete mode 100644 acceptance/bundle/dms/summary/script diff --git a/acceptance/bundle/dms/declined-deploy/output.txt b/acceptance/bundle/dms/declined-deploy/output.txt index faab3106201..ef8a2cb048b 100644 --- a/acceptance/bundle/dms/declined-deploy/output.txt +++ b/acceptance/bundle/dms/declined-deploy/output.txt @@ -96,3 +96,7 @@ This action will result in the deletion of the following UC schemas. Any underly All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/dms-declined-deploy-[UNIQUE_NAME]/default Destroy: 1 deleted + +=== After the destroy the deployment is gone, so the summary reports no history rather than a dangling id +>>> [CLI] bundle summary -o json +false diff --git a/acceptance/bundle/dms/declined-deploy/script b/acceptance/bundle/dms/declined-deploy/script index c8aa0a40301..df539b83b29 100644 --- a/acceptance/bundle/dms/declined-deploy/script +++ b/acceptance/bundle/dms/declined-deploy/script @@ -23,4 +23,6 @@ deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/$ trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' trace $CLI bundle destroy --auto-approve +title "After the destroy the deployment is gone, so the summary reports no history rather than a dangling id" +trace $CLI bundle summary -o json | jq '.bundle.deployment | has("history")' rm -f out.requests.txt diff --git a/acceptance/bundle/dms/no-drift/output.txt b/acceptance/bundle/dms/no-drift/output.txt index 7b23792edcd..952b5b2731e 100644 --- a/acceptance/bundle/dms/no-drift/output.txt +++ b/acceptance/bundle/dms/no-drift/output.txt @@ -21,6 +21,12 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged } ] +>>> [CLI] bundle summary -o json +{ + "deployment_id": "[NUMID]", + "latest_version_id": "1" +} + === Plan without touching anything: the deployment stamp must not show as drift >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged @@ -121,6 +127,12 @@ Resources: 0 created, 1 changed, 0 deleted, 1 unchanged } ] +>>> [CLI] bundle summary -o json +{ + "deployment_id": "[NUMID]", + "latest_version_id": "2" +} + === Plan again: no residual drift, and the version stamps are ignored >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged diff --git a/acceptance/bundle/dms/no-drift/script b/acceptance/bundle/dms/no-drift/script index 8a18d13f258..dfcb0b55ef5 100644 --- a/acceptance/bundle/dms/no-drift/script +++ b/acceptance/bundle/dms/no-drift/script @@ -5,6 +5,7 @@ trace $CLI bundle deploy deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-no-drift-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' +trace $CLI bundle summary -o json | jq .bundle.deployment.history # Only deploy stamps deployment.deployment_id. Plan sees it in state but not in config, # so it must ignore it rather than report spurious changes. @@ -17,6 +18,7 @@ trace update_file.py databricks.yml 'serverless: true' 'serverless: true trace $CLI bundle deploy trace print_requests.py --dms //api/2.2/jobs //api/2.0/pipelines --sort trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' +trace $CLI bundle summary -o json | jq .bundle.deployment.history title "Plan again: no residual drift, and the version stamps are ignored" trace $CLI bundle plan | contains.py "Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged" diff --git a/acceptance/bundle/dms/summary/databricks.yml.tmpl b/acceptance/bundle/dms/summary/databricks.yml.tmpl deleted file mode 100644 index 1c2fa3c7f20..00000000000 --- a/acceptance/bundle/dms/summary/databricks.yml.tmpl +++ /dev/null @@ -1,9 +0,0 @@ -bundle: - name: dms-summary-$UNIQUE_NAME -experimental: - record_deployment_history: true - -resources: - jobs: - foo: - name: foo diff --git a/acceptance/bundle/dms/summary/out.test.toml b/acceptance/bundle/dms/summary/out.test.toml deleted file mode 100644 index 23c07f70dca..00000000000 --- a/acceptance/bundle/dms/summary/out.test.toml +++ /dev/null @@ -1,3 +0,0 @@ -Cloud = true -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -EnvMatrix.DMS = ["true"] diff --git a/acceptance/bundle/dms/summary/output.txt b/acceptance/bundle/dms/summary/output.txt deleted file mode 100644 index 233f2416dc3..00000000000 --- a/acceptance/bundle/dms/summary/output.txt +++ /dev/null @@ -1,37 +0,0 @@ - -=== Summary reports the deployment recorded with the metadata service, so a caller can find the deployment and the version it is on ->>> [CLI] bundle deploy -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-summary-[UNIQUE_NAME]/default/files... -Created jobs.foo -Files: 5 uploaded, 0 deleted -Resources: 1 created, 0 changed, 0 deleted, 0 unchanged - ->>> [CLI] bundle summary -o json -{ - "deployment_id": "[NUMID]", - "latest_version_id": "1" -} - -=== Redeploying advances the version the summary reports ->>> [CLI] bundle deploy -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-summary-[UNIQUE_NAME]/default/files... -Files: 2 uploaded, 0 deleted -Resources: 0 created, 0 changed, 0 deleted, 1 unchanged - ->>> [CLI] bundle summary -o json -{ - "deployment_id": "[NUMID]", - "latest_version_id": "2" -} - -=== After a destroy the deployment is gone, so the summary reports no history rather than a dangling ID ->>> [CLI] bundle destroy --auto-approve -The following resources will be deleted: - delete resources.jobs.foo - -All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/dms-summary-[UNIQUE_NAME]/default - -Destroy: 1 deleted - ->>> [CLI] bundle summary -o json -false diff --git a/acceptance/bundle/dms/summary/script b/acceptance/bundle/dms/summary/script deleted file mode 100644 index 75ec8612ffe..00000000000 --- a/acceptance/bundle/dms/summary/script +++ /dev/null @@ -1,17 +0,0 @@ -envsubst < databricks.yml.tmpl > databricks.yml - -title "Summary reports the deployment recorded with the metadata service, so a caller can find the deployment and the version it is on" -trace $CLI bundle deploy -trace $CLI bundle summary -o json | jq .bundle.deployment.history - -title "Redeploying advances the version the summary reports" -trace $CLI bundle deploy -trace $CLI bundle summary -o json | jq .bundle.deployment.history - -title "After a destroy the deployment is gone, so the summary reports no history rather than a dangling ID" -trace $CLI bundle destroy --auto-approve -trace $CLI bundle summary -o json | jq '.bundle.deployment | has("history")' - -# This test asserts the summary output, not the requests behind it; the file uploads -# recorded here are ordered nondeterministically. -rm -f out.requests.txt From 0978bb8d140fa143e04c589ad00fc6f3408d3d64 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Fri, 4 Sep 2026 08:17:06 +0000 Subject: [PATCH 06/46] libs/dms: document the three SDK gaps the raw client works around; drop redundant operation-upload-message test Co-authored-by: Isaac --- .../operation-upload-message/databricks.yml.tmpl | 9 --------- .../dms/operation-upload-message/out.test.toml | 3 --- .../dms/operation-upload-message/output.txt | 16 ---------------- .../bundle/dms/operation-upload-message/script | 5 ----- .../dms/operation-upload-message/test.toml | 8 -------- libs/dms/client.go | 9 ++++++++- 6 files changed, 8 insertions(+), 42 deletions(-) delete mode 100644 acceptance/bundle/dms/operation-upload-message/databricks.yml.tmpl delete mode 100644 acceptance/bundle/dms/operation-upload-message/out.test.toml delete mode 100644 acceptance/bundle/dms/operation-upload-message/output.txt delete mode 100644 acceptance/bundle/dms/operation-upload-message/script delete mode 100644 acceptance/bundle/dms/operation-upload-message/test.toml diff --git a/acceptance/bundle/dms/operation-upload-message/databricks.yml.tmpl b/acceptance/bundle/dms/operation-upload-message/databricks.yml.tmpl deleted file mode 100644 index bc5b4ec97ad..00000000000 --- a/acceptance/bundle/dms/operation-upload-message/databricks.yml.tmpl +++ /dev/null @@ -1,9 +0,0 @@ -bundle: - name: dms-operation-upload-message-$UNIQUE_NAME -experimental: - record_deployment_history: true - -resources: - jobs: - foo: - name: foo diff --git a/acceptance/bundle/dms/operation-upload-message/out.test.toml b/acceptance/bundle/dms/operation-upload-message/out.test.toml deleted file mode 100644 index 9921e91a794..00000000000 --- a/acceptance/bundle/dms/operation-upload-message/out.test.toml +++ /dev/null @@ -1,3 +0,0 @@ -Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -EnvMatrix.DMS = ["true"] diff --git a/acceptance/bundle/dms/operation-upload-message/output.txt b/acceptance/bundle/dms/operation-upload-message/output.txt deleted file mode 100644 index 0791010b1f1..00000000000 --- a/acceptance/bundle/dms/operation-upload-message/output.txt +++ /dev/null @@ -1,16 +0,0 @@ - -=== What a user sees when an operation cannot be recorded ->>> fault.py PATCH /api/2.0/bundle/* 500 0 1 INTERNAL_ERROR - ->>> errcode [CLI] bundle deploy -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-operation-upload-message-[UNIQUE_NAME]/default/files... -Error: recording operation for resources.jobs.foo: Fault injected by test. (500 INTERNAL_ERROR) - -Endpoint: PATCH [DATABRICKS_URL]/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.foo?update_mask=state%2Cerror_message%2Cresource_id%2Cstatus -HTTP Status: 500 Internal Server Error -API error_code: INTERNAL_ERROR -API message: Fault injected by test. - -Files: 5 uploaded, 0 deleted - -Exit code: 1 diff --git a/acceptance/bundle/dms/operation-upload-message/script b/acceptance/bundle/dms/operation-upload-message/script deleted file mode 100644 index c6b796e4149..00000000000 --- a/acceptance/bundle/dms/operation-upload-message/script +++ /dev/null @@ -1,5 +0,0 @@ -envsubst < databricks.yml.tmpl > databricks.yml - -title "What a user sees when an operation cannot be recorded" -trace fault.py "PATCH /api/2.0/bundle/*" 500 0 1 INTERNAL_ERROR -trace errcode $CLI bundle deploy diff --git a/acceptance/bundle/dms/operation-upload-message/test.toml b/acceptance/bundle/dms/operation-upload-message/test.toml deleted file mode 100644 index 1dbd55da1f0..00000000000 --- a/acceptance/bundle/dms/operation-upload-message/test.toml +++ /dev/null @@ -1,8 +0,0 @@ -# Local only: this case needs an injected failure - DMS itself must reject the operation upload - and fault.py registers -# its rule on the fake, which a cloud run does not have. -Cloud = false - -# One resource, so which operation is refused is fixed and the error a user sees can be -# asserted. operation-upload-fails covers a deploy of several resources, where which ones -# were reached before the failure landed is not. -RecordRequests = false diff --git a/libs/dms/client.go b/libs/dms/client.go index 8dbf9915399..fc1f482c6b2 100644 --- a/libs/dms/client.go +++ b/libs/dms/client.go @@ -158,7 +158,14 @@ type operationResponse struct { SequenceId string `json:"sequence_id,omitempty"` } -// rawClient sends the requests the generated client cannot express. +// rawClient sends the three DMS requests the generated SDK client cannot express yet, while the +// endpoints are at DEVELOPMENT stage. TODO(DMS): drop each once the SDK catches up. +// - UpdateDeployment: the SDK exposes no method for it. +// - CreateVersion: the SDK's Version type has no operations field, so a version cannot stage +// its operations through it. +// - UpdateOperation: the SDK types sequence_id as int64, but the service sends and expects it +// as a JSON string. +// CreateDeployment, DeleteDeployment, CompleteVersion and ListResources go through the SDK (see Client). type rawClient struct { client *client.DatabricksClient } From 6163285765742e9b2a9af70bcf47ffd74413b5f1 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Fri, 4 Sep 2026 08:21:59 +0000 Subject: [PATCH 07/46] acceptance/dms: fold serialized-plan header assertion into stale-plan, drop serialized-plan --- .../dms/serialized-plan/databricks.yml.tmpl | 9 ---- .../bundle/dms/serialized-plan/out.test.toml | 3 -- .../bundle/dms/serialized-plan/output.txt | 41 ------------------- acceptance/bundle/dms/serialized-plan/script | 20 --------- acceptance/bundle/dms/stale-plan/output.txt | 8 ++++ acceptance/bundle/dms/stale-plan/script | 3 ++ 6 files changed, 11 insertions(+), 73 deletions(-) delete mode 100644 acceptance/bundle/dms/serialized-plan/databricks.yml.tmpl delete mode 100644 acceptance/bundle/dms/serialized-plan/out.test.toml delete mode 100644 acceptance/bundle/dms/serialized-plan/output.txt delete mode 100644 acceptance/bundle/dms/serialized-plan/script diff --git a/acceptance/bundle/dms/serialized-plan/databricks.yml.tmpl b/acceptance/bundle/dms/serialized-plan/databricks.yml.tmpl deleted file mode 100644 index 01d891c3880..00000000000 --- a/acceptance/bundle/dms/serialized-plan/databricks.yml.tmpl +++ /dev/null @@ -1,9 +0,0 @@ -bundle: - name: dms-serialized-plan-$UNIQUE_NAME -experimental: - record_deployment_history: true - -resources: - jobs: - foo: - name: foo diff --git a/acceptance/bundle/dms/serialized-plan/out.test.toml b/acceptance/bundle/dms/serialized-plan/out.test.toml deleted file mode 100644 index 23c07f70dca..00000000000 --- a/acceptance/bundle/dms/serialized-plan/out.test.toml +++ /dev/null @@ -1,3 +0,0 @@ -Cloud = true -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -EnvMatrix.DMS = ["true"] diff --git a/acceptance/bundle/dms/serialized-plan/output.txt b/acceptance/bundle/dms/serialized-plan/output.txt deleted file mode 100644 index fc84deb9f4c..00000000000 --- a/acceptance/bundle/dms/serialized-plan/output.txt +++ /dev/null @@ -1,41 +0,0 @@ - -=== Deploy once to create the deployment (records version 1) ->>> [CLI] bundle deploy -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-serialized-plan-[UNIQUE_NAME]/default/files... -Created jobs.foo -Files: 5 uploaded, 0 deleted -Resources: 1 created, 0 changed, 0 deleted, 0 unchanged - -=== A saved plan carries the deployment id and the versions it targets (last 1, next 2) ->>> [CLI] bundle plan -o json - ->>> jq {deployment_id, last_version_id, next_version_id} tmp.plan.json -{ - "deployment_id": "[NUMID]", - "last_version_id": "1", - "next_version_id": "2" -} - -=== Change the job and deploy again: advances the deployment to version 2 ->>> update_file.py databricks.yml name: foo name: foo-v2 - ->>> [CLI] bundle deploy -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-serialized-plan-[UNIQUE_NAME]/default/files... -Updated jobs.foo -Files: 4 uploaded, 0 deleted -Resources: 0 created, 1 changed, 0 deleted, 0 unchanged - -=== Replaying the earlier plan is rejected: it predates the deployment's current version ->>> musterr [CLI] bundle deploy --plan tmp.plan.json -Error: this plan predates the deployment's current version 2; run 'bundle plan' again - - -=== Only versions 1 and 2 were recorded; the rejected replay created none ->>> print_requests.py --dms //api/2.0/bundle --oneline -{"method": "POST", "path": "/api/2.0/bundle/deployments", "body": {"display_name": "dms-serialized-plan-[UNIQUE_NAME]", "initial_parent_path": "/Workspace/Users/[USERNAME]/.bundle/dms-serialized-plan-[UNIQUE_NAME]/default/state", "target_name": "default", "workspace_info": {"file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-serialized-plan-[UNIQUE_NAME]/default/files", "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-serialized-plan-[UNIQUE_NAME]/default"}}} -{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": {"version_id": "1"}, "body": {"cli_version": "[CLI_VERSION]", "version_type": "VERSION_TYPE_DEPLOY", "operations": [{"resource_key": "jobs.foo", "action_type": "OPERATION_ACTION_TYPE_CREATE"}]}} -{"method": "PATCH", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.foo", "q": {"update_mask": "state,error_message,resource_id,status"}, "body": {"error_message": "", "resource_id": "[NUMID]", "sequence_id": "0", "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-serialized-plan-[UNIQUE_NAME]/default/state/metadata.json\",\"version_id\":\"1\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"foo\",\"queue\":{\"enabled\":true}}}", "status": "OPERATION_STATUS_SUCCEEDED"}} -{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/complete", "body": {"completion_reason": "VERSION_COMPLETE_SUCCESS"}} -{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": {"version_id": "2"}, "body": {"cli_version": "[CLI_VERSION]", "version_type": "VERSION_TYPE_DEPLOY", "previous_version_id": "1", "operations": [{"resource_key": "jobs.foo", "action_type": "OPERATION_ACTION_TYPE_UPDATE"}]}} -{"method": "PATCH", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations/jobs.foo", "q": {"update_mask": "state,error_message,resource_id,status"}, "body": {"error_message": "", "resource_id": "[NUMID]", "sequence_id": "0", "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-serialized-plan-[UNIQUE_NAME]/default/state/metadata.json\",\"version_id\":\"2\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"foo-v2\",\"queue\":{\"enabled\":true}}}", "status": "OPERATION_STATUS_SUCCEEDED"}} -{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/complete", "body": {"completion_reason": "VERSION_COMPLETE_SUCCESS"}} diff --git a/acceptance/bundle/dms/serialized-plan/script b/acceptance/bundle/dms/serialized-plan/script deleted file mode 100644 index 0800e1a8265..00000000000 --- a/acceptance/bundle/dms/serialized-plan/script +++ /dev/null @@ -1,20 +0,0 @@ -envsubst < databricks.yml.tmpl > databricks.yml - -title "Deploy once to create the deployment (records version 1)" -trace $CLI bundle deploy - -title "A saved plan carries the deployment id and the versions it targets (last 1, next 2)" -trace $CLI bundle plan -o json > tmp.plan.json -trace jq '{deployment_id, last_version_id, next_version_id}' tmp.plan.json - -title "Change the job and deploy again: advances the deployment to version 2" -trace update_file.py databricks.yml "name: foo" "name: foo-v2" -trace $CLI bundle deploy - -title "Replaying the earlier plan is rejected: it predates the deployment's current version" -trace musterr $CLI bundle deploy --plan tmp.plan.json - -title "Only versions 1 and 2 were recorded; the rejected replay created none" -trace print_requests.py --dms //api/2.0/bundle --oneline - -rm -f tmp.plan.json diff --git a/acceptance/bundle/dms/stale-plan/output.txt b/acceptance/bundle/dms/stale-plan/output.txt index 2ce3b598d1c..f39b3049697 100644 --- a/acceptance/bundle/dms/stale-plan/output.txt +++ b/acceptance/bundle/dms/stale-plan/output.txt @@ -8,6 +8,14 @@ Created jobs.foo Files: 6 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged +=== A saved plan carries the deployment id and the versions it targets (last 1, next 2) +>>> [CLI] bundle plan -o json +{ + "deployment_id": "[NUMID]", + "last_version_id": "1", + "next_version_id": "2" +} + === Replay: re-deploying the same plan is rejected - it predates the recorded version >>> musterr [CLI] bundle deploy --plan tmp.plan.json Error: this plan predates the deployment's current version 1; run 'bundle plan' again diff --git a/acceptance/bundle/dms/stale-plan/script b/acceptance/bundle/dms/stale-plan/script index 1f9e792f30f..09724d7d545 100644 --- a/acceptance/bundle/dms/stale-plan/script +++ b/acceptance/bundle/dms/stale-plan/script @@ -4,6 +4,9 @@ title "Save a plan, then deploy it: records version 1" trace $CLI bundle plan -o json > tmp.plan.json trace $CLI bundle deploy --plan tmp.plan.json +title "A saved plan carries the deployment id and the versions it targets (last 1, next 2)" +trace $CLI bundle plan -o json | jq '{deployment_id, last_version_id, next_version_id}' + title "Replay: re-deploying the same plan is rejected - it predates the recorded version" trace musterr $CLI bundle deploy --plan tmp.plan.json From 632b8ea121425052cae68a71173324266fbeda33 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Fri, 4 Sep 2026 08:27:05 +0000 Subject: [PATCH 08/46] acceptance/bin: centralize recorded-state reconstruction in dms_resources.get_recorded_state --- acceptance/bin/dms_resources.py | 12 ++++++++++++ acceptance/bin/print_state.py | 10 ++-------- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/acceptance/bin/dms_resources.py b/acceptance/bin/dms_resources.py index 25e469c5bad..77b1da5b23f 100755 --- a/acceptance/bin/dms_resources.py +++ b/acceptance/bin/dms_resources.py @@ -103,3 +103,15 @@ def get_resources(target): page_token = listed.get("next_page_token") if not page_token: return result + + +def get_recorded_state(target): + """The recorded resources in the on-disk state file's `state` shape (resources. -> + {__id__, state, depends_on}), so a recording run prints the same shape as a non-recording one.""" + state = {} + for key, value in sorted(get_resources(target).items()): + entry = {"__id__": value["id"], "state": value["state"]} + if value["depends_on"]: + entry["depends_on"] = value["depends_on"] + state[f"resources.{key}"] = entry + return state diff --git a/acceptance/bin/print_state.py b/acceptance/bin/print_state.py index 2959a76f81b..f63189c02b3 100755 --- a/acceptance/bin/print_state.py +++ b/acceptance/bin/print_state.py @@ -65,16 +65,10 @@ def print_recorded_state(filename, target): printing it raw would show an empty state and differ from the same test's non-recording run. """ # Imported here rather than at module level: dms_resources reads get_state_file from this module. - from dms_resources import get_resources + from dms_resources import get_recorded_state data = json.loads(open(filename).read()) - state = {} - for key, value in sorted(get_resources(target).items()): - entry = {"__id__": value["id"], "state": value["state"]} - if value["depends_on"]: - entry["depends_on"] = value["depends_on"] - state[f"resources.{key}"] = entry - data["state"] = state + data["state"] = get_recorded_state(target) print(json.dumps(data, indent=1)) From 1b0b45f15cec3f66a1bd1d8470ee02e499a924de Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Fri, 4 Sep 2026 08:59:40 +0000 Subject: [PATCH 09/46] acceptance/dms: record-failure uses a real bogus node type + a scoped stub instead of fault.py --- .../bundle/dms/record-failure/databricks.yml.tmpl | 8 ++++++++ acceptance/bundle/dms/record-failure/out.test.toml | 2 +- acceptance/bundle/dms/record-failure/output.txt | 8 +++----- acceptance/bundle/dms/record-failure/script | 1 - acceptance/bundle/dms/record-failure/test.toml | 14 +++++++++----- 5 files changed, 21 insertions(+), 12 deletions(-) diff --git a/acceptance/bundle/dms/record-failure/databricks.yml.tmpl b/acceptance/bundle/dms/record-failure/databricks.yml.tmpl index 87f0eb4d1a9..f1aeba83d9a 100644 --- a/acceptance/bundle/dms/record-failure/databricks.yml.tmpl +++ b/acceptance/bundle/dms/record-failure/databricks.yml.tmpl @@ -7,3 +7,11 @@ resources: jobs: doomed: name: doomed + tasks: + - task_key: doomed + new_cluster: + node_type_id: nonexistent-node-type + spark_version: 15.4.x-scala2.12 + num_workers: 1 + notebook_task: + notebook_path: /nonexistent diff --git a/acceptance/bundle/dms/record-failure/out.test.toml b/acceptance/bundle/dms/record-failure/out.test.toml index 9921e91a794..23c07f70dca 100644 --- a/acceptance/bundle/dms/record-failure/out.test.toml +++ b/acceptance/bundle/dms/record-failure/out.test.toml @@ -1,3 +1,3 @@ -Cloud = false +Cloud = true EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] EnvMatrix.DMS = ["true"] diff --git a/acceptance/bundle/dms/record-failure/output.txt b/acceptance/bundle/dms/record-failure/output.txt index 479acca7a84..3556ba54b95 100644 --- a/acceptance/bundle/dms/record-failure/output.txt +++ b/acceptance/bundle/dms/record-failure/output.txt @@ -1,15 +1,13 @@ === A resource that fails to apply is recorded as a failed operation carrying the error, so the deployment history says why rather than omitting the resource ->>> fault.py POST /api/2.2/jobs/create 400 0 1 INVALID_PARAMETER_VALUE - >>> musterr [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-record-failure-[UNIQUE_NAME]/default/files... -Error: cannot create resources.jobs.doomed: Fault injected by test. (400 INVALID_PARAMETER_VALUE) +Error: cannot create resources.jobs.doomed: Cluster validation failed: node type nonexistent-node-type is not supported (400 INVALID_PARAMETER_VALUE) Endpoint: POST [DATABRICKS_URL]/api/2.2/jobs/create HTTP Status: 400 Bad Request API error_code: INVALID_PARAMETER_VALUE -API message: Fault injected by test. +API message: Cluster validation failed: node type nonexistent-node-type is not supported Files: 6 uploaded, 0 deleted @@ -51,7 +49,7 @@ Files: 6 uploaded, 0 deleted "update_mask": "error_message,status" }, "body": { - "error_message": "Fault injected by test. (400 INVALID_PARAMETER_VALUE)", + "error_message": "Cluster validation failed: node type nonexistent-node-type is not supported (400 INVALID_PARAMETER_VALUE)", "sequence_id": "0", "status": "OPERATION_STATUS_FAILED" } diff --git a/acceptance/bundle/dms/record-failure/script b/acceptance/bundle/dms/record-failure/script index bc0947a1d91..8b629dfd6be 100644 --- a/acceptance/bundle/dms/record-failure/script +++ b/acceptance/bundle/dms/record-failure/script @@ -1,7 +1,6 @@ envsubst < databricks.yml.tmpl > databricks.yml title "A resource that fails to apply is recorded as a failed operation carrying the error, so the deployment history says why rather than omitting the resource" -trace fault.py "POST /api/2.2/jobs/create" 400 0 1 INVALID_PARAMETER_VALUE trace musterr $CLI bundle deploy trace print_requests.py --dms //api/2.0/bundle diff --git a/acceptance/bundle/dms/record-failure/test.toml b/acceptance/bundle/dms/record-failure/test.toml index 8a850ba07c5..e4b49d83c1c 100644 --- a/acceptance/bundle/dms/record-failure/test.toml +++ b/acceptance/bundle/dms/record-failure/test.toml @@ -1,5 +1,9 @@ -# Local only: the failure is injected, and fault.py registers its rule on the fake, which -# a cloud run does not have. Simulating it for real instead - a catalog or principal that -# does not exist - does not work either: the fake accepts both, so the case would stop -# failing locally. Making it portable needs the fake to model that existence. -Cloud = false +# The job's cluster names a node type that does not exist, so the create fails for real: on cloud +# the Jobs API rejects it. The local fake is lax and would accept it, so this one test stubs the +# create to return the same error - scoped here, not the shared fake, so no other test is affected. +Cloud = true + +[[Server]] +Pattern = "POST /api/2.2/jobs/create" +Response.StatusCode = 400 +Response.Body = '''{"error_code": "INVALID_PARAMETER_VALUE", "message": "Cluster validation failed: node type nonexistent-node-type is not supported"}''' From 09f61c3d98863ed3d802c9e656eb8331b2c40a88 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Fri, 4 Sep 2026 09:30:01 +0000 Subject: [PATCH 10/46] libs/dms: gofmt the rawClient doc comment --- libs/dms/client.go | 1 + 1 file changed, 1 insertion(+) diff --git a/libs/dms/client.go b/libs/dms/client.go index fc1f482c6b2..611bdf9c11b 100644 --- a/libs/dms/client.go +++ b/libs/dms/client.go @@ -165,6 +165,7 @@ type operationResponse struct { // its operations through it. // - UpdateOperation: the SDK types sequence_id as int64, but the service sends and expects it // as a JSON string. +// // CreateDeployment, DeleteDeployment, CompleteVersion and ListResources go through the SDK (see Client). type rawClient struct { client *client.DatabricksClient From d42b08e82806a3ebc0a644267223208dfb9d7d94 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Fri, 4 Sep 2026 10:58:50 +0000 Subject: [PATCH 11/46] bundle: make the recorded-state marker the source of truth for DMS; Open reads the workspace client from context Co-authored-by: Isaac --- .../bundle/dms/record-failure/output.txt | 6 +- .../bundle/dms/record-failure/test.toml | 7 +- bundle/configsync/diff.go | 2 +- bundle/configsync/variables.go | 2 +- bundle/direct/bind.go | 12 +-- bundle/direct/dstate/state.go | 94 ++++++++++--------- bundle/direct/dstate/state_test.go | 36 +++---- bundle/statemgmt/direct_migration.go | 2 +- cmd/bundle/generate/dashboard.go | 2 +- cmd/bundle/generate/genie_space.go | 2 +- cmd/bundle/utils/process.go | 34 +++---- 11 files changed, 103 insertions(+), 96 deletions(-) diff --git a/acceptance/bundle/dms/record-failure/output.txt b/acceptance/bundle/dms/record-failure/output.txt index 3556ba54b95..819004844d7 100644 --- a/acceptance/bundle/dms/record-failure/output.txt +++ b/acceptance/bundle/dms/record-failure/output.txt @@ -2,12 +2,12 @@ === A resource that fails to apply is recorded as a failed operation carrying the error, so the deployment history says why rather than omitting the resource >>> musterr [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-record-failure-[UNIQUE_NAME]/default/files... -Error: cannot create resources.jobs.doomed: Cluster validation failed: node type nonexistent-node-type is not supported (400 INVALID_PARAMETER_VALUE) +Error: cannot create resources.jobs.doomed: Node type nonexistent-node-type is not supported. Supported node types: [NODE_TYPES] (400 INVALID_PARAMETER_VALUE) Endpoint: POST [DATABRICKS_URL]/api/2.2/jobs/create HTTP Status: 400 Bad Request API error_code: INVALID_PARAMETER_VALUE -API message: Cluster validation failed: node type nonexistent-node-type is not supported +API message: Node type nonexistent-node-type is not supported. Supported node types: [NODE_TYPES] Files: 6 uploaded, 0 deleted @@ -49,7 +49,7 @@ Files: 6 uploaded, 0 deleted "update_mask": "error_message,status" }, "body": { - "error_message": "Cluster validation failed: node type nonexistent-node-type is not supported (400 INVALID_PARAMETER_VALUE)", + "error_message": "Node type nonexistent-node-type is not supported. Supported node types: [NODE_TYPES] (400 INVALID_PARAMETER_VALUE)", "sequence_id": "0", "status": "OPERATION_STATUS_FAILED" } diff --git a/acceptance/bundle/dms/record-failure/test.toml b/acceptance/bundle/dms/record-failure/test.toml index e4b49d83c1c..49f63c399ac 100644 --- a/acceptance/bundle/dms/record-failure/test.toml +++ b/acceptance/bundle/dms/record-failure/test.toml @@ -3,7 +3,12 @@ # create to return the same error - scoped here, not the shared fake, so no other test is affected. Cloud = true +# The Jobs API error lists every supported node type, and the set grows over time; mask it. +[[Repls]] +Old = 'Supported node types: [^(\n]*[^ (\n]' +New = 'Supported node types: [NODE_TYPES]' + [[Server]] Pattern = "POST /api/2.2/jobs/create" Response.StatusCode = 400 -Response.Body = '''{"error_code": "INVALID_PARAMETER_VALUE", "message": "Cluster validation failed: node type nonexistent-node-type is not supported"}''' +Response.Body = '''{"error_code": "INVALID_PARAMETER_VALUE", "message": "Node type nonexistent-node-type is not supported. Supported node types: r3.xlarge, m5.large"}''' diff --git a/bundle/configsync/diff.go b/bundle/configsync/diff.go index 40511ae1dcd..f5afb8d5b1a 100644 --- a/bundle/configsync/diff.go +++ b/bundle/configsync/diff.go @@ -149,7 +149,7 @@ func OpenDeploymentState(ctx context.Context, b *bundle.Bundle, engine engine.En deployBundle := &direct.DeploymentBundle{} _, statePath := b.StateFilenameConfigSnapshot(ctx) - if err := deployBundle.StateDB.Open(ctx, statePath, dstate.WithRecovery(true), dstate.WithWrite(false), nil, dstate.WithDeploymentHistory(false), ""); err != nil { + if err := deployBundle.StateDB.Open(ctx, statePath, dstate.WithRecovery(true), dstate.WithWrite(false), dstate.WithDeploymentHistory(false), ""); err != nil { return nil, fmt.Errorf("failed to open state: %w", err) } return deployBundle, nil diff --git a/bundle/configsync/variables.go b/bundle/configsync/variables.go index c16655a3518..462c0b4c71f 100644 --- a/bundle/configsync/variables.go +++ b/bundle/configsync/variables.go @@ -156,7 +156,7 @@ func resourceIDLookup(ctx context.Context, b *bundle.Bundle) func(string) string } _, statePath := b.StateFilenameConfigSnapshot(ctx) db := &dstate.DeploymentState{} - if err := db.Open(ctx, statePath, dstate.WithRecovery(false), dstate.WithWrite(false), nil, dstate.WithDeploymentHistory(false), ""); err != nil { + if err := db.Open(ctx, statePath, dstate.WithRecovery(false), dstate.WithWrite(false), dstate.WithDeploymentHistory(false), ""); err != nil { log.Debugf(ctx, "variable restoration: failed to open state DB at %s: %v", statePath, err) return nil } diff --git a/bundle/direct/bind.go b/bundle/direct/bind.go index 9c4ab260e03..a65cacd7eb4 100644 --- a/bundle/direct/bind.go +++ b/bundle/direct/bind.go @@ -64,7 +64,7 @@ func (b *DeploymentBundle) Bind(ctx context.Context, client *databricks.Workspac // The state is opened without a DMS client, so the writes below record nothing; // phases.Bind and phases.Unbind refuse to run at all when recording is enabled. var checkStateDB dstate.DeploymentState - if err := checkStateDB.Open(ctx, statePath, dstate.WithRecovery(true), dstate.WithWrite(false), nil, dstate.WithDeploymentHistory(false), ""); err == nil { + if err := checkStateDB.Open(ctx, statePath, dstate.WithRecovery(true), dstate.WithWrite(false), dstate.WithDeploymentHistory(false), ""); err == nil { existingID := checkStateDB.GetResourceID(resourceKey) if _, err := checkStateDB.Finalize(ctx); err != nil { log.Warnf(ctx, "failed to finalize state: %v", err) @@ -88,7 +88,7 @@ func (b *DeploymentBundle) Bind(ctx context.Context, client *databricks.Workspac } // Open temp state - err := b.StateDB.Open(ctx, tmpStatePath, dstate.WithRecovery(false), dstate.WithWrite(true), nil, dstate.WithDeploymentHistory(false), "") + err := b.StateDB.Open(ctx, tmpStatePath, dstate.WithRecovery(false), dstate.WithWrite(true), dstate.WithDeploymentHistory(false), "") if err != nil { os.Remove(tmpStatePath) return nil, err @@ -111,7 +111,7 @@ func (b *DeploymentBundle) Bind(ctx context.Context, client *databricks.Workspac log.Infof(ctx, "Bound %s to id=%s (in temp state)", resourceKey, resourceID) // First plan + update: populate state with resolved config - err = b.StateDB.Open(ctx, tmpStatePath, dstate.WithRecovery(true), dstate.WithWrite(false), nil, dstate.WithDeploymentHistory(false), "") + err = b.StateDB.Open(ctx, tmpStatePath, dstate.WithRecovery(true), dstate.WithWrite(false), dstate.WithDeploymentHistory(false), "") if err != nil { os.Remove(tmpStatePath) return nil, err @@ -147,7 +147,7 @@ func (b *DeploymentBundle) Bind(ctx context.Context, client *databricks.Workspac } } - err = b.StateDB.Open(ctx, tmpStatePath, dstate.WithRecovery(true), dstate.WithWrite(true), nil, dstate.WithDeploymentHistory(false), "") + err = b.StateDB.Open(ctx, tmpStatePath, dstate.WithRecovery(true), dstate.WithWrite(true), dstate.WithDeploymentHistory(false), "") if err != nil { os.Remove(tmpStatePath) return nil, err @@ -167,7 +167,7 @@ func (b *DeploymentBundle) Bind(ctx context.Context, client *databricks.Workspac } // Second plan: this is the plan to present to the user (change between remote resource and config) - err = b.StateDB.Open(ctx, tmpStatePath, dstate.WithRecovery(true), dstate.WithWrite(false), nil, dstate.WithDeploymentHistory(false), "") + err = b.StateDB.Open(ctx, tmpStatePath, dstate.WithRecovery(true), dstate.WithWrite(false), dstate.WithDeploymentHistory(false), "") if err != nil { os.Remove(tmpStatePath) return nil, err @@ -217,7 +217,7 @@ func (result *BindResult) Cancel() { // Unbind removes a resource from direct engine state without deleting // the workspace resource. Also removes associated permissions/grants entries. func (b *DeploymentBundle) Unbind(ctx context.Context, statePath, resourceKey string) error { - err := b.StateDB.Open(ctx, statePath, dstate.WithRecovery(true), dstate.WithWrite(true), nil, dstate.WithDeploymentHistory(false), "") + err := b.StateDB.Open(ctx, statePath, dstate.WithRecovery(true), dstate.WithWrite(true), dstate.WithDeploymentHistory(false), "") if err != nil { return err } diff --git a/bundle/direct/dstate/state.go b/bundle/direct/dstate/state.go index 4e2f8c03344..6a9d1c4d2ee 100644 --- a/bundle/direct/dstate/state.go +++ b/bundle/direct/dstate/state.go @@ -18,9 +18,9 @@ import ( "github.com/databricks/cli/bundle/deployplan" "github.com/databricks/cli/bundle/statemgmt/resourcestate" "github.com/databricks/cli/internal/build" + "github.com/databricks/cli/libs/cmdctx" "github.com/databricks/cli/libs/dms" "github.com/databricks/cli/libs/log" - "github.com/databricks/databricks-sdk-go" "github.com/google/uuid" ) @@ -364,7 +364,7 @@ type ( // recorded deploy); lineage and serial still come from the file, since that is what the write path // increments. Open only reads through the client - StartRecording installs the write path once a // version exists. -func (db *DeploymentState) Open(ctx context.Context, path string, withRecovery WithRecovery, withWrite WithWrite, wsClient *databricks.WorkspaceClient, withDeploymentHistory WithDeploymentHistory, dmsDeploymentID string) error { +func (db *DeploymentState) Open(ctx context.Context, path string, withRecovery WithRecovery, withWrite WithWrite, withDeploymentHistory WithDeploymentHistory, dmsDeploymentID string) error { db.mu.Lock() defer db.mu.Unlock() @@ -372,7 +372,7 @@ func (db *DeploymentState) Open(ctx context.Context, path string, withRecovery W panic(fmt.Sprintf("state already opened: %v, cannot open %v", db.Path, path)) } - err := db.unlockedOpen(ctx, path, withRecovery, withWrite, wsClient, withDeploymentHistory, dmsDeploymentID) + err := db.unlockedOpen(ctx, path, withRecovery, withWrite, withDeploymentHistory, dmsDeploymentID) if err != nil { // A failed open must leave the receiver closed. unlockedOpen assigns // db.Path before every fallible step, so without this the receiver stays @@ -394,20 +394,11 @@ func (db *DeploymentState) reset() { db.stateIDs = nil } -func (db *DeploymentState) unlockedOpen(ctx context.Context, path string, withRecovery WithRecovery, withWrite WithWrite, wsClient *databricks.WorkspaceClient, withDeploymentHistory WithDeploymentHistory, dmsDeploymentID string) error { +func (db *DeploymentState) unlockedOpen(ctx context.Context, path string, withRecovery WithRecovery, withWrite WithWrite, withDeploymentHistory WithDeploymentHistory, dmsDeploymentID string) error { db.Path = path - if withDeploymentHistory { - // The service is the source of truth for a recorded deployment, so a leftover WAL is - // discarded below rather than replayed: force recovery off regardless of what was asked. - withRecovery = false - client, err := dms.NewClient(wsClient) - if err != nil { - return err - } - db.dmsClient = client - } - + // The state file is the source of truth for whether this deployment records history: read it + // first, then reconcile the config against the marker it carries. data, err := os.ReadFile(db.Path) if err != nil { if errors.Is(err, fs.ErrNotExist) { @@ -421,6 +412,16 @@ func (db *DeploymentState) unlockedOpen(ctx context.Context, path string, withRe } } + if err := migrateState(&db.Data); err != nil { + return fmt.Errorf("migrating state %s: %w", path, err) + } + + if err := checkStateFeatures(db.Data.Features); err != nil { + return err + } + + recording := bool(withDeploymentHistory) + db.stateIDs = make(map[string]string) for key, entry := range db.Data.State { db.stateIDs[key] = entry.ID @@ -433,56 +434,57 @@ func (db *DeploymentState) unlockedOpen(ctx context.Context, path string, withRe return fmt.Errorf("failed to stat WAL file %s: %w", walPath, statErr) } - // A recorded deployment gets its resources from the service (applyDMSState below), so the WAL - - // a local log of an uncommitted deploy - is not merged. A leftover from a crash or a declined - // deploy is removed so the next write's exclusive create of the WAL does not fail. Without + // A recording open discards a leftover WAL - the service owns the resources, so a local log from + // a crash or a declined deploy is irrelevant, even when recording is refused below. Without // recording, replay the WAL as usual. if walExists { - if withDeploymentHistory { + switch { + case recording: if err := os.Remove(walPath); err != nil { return fmt.Errorf("removing WAL file %s: %w", walPath, err) } - } else if withRecovery { + case bool(withRecovery): if err := db.replayWAL(ctx); err != nil { return fmt.Errorf("reading state from %s: %w", path, err) } - } else { + default: return fmt.Errorf("unexpected WAL file found at %s", walPath) } } - if err := migrateState(&db.Data); err != nil { - return fmt.Errorf("migrating state %s: %w", path, err) - } - - if err := checkStateFeatures(db.Data.Features); err != nil { - return err - } - - if withDeploymentHistory { - // The service is the source of truth for a recorded deployment; local state is irrelevant. - // This state was pulled from the workspace, so it reflects the remote and not just a local - // cache, and a leftover WAL was discarded above rather than replayed. It records the feature - // once recording owns it, so a recorded deployment carries no resources here (a tombstone) - // and applyDMSState below loads the service's. A state that still tracks resources without - // the feature is a deployment made before recording: treating the service as authoritative - // would create those resources a second time, so refuse it (adopting one is a followup). - if _, recorded := db.Data.Features[featureRecordDeploymentHistory]; !recorded && len(db.Data.State) > 0 { - return errors.New(`this deployment already exists and is not recorded with the deployment metadata service, so it cannot be recorded without redeploying its resources + // Reconcile the config against the state. A brand-new deployment is where recording begins, so + // the config bootstraps the marker there. On an existing deployment the state is authoritative: + // the config can neither start recording one that was not (its resources would be created a + // second time) nor stop recording one that is (the service still holds it). + _, recorded := db.Data.Features[featureRecordDeploymentHistory] + switch { + case recording && !recorded && len(db.Data.State) == 0: + if db.Data.Features == nil { + db.Data.Features = make(map[string]struct{}, 1) + } + db.Data.Features[featureRecordDeploymentHistory] = struct{}{} + recorded = true + case recording && !recorded: + return errors.New(`this deployment already exists and is not recorded with the deployment metadata service, so it cannot be recorded without redeploying its resources To record this bundle's history, start it over as a new deployment: 1. remove experimental.record_deployment_history from your bundle configuration 2. run "databricks bundle destroy" to delete the existing resources 3. add experimental.record_deployment_history back and deploy again`) - } + case !recording && recorded: + return errors.New(`unsetting experimental.record_deployment_history is not supported - // Mark the state as depending on the service so this CLI keeps recording it (see - // RequiresDeploymentHistory). unlockedSave then writes the header alone. Bumping the state - // version so older CLIs refuse a recorded state is a followup. - if db.Data.Features == nil { - db.Data.Features = make(map[string]struct{}, 1) +This deployment's resources are recorded with the deployment metadata service. Set experimental.record_deployment_history: true to deploy or destroy this bundle`) + } + + if recorded { + // The service is the source of truth for a recorded deployment; the file is a tombstone + // carrying only the marker, and applyDMSState loads the resources the service holds. + client, err := dms.NewClient(cmdctx.WorkspaceClient(ctx)) + if err != nil { + return err } - db.Data.Features[featureRecordDeploymentHistory] = struct{}{} + db.dmsClient = client if dmsDeploymentID != "" { resources, err := db.dmsClient.ListResources(ctx, dmsDeploymentID) diff --git a/bundle/direct/dstate/state_test.go b/bundle/direct/dstate/state_test.go index 8ccd6277ecb..f7d605e39f8 100644 --- a/bundle/direct/dstate/state_test.go +++ b/bundle/direct/dstate/state_test.go @@ -21,14 +21,14 @@ func TestOpenSaveFinalizeRoundTrip(t *testing.T) { path := filepath.Join(t.TempDir(), "state.json") var db DeploymentState - require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil, WithDeploymentHistory(false), "")) + require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), WithDeploymentHistory(false), "")) require.NoError(t, db.SaveState(t.Context(), "jobs.my_job", "123", map[string]string{"key": "val"}, nil)) mustFinalize(t, &db) // Re-open and verify persisted data. var db2 DeploymentState - require.NoError(t, db2.Open(t.Context(), path, WithRecovery(false), WithWrite(false), nil, WithDeploymentHistory(false), "")) + require.NoError(t, db2.Open(t.Context(), path, WithRecovery(false), WithWrite(false), WithDeploymentHistory(false), "")) assert.Equal(t, 1, db2.Data.Serial) assert.Equal(t, "123", db2.GetResourceID("jobs.my_job")) mustFinalize(t, &db2) @@ -38,7 +38,7 @@ func TestFinalizeWithNoEntriesDoesNotWriteStateFile(t *testing.T) { path := filepath.Join(t.TempDir(), "state.json") var db DeploymentState - require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil, WithDeploymentHistory(false), "")) + require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), WithDeploymentHistory(false), "")) mustFinalize(t, &db) _, err := os.Stat(path) @@ -94,10 +94,10 @@ func TestPanicOnDoubleOpen(t *testing.T) { path := filepath.Join(t.TempDir(), "state.json") var db DeploymentState - require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil, WithDeploymentHistory(false), "")) + require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), WithDeploymentHistory(false), "")) assert.Panics(t, func() { - _ = db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil, WithDeploymentHistory(false), "") + _ = db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), WithDeploymentHistory(false), "") }) mustFinalize(t, &db) } @@ -115,12 +115,12 @@ func TestCLIVersionRecordsLastWriter(t *testing.T) { require.NoError(t, os.WriteFile(path, []byte(seed), 0o600)) var db DeploymentState - require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil, WithDeploymentHistory(false), "")) + require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), WithDeploymentHistory(false), "")) require.NoError(t, db.SaveState(t.Context(), "resources.jobs.my_job", "123", map[string]string{"k": "v"}, nil)) mustFinalize(t, &db) var reopened DeploymentState - require.NoError(t, reopened.Open(t.Context(), path, WithRecovery(false), WithWrite(false), nil, WithDeploymentHistory(false), "")) + require.NoError(t, reopened.Open(t.Context(), path, WithRecovery(false), WithWrite(false), WithDeploymentHistory(false), "")) assert.Equal(t, build.GetInfo().Version, reopened.Data.CLIVersion) assert.Equal(t, 2, reopened.Data.Serial) mustFinalize(t, &reopened) @@ -142,7 +142,7 @@ func TestHeaderOnlyWALDoesNotUpdateCLIVersion(t *testing.T) { require.NoError(t, os.WriteFile(walPath, append(headerLine, '\n'), 0o600)) var recovered DeploymentState - require.NoError(t, recovered.Open(t.Context(), path, WithRecovery(true), WithWrite(false), nil, WithDeploymentHistory(false), "")) + require.NoError(t, recovered.Open(t.Context(), path, WithRecovery(true), WithWrite(false), WithDeploymentHistory(false), "")) assert.Equal(t, "0.1.2", recovered.Data.CLIVersion, "a header-only WAL wrote no state, so the version must not move") assert.Equal(t, 1, recovered.Data.Serial) mustFinalize(t, &recovered) @@ -154,12 +154,12 @@ func TestHeaderOnlyWALRecoveryDoesNotAdvanceSerial(t *testing.T) { // Commit serial 1 with one resource. var db DeploymentState - require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil, WithDeploymentHistory(false), "")) + require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), WithDeploymentHistory(false), "")) require.NoError(t, db.SaveState(t.Context(), "jobs.my_job", "123", map[string]string{}, nil)) mustFinalize(t, &db) var committed DeploymentState - require.NoError(t, committed.Open(t.Context(), path, WithRecovery(false), WithWrite(false), nil, WithDeploymentHistory(false), "")) + require.NoError(t, committed.Open(t.Context(), path, WithRecovery(false), WithWrite(false), WithDeploymentHistory(false), "")) lineage := committed.Data.Lineage require.Equal(t, 1, committed.Data.Serial) mustFinalize(t, &committed) @@ -175,7 +175,7 @@ func TestHeaderOnlyWALRecoveryDoesNotAdvanceSerial(t *testing.T) { require.NoError(t, os.WriteFile(walPath, append(headerLine, '\n'), 0o600)) var recovered DeploymentState - require.NoError(t, recovered.Open(t.Context(), path, WithRecovery(true), WithWrite(false), nil, WithDeploymentHistory(false), "")) + require.NoError(t, recovered.Open(t.Context(), path, WithRecovery(true), WithWrite(false), WithDeploymentHistory(false), "")) assert.Equal(t, 1, recovered.Data.Serial) assert.Equal(t, "123", recovered.GetResourceID("jobs.my_job")) assert.NoFileExists(t, walPath) @@ -186,17 +186,17 @@ func TestDeleteState(t *testing.T) { path := filepath.Join(t.TempDir(), "state.json") var db DeploymentState - require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil, WithDeploymentHistory(false), "")) + require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), WithDeploymentHistory(false), "")) require.NoError(t, db.SaveState(t.Context(), "jobs.my_job", "123", map[string]string{}, nil)) mustFinalize(t, &db) var db2 DeploymentState - require.NoError(t, db2.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil, WithDeploymentHistory(false), "")) + require.NoError(t, db2.Open(t.Context(), path, WithRecovery(true), WithWrite(true), WithDeploymentHistory(false), "")) require.NoError(t, db2.DeleteState(t.Context(), "jobs.my_job", false)) mustFinalize(t, &db2) var db3 DeploymentState - require.NoError(t, db3.Open(t.Context(), path, WithRecovery(false), WithWrite(false), nil, WithDeploymentHistory(false), "")) + require.NoError(t, db3.Open(t.Context(), path, WithRecovery(false), WithWrite(false), WithDeploymentHistory(false), "")) assert.Equal(t, 2, db3.Data.Serial) assert.Empty(t, db3.GetResourceID("jobs.my_job")) mustFinalize(t, &db3) @@ -208,7 +208,7 @@ func TestGetOrInitLineageReadableBeforeWriteAndPersisted(t *testing.T) { // Fresh state opened read-only, as the deploy does before planning: no // lineage yet. var db DeploymentState - require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(false), nil, WithDeploymentHistory(false), "")) + require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(false), WithDeploymentHistory(false), "")) require.Empty(t, db.Data.Lineage) // GetOrInitLineage initializes the lineage and makes it readable before any @@ -225,7 +225,7 @@ func TestGetOrInitLineageReadableBeforeWriteAndPersisted(t *testing.T) { // Re-open: the persisted lineage matches the one read before the write. var reopened DeploymentState - require.NoError(t, reopened.Open(t.Context(), path, WithRecovery(false), WithWrite(false), nil, WithDeploymentHistory(false), "")) + require.NoError(t, reopened.Open(t.Context(), path, WithRecovery(false), WithWrite(false), WithDeploymentHistory(false), "")) assert.Equal(t, lineage, reopened.Data.Lineage) mustFinalize(t, &reopened) } @@ -239,13 +239,13 @@ func TestOpenFailureLeavesStateClosed(t *testing.T) { require.NoError(t, os.WriteFile(path, []byte("{not json"), 0o600)) var db DeploymentState - require.Error(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil, WithDeploymentHistory(false), "")) + require.Error(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), WithDeploymentHistory(false), "")) assert.Empty(t, db.Path) // Once the state file is readable, the same receiver opens without panicking. seed := `{"state_version":2,"cli_version":"0.1.2","lineage":"test-lineage","serial":1,"state":{}}` require.NoError(t, os.WriteFile(path, []byte(seed), 0o600)) - require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), nil, WithDeploymentHistory(false), "")) + require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), WithDeploymentHistory(false), "")) assert.Equal(t, "test-lineage", db.Data.Lineage) mustFinalize(t, &db) } diff --git a/bundle/statemgmt/direct_migration.go b/bundle/statemgmt/direct_migration.go index 5d54e8702e5..f2140823b28 100644 --- a/bundle/statemgmt/direct_migration.go +++ b/bundle/statemgmt/direct_migration.go @@ -173,7 +173,7 @@ func checkPlanOnTempState(ctx context.Context, b *bundle.Bundle, tempStatePath s var planBundle direct.DeploymentBundle // This plan check does not record deployment history, so no DMS client or deployment id. - if err := planBundle.StateDB.Open(planCtx, tempStatePath, false, false, nil, dstate.WithDeploymentHistory(false), ""); err != nil { + if err := planBundle.StateDB.Open(planCtx, tempStatePath, false, false, dstate.WithDeploymentHistory(false), ""); err != nil { return fmt.Errorf("opening migrated state for plan check: %w", err) } diff --git a/cmd/bundle/generate/dashboard.go b/cmd/bundle/generate/dashboard.go index 14bc85777c1..6047655b412 100644 --- a/cmd/bundle/generate/dashboard.go +++ b/cmd/bundle/generate/dashboard.go @@ -404,7 +404,7 @@ func (d *dashboard) runForResource(ctx context.Context, b *bundle.Bundle) { var state statemgmt.ExportedResourcesMap if stateDesc.Engine.IsDirect() { _, localPath := b.StateFilenameDirect(ctx) - if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WithRecovery(true), dstate.WithWrite(false), nil, dstate.WithDeploymentHistory(false), ""); err != nil { + if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WithRecovery(true), dstate.WithWrite(false), dstate.WithDeploymentHistory(false), ""); err != nil { logdiag.LogError(ctx, err) return } diff --git a/cmd/bundle/generate/genie_space.go b/cmd/bundle/generate/genie_space.go index ea37ff50990..f87529b869a 100644 --- a/cmd/bundle/generate/genie_space.go +++ b/cmd/bundle/generate/genie_space.go @@ -322,7 +322,7 @@ func (g *genieSpace) runForResource(ctx context.Context, b *bundle.Bundle) { var state statemgmt.ExportedResourcesMap if stateDesc.Engine.IsDirect() { _, localPath := b.StateFilenameDirect(ctx) - if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WithRecovery(true), dstate.WithWrite(false), nil, dstate.WithDeploymentHistory(false), ""); err != nil { + if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WithRecovery(true), dstate.WithWrite(false), dstate.WithDeploymentHistory(false), ""); err != nil { logdiag.LogError(ctx, err) return } diff --git a/cmd/bundle/utils/process.go b/cmd/bundle/utils/process.go index 1d671fbc306..3074ba4272c 100644 --- a/cmd/bundle/utils/process.go +++ b/cmd/bundle/utils/process.go @@ -23,6 +23,7 @@ import ( "github.com/databricks/cli/bundle/statemgmt" "github.com/databricks/cli/cmd/root" "github.com/databricks/cli/internal/build" + "github.com/databricks/cli/libs/cmdctx" "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dms" @@ -283,25 +284,24 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle if logdiag.HasError(ctx) { return b, stateDesc, root.ErrAlreadyPrinted } - } - if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WithRecovery(true), dstate.WithWrite(false), b.WorkspaceClient(ctx), dstate.WithDeploymentHistory(b.ConfiguresDeploymentHistory(ctx)), dmsDeploymentID); err != nil { - logdiag.LogError(ctx, err) - return b, stateDesc, root.ErrAlreadyPrinted - } + // StateDB.Open builds the DMS client from the workspace client on the context. A bundle + // command resolves that client lazily through b, so it is not otherwise on the context. + if !cmdctx.HasWorkspaceClient(ctx) { + ctx = cmdctx.SetWorkspaceClient(ctx, b.WorkspaceClient(ctx)) + } + if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WithRecovery(false), dstate.WithWrite(false), dstate.WithDeploymentHistory(true), dmsDeploymentID); err != nil { + logdiag.LogError(ctx, err) + return b, stateDesc, root.ErrAlreadyPrinted + } - // Open built the DMS client from the workspace client when recording; hand it to the - // phases that create versions and deployments. - if b.ConfiguresDeploymentHistory(ctx) { + // Open built the DMS client from the workspace client on the context; hand it to the + // phases that create versions and deployments. b.DeploymentBundle.DmsApiClient = b.DeploymentBundle.StateDB.DmsClient() - } - - // The service holds this deployment, so it has to keep being recorded: deploying - // without recording would leave it describing resources that have moved on. - if !b.ConfiguresDeploymentHistory(ctx) && b.DeploymentBundle.StateDB.RequiresDeploymentHistory() { - logdiag.LogError(ctx, errors.New(`unsetting experimental.record_deployment_history is not supported - -This deployment's resources are recorded with the deployment metadata service. Set experimental.record_deployment_history: true to deploy or destroy this bundle`)) - return b, stateDesc, root.ErrAlreadyPrinted + } else { + if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WithRecovery(true), dstate.WithWrite(false), dstate.WithDeploymentHistory(false), ""); err != nil { + logdiag.LogError(ctx, err) + return b, stateDesc, root.ErrAlreadyPrinted + } } // Warn when the state was last written by a newer CLI than the one From 8dc139da2eb2fbc5da634faffabb4119c74c5eec Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Fri, 4 Sep 2026 11:14:19 +0000 Subject: [PATCH 12/46] bundle: move the DMS client and operation buffer into StateDB; drop them from DeploymentBundle The phases now reach DMS through StateDB (DmsClient, OperationBuffer, TakeOperationBuffer) rather than DeploymentBundle mirror fields. The operation drain stays before CompleteVersion. Co-authored-by: Isaac --- bundle/direct/bundle_apply.go | 4 ++-- bundle/direct/dstate/state.go | 22 +++++++++++++++++++--- bundle/direct/pkg.go | 8 -------- bundle/phases/deploy.go | 2 +- bundle/phases/destroy.go | 4 ++-- bundle/phases/dms.go | 17 +++++++++-------- cmd/bundle/utils/process.go | 4 ---- 7 files changed, 33 insertions(+), 28 deletions(-) diff --git a/bundle/direct/bundle_apply.go b/bundle/direct/bundle_apply.go index 72465fad346..818a1e95a23 100644 --- a/bundle/direct/bundle_apply.go +++ b/bundle/direct/bundle_apply.go @@ -77,7 +77,7 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa } // Stop resource CRUD once recording state with DMS has failed. - if buf := b.DmsAsyncOperationClient; buf != nil { + if buf := b.StateDB.OperationBuffer(); buf != nil { if err := buf.Err(); err != nil { logdiag.LogError(ctx, fmt.Errorf("%s: %w", errorPrefix, err)) return false @@ -183,7 +183,7 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa // Wait for the uploads and report a failure here, with the deploy's other errors. The phase // completes the version afterwards and drains again, quietly, so this is not printed twice. - if buf := b.DmsAsyncOperationClient; buf != nil { + if buf := b.StateDB.OperationBuffer(); buf != nil { if err := buf.Drain(); err != nil { logdiag.LogError(ctx, err) } diff --git a/bundle/direct/dstate/state.go b/bundle/direct/dstate/state.go index 6a9d1c4d2ee..ebddd3683b3 100644 --- a/bundle/direct/dstate/state.go +++ b/bundle/direct/dstate/state.go @@ -138,14 +138,30 @@ type WALEntry struct { // StartRecording opens the operation buffer that records each subsequent state write with DMS // under deploymentID and versionID, so what the service holds mirrors the WAL. It uses the client // Open built, so it is called only when the deployment records history, and once the version -// exists (after approval) - which is why it is not an Open option. Returns the buffer so the -// deploy can drain it. -func (db *DeploymentState) StartRecording(ctx context.Context, deploymentID string, versionID int64) *dms.OperationBuffer { +// exists (after approval) - which is why it is not an Open option. +func (db *DeploymentState) StartRecording(ctx context.Context, deploymentID string, versionID int64) { buf := dms.StartOperationBuffer(ctx, db.dmsClient, deploymentID, versionID) db.mu.Lock() defer db.mu.Unlock() db.operationBuffer = buf +} + +// OperationBuffer returns the buffer recording state writes to DMS, or nil when the bundle does not +// record deployment history. Apply reads it to surface a recording failure and to drain uploads. +func (db *DeploymentState) OperationBuffer() *dms.OperationBuffer { + db.mu.Lock() + defer db.mu.Unlock() + return db.operationBuffer +} + +// TakeOperationBuffer returns the operation buffer and clears it, so a deferred safety-net +// completion after an explicit one is a no-op. +func (db *DeploymentState) TakeOperationBuffer() *dms.OperationBuffer { + db.mu.Lock() + defer db.mu.Unlock() + buf := db.operationBuffer + db.operationBuffer = nil return buf } diff --git a/bundle/direct/pkg.go b/bundle/direct/pkg.go index 764397d5fc3..ea7ae1e51b6 100644 --- a/bundle/direct/pkg.go +++ b/bundle/direct/pkg.go @@ -11,7 +11,6 @@ import ( "github.com/databricks/cli/bundle/direct/dresources" "github.com/databricks/cli/bundle/direct/dstate" "github.com/databricks/cli/bundle/statemgmt/resourcestate" - "github.com/databricks/cli/libs/dms" "github.com/databricks/cli/libs/structs/structvar" ) @@ -51,13 +50,6 @@ type DeploymentBundle struct { Plan *deployplan.Plan RemoteStateCache sync.Map StateCache structvar.Cache - - // DMS carries deployment-history recording across the phases; nil unless the bundle records - // history. DmsApiClient is set where the state is opened. The deployment id and version live - // in bundle.deployment.history, not here. DmsAsyncOperationClient is opened once the deploy - // phase creates the version, and Apply drains it before returning. - DmsApiClient *dms.Client - DmsAsyncOperationClient *dms.OperationBuffer } // SetRemoteState updates the remote state with type validation and marks as fresh. diff --git a/bundle/phases/deploy.go b/bundle/phases/deploy.go index 5fe25dd2eee..b35757af167 100644 --- a/bundle/phases/deploy.go +++ b/bundle/phases/deploy.go @@ -326,7 +326,7 @@ func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHand // Create the deployment now that the plan is approved, so a declined deploy leaves none behind. // A first deploy's id did not exist at plan time - the version and any existing id were stamped // then - so stamp the one just created into the plan the apply reads. - if b.DeploymentBundle.DmsApiClient != nil { + if b.DeploymentBundle.StateDB.DmsClient() != nil { existingID, _ := deploymentAndNextVersion(b) createOrUpdateDeployment(ctx, b, dmsDeployment) if logdiag.HasError(ctx) { diff --git a/bundle/phases/destroy.go b/bundle/phases/destroy.go index 720f4ef7fa9..420e2a882d3 100644 --- a/bundle/phases/destroy.go +++ b/bundle/phases/destroy.go @@ -168,7 +168,7 @@ func destroyCore(ctx context.Context, b *bundle.Bundle, plan *deployplan.Plan, e // A completed destroy's resources are gone, so its deployment record is deleted too. if completed { deploymentID, _ := deploymentAndNextVersion(b) - if err := b.DeploymentBundle.DmsApiClient.DeleteDeployment(ctx, deploymentID); err != nil { + if err := b.DeploymentBundle.StateDB.DmsClient().DeleteDeployment(ctx, deploymentID); err != nil { logdiag.LogError(ctx, fmt.Errorf("failed to delete deployment: %w", err)) return } @@ -222,7 +222,7 @@ func Destroy(ctx context.Context, b *bundle.Bundle, engine engine.EngineType) { } else if completed { // A completed destroy's resources are gone, so its deployment record is deleted too. deploymentID, _ := deploymentAndNextVersion(b) - if err := b.DeploymentBundle.DmsApiClient.DeleteDeployment(ctx, deploymentID); err != nil { + if err := b.DeploymentBundle.StateDB.DmsClient().DeleteDeployment(ctx, deploymentID); err != nil { logdiag.LogError(ctx, fmt.Errorf("failed to delete deployment: %w", err)) } } diff --git a/bundle/phases/dms.go b/bundle/phases/dms.go index 40257fe5f5c..fc578aa62a6 100644 --- a/bundle/phases/dms.go +++ b/bundle/phases/dms.go @@ -85,18 +85,19 @@ func deploymentAndNextVersion(b *bundle.Bundle) (string, int64) { // no deployment behind; a first deploy's new id is then stamped into the plan (StampDeploymentID). func createOrUpdateDeployment(ctx context.Context, b *bundle.Bundle, current *bundledeployments.Deployment) { db := &b.DeploymentBundle + dmsClient := db.StateDB.DmsClient() metadata := deploymentMetadata(b) deploymentID, _ := deploymentAndNextVersion(b) switch mask := metadata.StaleFields(current); { case deploymentID == "": - id, err := db.DmsApiClient.CreateDeployment(ctx, b.Config.Workspace.StatePath, metadata) + id, err := dmsClient.CreateDeployment(ctx, b.Config.Workspace.StatePath, metadata) if err != nil { logdiag.LogError(ctx, fmt.Errorf("failed to create deployment: %w", err)) return } deploymentID = id case mask != "": - if err := db.DmsApiClient.UpdateDeployment(ctx, deploymentID, metadata, mask); err != nil { + if err := dmsClient.UpdateDeployment(ctx, deploymentID, metadata, mask); err != nil { logdiag.LogError(ctx, fmt.Errorf("failed to update deployment: %w", err)) return } @@ -116,7 +117,8 @@ func createOrUpdateDeployment(ctx context.Context, b *bundle.Bundle, current *bu // A no-op when the bundle does not record deployment history. func startVersion(ctx context.Context, b *bundle.Bundle, versionType dms.VersionType, staged []dms.StagedOperation) error { db := &b.DeploymentBundle - if db.DmsApiClient == nil { + dmsClient := db.StateDB.DmsClient() + if dmsClient == nil { return nil } deploymentID, versionID := deploymentAndNextVersion(b) @@ -139,7 +141,7 @@ func startVersion(ctx context.Context, b *bundle.Bundle, versionType dms.Version OriginUrl: git.OriginURL, } } - version, err := db.DmsApiClient.CreateVersion(ctx, deploymentID, versionID, dms.CreateVersionRequest{ + version, err := dmsClient.CreateVersion(ctx, deploymentID, versionID, dms.CreateVersionRequest{ CliVersion: build.GetInfo().Version, VersionType: versionType, PreviousVersionId: previousVersionID, @@ -151,7 +153,7 @@ func startVersion(ctx context.Context, b *bundle.Bundle, versionType dms.Version } log.Infof(ctx, "Created deployment version: deployment=%s version=%s", deploymentID, version.VersionId) - db.DmsAsyncOperationClient = db.StateDB.StartRecording(ctx, deploymentID, versionID) + db.StateDB.StartRecording(ctx, deploymentID, versionID) return nil } @@ -162,11 +164,10 @@ func startVersion(ctx context.Context, b *bundle.Bundle, versionType dms.Version // was created. func drainOperationsAndCompleteVersion(ctx context.Context, b *bundle.Bundle, success bool) (bool, error) { db := &b.DeploymentBundle - buf := db.DmsAsyncOperationClient + buf := db.StateDB.TakeOperationBuffer() if buf == nil { return false, nil } - db.DmsAsyncOperationClient = nil // Drain first; a recording error fails the deploy even when the resources applied. Its error // is left to whoever drained (apply), not returned twice. @@ -179,7 +180,7 @@ func drainOperationsAndCompleteVersion(ctx context.Context, b *bundle.Bundle, su reason = bundledeployments.VersionCompleteVersionCompleteFailure } deploymentID, versionID := deploymentAndNextVersion(b) - if err := db.DmsApiClient.CompleteVersion(ctx, deploymentID, versionID, reason); err != nil { + if err := db.StateDB.DmsClient().CompleteVersion(ctx, deploymentID, versionID, reason); err != nil { return false, err } log.Infof(ctx, "Completed deployment version: deployment=%s version=%d reason=%s", deploymentID, versionID, reason) diff --git a/cmd/bundle/utils/process.go b/cmd/bundle/utils/process.go index 3074ba4272c..620d6ae9048 100644 --- a/cmd/bundle/utils/process.go +++ b/cmd/bundle/utils/process.go @@ -293,10 +293,6 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle logdiag.LogError(ctx, err) return b, stateDesc, root.ErrAlreadyPrinted } - - // Open built the DMS client from the workspace client on the context; hand it to the - // phases that create versions and deployments. - b.DeploymentBundle.DmsApiClient = b.DeploymentBundle.StateDB.DmsClient() } else { if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WithRecovery(true), dstate.WithWrite(false), dstate.WithDeploymentHistory(false), ""); err != nil { logdiag.LogError(ctx, err) From 8ec86f85862575e1c4a07a5e32e5a8aeb3639884 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Fri, 4 Sep 2026 12:43:31 +0000 Subject: [PATCH 13/46] bundle/dms: StorageBackend enum as source of truth; hide operation buffer in StateDB - add dstate.StorageBackend (WORKSPACE_FILESYSTEM/DEPLOYMENT_METADATA_SERVICE), set by Open from the state marker; the plan carries its string form so deploy --plan can reject a plan whose target switched backends - StateDB owns the recording lifecycle: RecordingError() and CompleteVersion() (idempotent via versionCompleted), with the operation-buffer drain moved into Finalize; drop the exposed OperationBuffer/TakeOperationBuffer getters and the recorder() indirection - gate DMS behavior on StorageBackend()==DEPLOYMENT_METADATA_SERVICE / ConfiguresDeploymentHistory instead of DmsClient()!=nil - consolidate plan validation in validatePlan (config-vs-plan backend match + stale-plan guard) Co-authored-by: Isaac --- bundle/deployplan/plan.go | 5 ++ bundle/direct/bundle_apply.go | 15 +---- bundle/direct/bundle_plan.go | 6 ++ bundle/direct/dstate/state.go | 110 +++++++++++++++++++++++++++------- bundle/phases/deploy.go | 11 +++- bundle/phases/destroy.go | 4 +- bundle/phases/dms.go | 31 ---------- cmd/bundle/utils/process.go | 63 ++++++++++++------- 8 files changed, 152 insertions(+), 93 deletions(-) diff --git a/bundle/deployplan/plan.go b/bundle/deployplan/plan.go index 3f24c82ee43..726e0de9a92 100644 --- a/bundle/deployplan/plan.go +++ b/bundle/deployplan/plan.go @@ -29,6 +29,11 @@ type Plan struct { NextVersionId string `json:"next_version_id,omitempty"` LastVersionId string `json:"last_version_id,omitempty"` + // StorageBackend is where this deployment's state lives, the string form of a + // dstate.StorageBackend. deploy --plan checks it against the target's config so a recorded plan + // is never applied to a non-recording target or the reverse. Set only for a recorded plan. + StorageBackend string `json:"storage_backend,omitempty"` + Plan map[string]*PlanEntry `json:"plan,omitzero"` // NotSelected is the number of resources removed by FilterToSelected via the diff --git a/bundle/direct/bundle_apply.go b/bundle/direct/bundle_apply.go index 818a1e95a23..8c2de1f29ee 100644 --- a/bundle/direct/bundle_apply.go +++ b/bundle/direct/bundle_apply.go @@ -77,11 +77,9 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa } // Stop resource CRUD once recording state with DMS has failed. - if buf := b.StateDB.OperationBuffer(); buf != nil { - if err := buf.Err(); err != nil { - logdiag.LogError(ctx, fmt.Errorf("%s: %w", errorPrefix, err)) - return false - } + if err := b.StateDB.RecordingError(); err != nil { + logdiag.LogError(ctx, fmt.Errorf("%s: %w", errorPrefix, err)) + return false } adapter, err := b.getAdapterForKey(resourceKey) @@ -181,13 +179,6 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa return true }) - // Wait for the uploads and report a failure here, with the deploy's other errors. The phase - // completes the version afterwards and drains again, quietly, so this is not printed twice. - if buf := b.StateDB.OperationBuffer(); buf != nil { - if err := buf.Drain(); err != nil { - logdiag.LogError(ctx, err) - } - } } func (b *DeploymentBundle) LookupReferencePostDeploy(ctx context.Context, path *structpath.PathNode) (any, error) { diff --git a/bundle/direct/bundle_plan.go b/bundle/direct/bundle_plan.go index 45d5b2d3663..4c1c4f4e3a8 100644 --- a/bundle/direct/bundle_plan.go +++ b/bundle/direct/bundle_plan.go @@ -172,6 +172,12 @@ func (b *DeploymentBundle) CalculatePlan(ctx context.Context, client *databricks // deploy --plan can reject a plan the deployment has moved on from, and pass last_version_id // as previous_version_id. History is set only while recording, so other plans leave these empty. // configRoot is nil for destroy, which records its version separately. + // The plan records where its state lives so deploy --plan can reject a plan whose target has + // since switched backends. Set from StateDB (the source of truth) so a first deploy - which has + // no History yet - still carries it. + if b.StateDB.StorageBackend() == dstate.StorageBackendDeploymentMetadataService { + plan.StorageBackend = string(dstate.StorageBackendDeploymentMetadataService) + } if configRoot != nil && configRoot.Bundle.Deployment.History != nil { h := configRoot.Bundle.Deployment.History next, err := dms.NextVersion(h.LatestVersionID) diff --git a/bundle/direct/dstate/state.go b/bundle/direct/dstate/state.go index ebddd3683b3..2e410ee25b8 100644 --- a/bundle/direct/dstate/state.go +++ b/bundle/direct/dstate/state.go @@ -21,6 +21,7 @@ import ( "github.com/databricks/cli/libs/cmdctx" "github.com/databricks/cli/libs/dms" "github.com/databricks/cli/libs/log" + "github.com/databricks/databricks-sdk-go/service/bundledeployments" "github.com/google/uuid" ) @@ -51,6 +52,16 @@ const featureRecordDeploymentHistory = "record_deployment_history" // state that depends on a feature it does not recognize. const featuresDocURL = "https://docs.databricks.com/aws/en/dev-tools/bundles/state-features#state-features" +// StorageBackend identifies where a deployment's resource state lives: the workspace filesystem +// (the state file) or the deployment metadata service. It is the source of truth for gating DMS +// behavior after Open; the plan carries its string form to check against the target's config. +type StorageBackend string + +const ( + StorageBackendWorkspaceFilesystem StorageBackend = "WORKSPACE_FILESYSTEM" + StorageBackendDeploymentMetadataService StorageBackend = "DEPLOYMENT_METADATA_SERVICE" +) + // recognizedFeatures is the set of state feature flags this CLI understands. A state depending on // any feature not listed here is refused (see checkStateFeatures), so a newer CLI's feature is not // silently clobbered by this one. @@ -95,6 +106,20 @@ type DeploymentState struct { // dmsClient talks to the deployment metadata service. Open builds it from the workspace // client when the deployment records history; nil otherwise. dmsClient *dms.Client + + // storageBackend is where this deployment's state lives, set by Open from the feature marker. + // It is the source of truth for gating DMS behavior after Open, in place of nil client checks. + storageBackend StorageBackend + + // deploymentID and versionID identify the recorded deployment and the version this run writes, + // stored by StartRecording. CompleteVersion reads them after Finalize's reset has cleared Path, + // Data and stateIDs, so - like operationBuffer and dmsClient - they must survive reset. + deploymentID string + versionID int64 + + // versionCompleted makes CompleteVersion a no-op after the first call, so a deferred safety-net + // completion after an explicit one does nothing. + versionCompleted bool } type Header struct { @@ -145,31 +170,55 @@ func (db *DeploymentState) StartRecording(ctx context.Context, deploymentID stri db.mu.Lock() defer db.mu.Unlock() db.operationBuffer = buf + db.deploymentID = deploymentID + db.versionID = versionID } -// OperationBuffer returns the buffer recording state writes to DMS, or nil when the bundle does not -// record deployment history. Apply reads it to surface a recording failure and to drain uploads. -func (db *DeploymentState) OperationBuffer() *dms.OperationBuffer { +// RecordingError reports whether recording state writes to the service has failed, so the apply +// stops touching resources once the service is no longer keeping up. Nil when the bundle does not +// record deployment history or recording is healthy. +func (db *DeploymentState) RecordingError() error { db.mu.Lock() - defer db.mu.Unlock() - return db.operationBuffer + buf := db.operationBuffer + db.mu.Unlock() + if buf == nil { + return nil + } + return buf.Err() } -// TakeOperationBuffer returns the operation buffer and clears it, so a deferred safety-net -// completion after an explicit one is a no-op. -func (db *DeploymentState) TakeOperationBuffer() *dms.OperationBuffer { +// CompleteVersion marks the recorded version done, reporting whether it completed here. A no-op +// returning false when no version was created (the bundle does not record history, or a deploy was +// declined) or the version was already completed, so a deferred safety-net call after an explicit +// one does nothing. Finalize has already drained the buffered operations. It reads only fields that +// survive Finalize's reset and asserts nothing, so it is safe to call after the state is closed. +func (db *DeploymentState) CompleteVersion(ctx context.Context, success bool) (bool, error) { db.mu.Lock() - defer db.mu.Unlock() - buf := db.operationBuffer - db.operationBuffer = nil - return buf + if db.operationBuffer == nil || db.versionCompleted { + db.mu.Unlock() + return false, nil + } + db.versionCompleted = true + deploymentID, versionID, client := db.deploymentID, db.versionID, db.dmsClient + db.mu.Unlock() + + reason := bundledeployments.VersionCompleteVersionCompleteSuccess + if !success { + reason = bundledeployments.VersionCompleteVersionCompleteFailure + } + if err := client.CompleteVersion(ctx, deploymentID, versionID, reason); err != nil { + return false, err + } + log.Infof(ctx, "Completed deployment version: deployment=%s version=%d reason=%s", deploymentID, versionID, reason) + // Report the success value the version completed with, not merely that it completed: a destroy + // deletes the deployment record only for a version that completed successfully. + return success, nil } // RecordFailure records that a resource did not apply, so the history says why rather than // leaving the resource out. resourceID is the id it had before the failure. func (db *DeploymentState) RecordFailure(resourceKey, resourceID string, cause error) { - r := db.recorder() - if r == nil { + if db.StorageBackend() != StorageBackendDeploymentMetadataService { return } @@ -186,15 +235,11 @@ func (db *DeploymentState) RecordFailure(resourceKey, resourceID string, cause e } } - r.RecordFailure(resourceKey, resourceID, recorded, cause) -} - -// recorder reads the buffer under db.mu, which guards it, and returns nil when the bundle does -// not record deployment history. -func (db *DeploymentState) recorder() *dms.OperationBuffer { + // A failure is recorded only while a version is open (after StartRecording), so the buffer is set. db.mu.Lock() - defer db.mu.Unlock() - return db.operationBuffer + buf := db.operationBuffer + db.mu.Unlock() + buf.RecordFailure(resourceKey, resourceID, recorded, cause) } func NewDatabase(lineage string, serial int) Database { @@ -333,6 +378,15 @@ func (db *DeploymentState) RequiresDeploymentHistory() bool { return ok } +// StorageBackend reports where this deployment's state lives - the source of truth for whether the +// bundle records deployment history. Set by Open; valid only after the state is opened. +func (db *DeploymentState) StorageBackend() StorageBackend { + db.AssertOpenedForReadOrWrite() + db.mu.Lock() + defer db.mu.Unlock() + return db.storageBackend +} + // DmsClient returns the deployment metadata service client Open built from the workspace client, // or nil when the deployment does not record history. func (db *DeploymentState) DmsClient() *dms.Client { @@ -493,7 +547,10 @@ To record this bundle's history, start it over as a new deployment: This deployment's resources are recorded with the deployment metadata service. Set experimental.record_deployment_history: true to deploy or destroy this bundle`) } + db.storageBackend = StorageBackendWorkspaceFilesystem if recorded { + db.storageBackend = StorageBackendDeploymentMetadataService + // The service is the source of truth for a recorded deployment; the file is a tombstone // carrying only the marker, and applyDMSState loads the resources the service holds. client, err := dms.NewClient(cmdctx.WorkspaceClient(ctx)) @@ -697,6 +754,15 @@ func (db *DeploymentState) Finalize(ctx context.Context) (resourcestate.Exported err = db.replayWAL(ctx) } + // A recorded deployment uploads each write to the service on a background goroutine; block until + // they land or fail so the version completes only over state the service actually holds. A nil + // buffer means the deployment does not record history. + if db.operationBuffer != nil { + if drainErr := db.operationBuffer.Drain(); drainErr != nil { + err = errors.Join(err, drainErr) + } + } + state := ExportStateFromData(db.Data) db.reset() diff --git a/bundle/phases/deploy.go b/bundle/phases/deploy.go index b35757af167..9ffe6eb3b44 100644 --- a/bundle/phases/deploy.go +++ b/bundle/phases/deploy.go @@ -19,6 +19,7 @@ import ( "github.com/databricks/cli/bundle/deploy/snapshot" "github.com/databricks/cli/bundle/deploy/terraform" "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/cli/bundle/direct/dstate" "github.com/databricks/cli/bundle/libraries" "github.com/databricks/cli/bundle/metrics" "github.com/databricks/cli/bundle/permissions" @@ -204,8 +205,12 @@ func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHand // The version is created only after approval; CompleteVersion is deferred before // lock.Release and no-ops until then. defer func() { - if _, err := drainOperationsAndCompleteVersion(ctx, b, !logdiag.HasError(ctx)); err != nil { - logdiag.LogError(ctx, err) + // Skip entirely when not recording; CompleteVersion runs after Finalize has closed the state, + // so it is gated on the config rather than the (reset) StateDB. + if b.ConfiguresDeploymentHistory(ctx) { + if _, err := b.DeploymentBundle.StateDB.CompleteVersion(ctx, !logdiag.HasError(ctx)); err != nil { + logdiag.LogError(ctx, err) + } } bundle.ApplyContext(ctx, b, lock.Release(lock.GoalDeploy)) }() @@ -326,7 +331,7 @@ func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHand // Create the deployment now that the plan is approved, so a declined deploy leaves none behind. // A first deploy's id did not exist at plan time - the version and any existing id were stamped // then - so stamp the one just created into the plan the apply reads. - if b.DeploymentBundle.StateDB.DmsClient() != nil { + if b.DeploymentBundle.StateDB.StorageBackend() == dstate.StorageBackendDeploymentMetadataService { existingID, _ := deploymentAndNextVersion(b) createOrUpdateDeployment(ctx, b, dmsDeployment) if logdiag.HasError(ctx) { diff --git a/bundle/phases/destroy.go b/bundle/phases/destroy.go index 420e2a882d3..309adb120da 100644 --- a/bundle/phases/destroy.go +++ b/bundle/phases/destroy.go @@ -160,7 +160,7 @@ func destroyCore(ctx context.Context, b *bundle.Bundle, plan *deployplan.Plan, e } // Complete version before deleting remote files; the deployment node is under statePath. - completed, err := drainOperationsAndCompleteVersion(ctx, b, true) + completed, err := b.DeploymentBundle.StateDB.CompleteVersion(ctx, true) if err != nil { logdiag.LogError(ctx, err) return @@ -216,7 +216,7 @@ func Destroy(ctx context.Context, b *bundle.Bundle, engine engine.EngineType) { // destroy records nothing. Deferred before lock.Release to hold the lock; a no-op once // destroyCore has completed the version. defer func() { - completed, err := drainOperationsAndCompleteVersion(ctx, b, !logdiag.HasError(ctx)) + completed, err := b.DeploymentBundle.StateDB.CompleteVersion(ctx, !logdiag.HasError(ctx)) if err != nil { logdiag.LogError(ctx, err) } else if completed { diff --git a/bundle/phases/dms.go b/bundle/phases/dms.go index fc578aa62a6..2db8bf57dca 100644 --- a/bundle/phases/dms.go +++ b/bundle/phases/dms.go @@ -157,37 +157,6 @@ func startVersion(ctx context.Context, b *bundle.Bundle, versionType dms.Version return nil } -// drainOperationsAndCompleteVersion drains the buffered operations and completes the version. -// It returns whether the version completed successfully, which a destroy uses to decide whether -// to delete the deployment. Idempotent: the first call clears the buffer, so a deferred -// safety-net after an explicit completion is a no-op returning false. Also false when no version -// was created. -func drainOperationsAndCompleteVersion(ctx context.Context, b *bundle.Bundle, success bool) (bool, error) { - db := &b.DeploymentBundle - buf := db.StateDB.TakeOperationBuffer() - if buf == nil { - return false, nil - } - - // Drain first; a recording error fails the deploy even when the resources applied. Its error - // is left to whoever drained (apply), not returned twice. - if buf.Drain() != nil { - success = false - } - - reason := bundledeployments.VersionCompleteVersionCompleteSuccess - if !success { - reason = bundledeployments.VersionCompleteVersionCompleteFailure - } - deploymentID, versionID := deploymentAndNextVersion(b) - if err := db.StateDB.DmsClient().CompleteVersion(ctx, deploymentID, versionID, reason); err != nil { - return false, err - } - log.Infof(ctx, "Completed deployment version: deployment=%s version=%d reason=%s", deploymentID, versionID, reason) - - return success, nil -} - // logDeploymentVersion logs the deployment version URL. Workspace ID is omitted // so the page stays clickable in a terminal and redirects correctly without it. func logDeploymentVersion(ctx context.Context, b *bundle.Bundle, deploymentID string, version int64) { diff --git a/cmd/bundle/utils/process.go b/cmd/bundle/utils/process.go index 620d6ae9048..74909c20997 100644 --- a/cmd/bundle/utils/process.go +++ b/cmd/bundle/utils/process.go @@ -364,29 +364,7 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle log.Warnf(ctx, "Plan was created with CLI version %s but current version is %s", plan.CLIVersion, currentVersion) } - // The plan records the DMS deployment and version it targeted. Reject it if the live - // deployment moved on - a newer version (someone deployed since, possibly from another - // machine) or a different id (deleted and recreated) - so a stale plan is never applied on - // top of newer state. This runs before the lineage/serial check and is the authoritative - // stale guard for recorded bundles, whose local state is only a tombstone (its serial does - // not catch a deploy from elsewhere). Both sides are empty when not recording, a no-op there. - remoteLastVersion := "" - if dmsDeployment != nil { - remoteLastVersion = dmsDeployment.LastVersionId - } - if plan.LastVersionId != remoteLastVersion { - logdiag.LogError(ctx, fmt.Errorf("this plan predates the deployment's current version %s; run 'bundle plan' again", remoteLastVersion)) - return b, stateDesc, root.ErrAlreadyPrinted - } - if plan.DeploymentId != dmsDeploymentID { - logdiag.LogError(ctx, errors.New("this plan targets a different deployment than the one now recorded for this bundle; run 'bundle plan' again")) - return b, stateDesc, root.ErrAlreadyPrinted - } - - // Validate that the plan's lineage and serial match the local state. This is the stale guard - // for non-recorded bundles (recorded ones are covered by the version check above). - err = direct.ValidatePlanAgainstState(&b.DeploymentBundle.StateDB, plan) - if err != nil { + if err := validatePlan(ctx, b, plan, dmsDeployment, dmsDeploymentID); err != nil { logdiag.LogError(ctx, err) return b, stateDesc, root.ErrAlreadyPrinted } @@ -484,6 +462,45 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle return b, stateDesc, nil } +// validatePlan rejects a --plan file that no longer matches the target: its recording shape must +// match the target's config, and a plan that targets an existing recorded deployment must not +// predate the deployment the service now holds. It ends with the local lineage/serial guard. +func validatePlan(ctx context.Context, b *bundle.Bundle, plan *deployplan.Plan, dmsDeployment *bundledeployments.Deployment, dmsDeploymentID string) error { + // A plan records its storage backend, which must match the target's current one, or it would + // deploy against the wrong backend. StateDB.StorageBackend is the target's source of truth, and + // it is set even for a first recorded deploy (unlike the version ids, which are empty then). + isDMSPlan := plan.StorageBackend == string(dstate.StorageBackendDeploymentMetadataService) + switch records := b.DeploymentBundle.StateDB.StorageBackend() == dstate.StorageBackendDeploymentMetadataService; { + case isDMSPlan && !records: + return errors.New("this plan records deployment history, but the target no longer does; run 'bundle plan' again") + case !isDMSPlan && records: + return errors.New("this plan does not record deployment history, but the target now does; run 'bundle plan' again") + } + + // The plan records the DMS deployment and version it targeted. Reject it if the live deployment + // moved on - a newer version (someone deployed since, possibly from another machine) or a + // different id (deleted and recreated) - so a stale plan is never applied on top of newer state. + // This is the authoritative stale guard for a recorded bundle, whose local state is only a + // tombstone (its serial does not catch a deploy from elsewhere). Gated on the plan being a DMS + // plan; a first-deploy plan has empty version ids that still get compared (and match) here. + if isDMSPlan { + remoteLastVersion := "" + if dmsDeployment != nil { + remoteLastVersion = dmsDeployment.LastVersionId + } + if plan.LastVersionId != remoteLastVersion { + return fmt.Errorf("this plan predates the deployment's current version %s; run 'bundle plan' again", remoteLastVersion) + } + if plan.DeploymentId != dmsDeploymentID { + return errors.New("this plan targets a different deployment than the one now recorded for this bundle; run 'bundle plan' again") + } + } + + // Validate that the plan's lineage and serial match the local state. This is the stale guard for + // non-recorded bundles (recorded ones are covered by the version check above). + return direct.ValidatePlanAgainstState(&b.DeploymentBundle.StateDB, plan) +} + // ResolveEngineSetting determines the effective engine setting by combining bundle config and env var. // Priority: bundle.engine config > DATABRICKS_BUNDLE_ENGINE env var > engine.Default. func ResolveEngineSetting(ctx context.Context, b *bundle.Bundle) (engine.EngineSetting, error) { From 3683aeddfcdb2de34ea6bd42f8e768dccd005e38 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Fri, 4 Sep 2026 12:55:38 +0000 Subject: [PATCH 14/46] bundle/dms: rename first-version stamp, inline env predicate, state-gate unbind, restore v3 read - rename StampDeploymentID -> StampDeploymentIdForFirstVersion (only used for the first version's id) - inline RecordDeploymentHistoryEnv into RecordsDeploymentHistory (single predicate) - unbind: drop the config-based phase check; DeploymentBundle.Unbind opens without recording, so the state marker (not the config flag) refuses a recorded deployment - restore forward-compat read of featureStateVersion=3: this CLI still writes v2, but reads a v3 state with no features and refuses one that carries a feature Co-authored-by: Isaac --- bundle/direct/bundle_plan.go | 6 +++--- bundle/direct/dstate/migrate.go | 18 ++++++++++++++++++ bundle/direct/dstate/state.go | 16 +++++++++++++--- bundle/env/dms.go | 17 +++++++---------- bundle/env/dms_test.go | 4 ++-- bundle/phases/bind.go | 8 +++----- bundle/phases/deploy.go | 2 +- 7 files changed, 47 insertions(+), 24 deletions(-) diff --git a/bundle/direct/bundle_plan.go b/bundle/direct/bundle_plan.go index 4c1c4f4e3a8..3b6b389bc57 100644 --- a/bundle/direct/bundle_plan.go +++ b/bundle/direct/bundle_plan.go @@ -64,7 +64,7 @@ func ValidatePlanAgainstState(stateDB *dstate.DeploymentState, plan *deployplan. // InitForApply initializes the DeploymentBundle for applying a pre-computed plan: it parses the // plan's entries (loaded as JSON when the plan came from a file) into the typed state cache the // apply reads. A first deployment's id does not exist until the deploy creates it after approval; -// StampDeploymentID fills it in then. StateDB must already be open for write before calling this. +// StampDeploymentIdForFirstVersion fills it in then. StateDB must already be open for write before calling this. func (b *DeploymentBundle) InitForApply(ctx context.Context, client *databricks.WorkspaceClient, plan *deployplan.Plan) error { b.StateDB.AssertOpenedForWrite() @@ -118,11 +118,11 @@ func (b *DeploymentBundle) InitForApply(ctx context.Context, client *databricks. return nil } -// StampDeploymentID fills deploymentID into the plan's jobs and pipelines that still lack one, in +// StampDeploymentIdForFirstVersion fills deploymentID into the plan's jobs and pipelines that still lack one, in // both the state cache the apply reads and the plan JSON. It is called after approval creates the // deployment, for a first deploy whose id did not exist at plan time; the version and any // pre-existing id were stamped then. -func (b *DeploymentBundle) StampDeploymentID(deploymentID string) error { +func (b *DeploymentBundle) StampDeploymentIdForFirstVersion(deploymentID string) error { for resourceKey, entry := range b.Plan.Plan { if entry.NewState == nil || len(entry.NewState.Value) == 0 { continue diff --git a/bundle/direct/dstate/migrate.go b/bundle/direct/dstate/migrate.go index dccb06686a5..6a891900c36 100644 --- a/bundle/direct/dstate/migrate.go +++ b/bundle/direct/dstate/migrate.go @@ -3,6 +3,8 @@ package dstate import ( "encoding/json" "fmt" + "slices" + "strings" "github.com/databricks/cli/bundle/direct/dresources" "github.com/databricks/cli/libs/structs/structpath" @@ -13,6 +15,22 @@ import ( // migrateState runs all necessary migrations on the database. // It is called after loading state from disk. func migrateState(db *Database) error { + // A featureStateVersion state carries a feature list this CLI does not write (see the + // featureStateVersion doc comment). With no features it is equivalent to currentStateVersion, so + // accept it and leave the on-disk version as-is; with any feature it depends on capabilities this + // CLI lacks, so refuse it and tell the user to upgrade. + if db.StateVersion == featureStateVersion { + if len(db.Features) == 0 { + return nil + } + features := make([]string, 0, len(db.Features)) + for name := range db.Features { + features = append(features, name) + } + slices.Sort(features) + return fmt.Errorf("the deployment state requires features this CLI does not support: %s; upgrade to the latest CLI version and see %s for more information", strings.Join(features, ", "), featuresDocURL) + } + if db.StateVersion == currentStateVersion { return nil } diff --git a/bundle/direct/dstate/state.go b/bundle/direct/dstate/state.go index 2e410ee25b8..a248f8328fb 100644 --- a/bundle/direct/dstate/state.go +++ b/bundle/direct/dstate/state.go @@ -33,9 +33,19 @@ const ( maxWalEntrySize = 10 * 1024 * 1024 walSuffix = ".wal" - // supportedStateVersion is the highest schema version this CLI can read: the - // version it writes. A state newer than this is rejected as too new. - supportedStateVersion = currentStateVersion + // featureStateVersion is the schema version a future CLI will write once it records feature flags + // in the state version itself. This CLI does not write it, but reads such states (see + // migrateState): featureStateVersion with no features is equivalent to currentStateVersion and is + // accepted as-is (the on-disk version is left at featureStateVersion, not flipped down); + // featureStateVersion with any feature depends on capabilities this CLI lacks, so it is refused. + // Forward-compat scaffolding so a later release can write featureStateVersion + features without + // this CLI rejecting them outright. Always 3. + featureStateVersion = 3 + + // supportedStateVersion is the highest schema version this CLI can read. Normally equal to + // currentStateVersion - the version it reads is the version it writes - but this CLI can also + // read (never write) featureStateVersion. A state newer than this is rejected as too new. + supportedStateVersion = featureStateVersion ) // featureRecordDeploymentHistory marks a state whose resources are also recorded with the diff --git a/bundle/env/dms.go b/bundle/env/dms.go index d7258a76867..66779aa3fe4 100644 --- a/bundle/env/dms.go +++ b/bundle/env/dms.go @@ -7,16 +7,13 @@ import "context" // Deliberately undocumented. const RecordDeploymentHistoryVariable = "DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY" -// RecordDeploymentHistoryEnv reports whether the environment turns on deployment history -// recording. Only "true" turns it on: anything else - including a typo like "TRUE" or "yes" - -// leaves the feature off rather than silently enabling it. -func RecordDeploymentHistoryEnv(ctx context.Context) bool { +// RecordsDeploymentHistory reports whether recording is on, from config or env var. Single predicate +// for all recording code paths, keeping them in sync. The env var only counts as "true": anything +// else - a typo like "TRUE" or "yes" - leaves recording off rather than silently enabling it. +func RecordsDeploymentHistory(ctx context.Context, configured bool) bool { + if configured { + return true + } value, _ := get(ctx, []string{RecordDeploymentHistoryVariable}) return value == "true" } - -// RecordsDeploymentHistory reports whether recording is on, from config or env var. -// Single predicate for all recording code paths; keeps them in sync. -func RecordsDeploymentHistory(ctx context.Context, configured bool) bool { - return configured || RecordDeploymentHistoryEnv(ctx) -} diff --git a/bundle/env/dms_test.go b/bundle/env/dms_test.go index 15bfe0e63f0..cfd91c0007a 100644 --- a/bundle/env/dms_test.go +++ b/bundle/env/dms_test.go @@ -22,12 +22,12 @@ func TestRecordDeploymentHistoryEnv(t *testing.T) { {"yes", false}, } { ctx := env.Set(t.Context(), RecordDeploymentHistoryVariable, tc.value) - assert.Equal(t, tc.want, RecordDeploymentHistoryEnv(ctx), "value %q", tc.value) + assert.Equal(t, tc.want, RecordsDeploymentHistory(ctx, false), "value %q", tc.value) } } func TestRecordDeploymentHistoryEnvUnset(t *testing.T) { - assert.False(t, RecordDeploymentHistoryEnv(t.Context())) + assert.False(t, RecordsDeploymentHistory(t.Context(), false)) } func TestRecordsDeploymentHistory(t *testing.T) { diff --git a/bundle/phases/bind.go b/bundle/phases/bind.go index eb141ea59b7..fc36122d1d8 100644 --- a/bundle/phases/bind.go +++ b/bundle/phases/bind.go @@ -135,11 +135,9 @@ func Unbind(ctx context.Context, b *bundle.Bundle, bundleType, tfResourceType, r }() if engine.IsDirect() { - if b.ConfiguresDeploymentHistory(ctx) { - logdiag.LogError(ctx, errors.New("unbind is not supported for a bundle that records deployment history")) - return - } - + // A recorded deployment cannot be unbound: DeploymentBundle.Unbind opens the state without + // recording, so the state guard refuses it when the marker is set. State is the source of + // truth, so a config flag that disagrees with the marker cannot slip an unbind through. groupName, ok := terraform.TerraformToGroupName[tfResourceType] if !ok { groupName = tfResourceType diff --git a/bundle/phases/deploy.go b/bundle/phases/deploy.go index 9ffe6eb3b44..b7dd3ea44d1 100644 --- a/bundle/phases/deploy.go +++ b/bundle/phases/deploy.go @@ -339,7 +339,7 @@ func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHand } if existingID == "" { deploymentID, _ := deploymentAndNextVersion(b) - if err := b.DeploymentBundle.StampDeploymentID(deploymentID); err != nil { + if err := b.DeploymentBundle.StampDeploymentIdForFirstVersion(deploymentID); err != nil { logdiag.LogError(ctx, err) return } From c0bc24dc908c47337b1d71dbe3a7444883c9adf0 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Fri, 4 Sep 2026 13:05:11 +0000 Subject: [PATCH 15/46] acceptance/dms: record plan JSON, list-resources, and raw recorded state Extend declined-deploy, depends-on and deployment-metadata-change to capture the serialized 'bundle plan -o json' (now carrying storage_backend), the DMS list-resources view, and the raw recorded state. deployment-metadata-change pins state_path, which print_state.py cannot resolve, so it relies on list-resources. Co-authored-by: Isaac --- .../bundle/dms/declined-deploy/output.txt | 55 ++++++ acceptance/bundle/dms/declined-deploy/script | 4 + acceptance/bundle/dms/depends-on/output.txt | 156 ++++++++++++++++++ acceptance/bundle/dms/depends-on/script | 6 + .../dms/deployment-metadata-change/output.txt | 104 ++++++++++++ .../dms/deployment-metadata-change/script | 13 ++ 6 files changed, 338 insertions(+) diff --git a/acceptance/bundle/dms/declined-deploy/output.txt b/acceptance/bundle/dms/declined-deploy/output.txt index ef8a2cb048b..d0ce19a2389 100644 --- a/acceptance/bundle/dms/declined-deploy/output.txt +++ b/acceptance/bundle/dms/declined-deploy/output.txt @@ -59,6 +59,61 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged } } +=== The plan is a no-op now and records DEPLOYMENT_METADATA_SERVICE; the service holds the schema at version 1 +>>> [CLI] bundle plan -o json +{ + "plan_version": 3, + "cli_version": "[CLI_VERSION]", + "lineage": "[UUID]", + "serial": 1, + "storage_backend": "DEPLOYMENT_METADATA_SERVICE", + "plan": { + "resources.schemas.foo": { + "action": "skip", + "remote_state": { + "browse_only": false, + "catalog_name": "main", + "catalog_type": "MANAGED_CATALOG", + "created_at": [UNIX_TIME_MILLIS][0], + "created_by": "[USERNAME]", + "effective_predictive_optimization_flag": { + "inherited_from_name": "[METASTORE_NAME]", + "inherited_from_type": "METASTORE", + "value": "ENABLE" + }, + "enable_predictive_optimization": "INHERIT", + "full_name": "main.dms_declined_deploy_schema_[UNIQUE_NAME]", + "metastore_id": "[UUID]", + "name": "dms_declined_deploy_schema_[UNIQUE_NAME]", + "owner": "[USERNAME]", + "schema_id": "[UUID]", + "updated_at": [UNIX_TIME_MILLIS][0], + "updated_by": "[USERNAME]" + } + } + } +} + +>>> print_state.py +{ + "state_version": 2, + "cli_version": "[CLI_VERSION]", + "lineage": "[UUID]", + "serial": 1, + "features": { + "record_deployment_history": {} + }, + "state": { + "resources.schemas.foo": { + "__id__": "main.dms_declined_deploy_schema_[UNIQUE_NAME]", + "state": { + "catalog_name": "main", + "name": "dms_declined_deploy_schema_[UNIQUE_NAME]" + } + } + } +} + === A destructive change without --auto-approve is declined: this console cannot prompt >>> update_file.py databricks.yml catalog_name: main catalog_name: dms_other_[UNIQUE_NAME] diff --git a/acceptance/bundle/dms/declined-deploy/script b/acceptance/bundle/dms/declined-deploy/script index df539b83b29..cc69f672e63 100644 --- a/acceptance/bundle/dms/declined-deploy/script +++ b/acceptance/bundle/dms/declined-deploy/script @@ -4,6 +4,10 @@ title "Deploy a schema, so the deployment and its first version exist" trace $CLI bundle deploy trace print_requests.py --dms //api/2.0/bundle +title "The plan is a no-op now and records DEPLOYMENT_METADATA_SERVICE; the service holds the schema at version 1" +trace $CLI bundle plan -o json | nostamp.py +trace print_state.py + title "A destructive change without --auto-approve is declined: this console cannot prompt" # Changing the catalog recreates the schema, which needs approval. # Unity Catalog has no catalog called "other", so make one to move the schema into. diff --git a/acceptance/bundle/dms/depends-on/output.txt b/acceptance/bundle/dms/depends-on/output.txt index b36e5cfb154..4fb3a8caab5 100644 --- a/acceptance/bundle/dms/depends-on/output.txt +++ b/acceptance/bundle/dms/depends-on/output.txt @@ -37,6 +37,105 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged } } +=== The plan is a no-op now and records DEPLOYMENT_METADATA_SERVICE as its backend +>>> [CLI] bundle plan -o json +{ + "plan_version": 3, + "cli_version": "[CLI_VERSION]", + "lineage": "[UUID]", + "serial": 1, + "storage_backend": "DEPLOYMENT_METADATA_SERVICE", + "plan": { + "resources.jobs.child": { + "depends_on": [ + { + "node": "resources.jobs.parent", + "label": "${resources.jobs.parent.id}" + } + ], + "action": "skip", + "remote_state": { + "created_time": [UNIX_TIME_MILLIS][0], + "creator_user_name": "[USERNAME]", + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-depends-on-[UNIQUE_NAME]/default/state/metadata.json" + }, + "description": "depends on [NUMID]", + "edit_mode": "UI_LOCKED", + "email_notifications": {}, + "format": "MULTI_TASK", + "job_id": [NUMID], + "max_concurrent_runs": 1, + "name": "child", + "queue": { + "enabled": true + }, + "run_as_user_name": "[USERNAME]", + "timeout_seconds": 0, + "webhook_notifications": {} + }, + "changes": { + "email_notifications": { + "action": "skip", + "reason": "empty", + "remote": {} + }, + "timeout_seconds": { + "action": "skip", + "reason": "empty", + "remote": 0 + }, + "webhook_notifications": { + "action": "skip", + "reason": "empty", + "remote": {} + } + } + }, + "resources.jobs.parent": { + "action": "skip", + "remote_state": { + "created_time": [UNIX_TIME_MILLIS][1], + "creator_user_name": "[USERNAME]", + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-depends-on-[UNIQUE_NAME]/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "email_notifications": {}, + "format": "MULTI_TASK", + "job_id": [NUMID], + "max_concurrent_runs": 1, + "name": "parent", + "queue": { + "enabled": true + }, + "run_as_user_name": "[USERNAME]", + "timeout_seconds": 0, + "webhook_notifications": {} + }, + "changes": { + "email_notifications": { + "action": "skip", + "reason": "empty", + "remote": {} + }, + "timeout_seconds": { + "action": "skip", + "reason": "empty", + "remote": 0 + }, + "webhook_notifications": { + "action": "skip", + "reason": "empty", + "remote": {} + } + } + } + } +} + === The service lists both jobs, each recorded at version 1 >>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[NUMID] -o json [ @@ -52,6 +151,63 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged } ] +=== The raw recorded state the service holds for this deployment +>>> print_state.py +{ + "state_version": 2, + "cli_version": "[CLI_VERSION]", + "lineage": "[UUID]", + "serial": 1, + "features": { + "record_deployment_history": {} + }, + "state": { + "resources.jobs.child": { + "__id__": "[NUMID]", + "state": { + "deployment": { + "deployment_id": "[NUMID]", + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-depends-on-[UNIQUE_NAME]/default/state/metadata.json", + "version_id": "1" + }, + "description": "depends on [NUMID]", + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "child", + "queue": { + "enabled": true + } + }, + "depends_on": [ + { + "node": "resources.jobs.parent", + "label": "${resources.jobs.parent.id}" + } + ] + }, + "resources.jobs.parent": { + "__id__": "[NUMID]", + "state": { + "deployment": { + "deployment_id": "[NUMID]", + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-depends-on-[UNIQUE_NAME]/default/state/metadata.json", + "version_id": "1" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "parent", + "queue": { + "enabled": true + } + } + } + } +} + >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: delete resources.jobs.child diff --git a/acceptance/bundle/dms/depends-on/script b/acceptance/bundle/dms/depends-on/script index dd168459b9a..022f8e3dd11 100644 --- a/acceptance/bundle/dms/depends-on/script +++ b/acceptance/bundle/dms/depends-on/script @@ -4,9 +4,15 @@ title "Deploy a job that references another: depends_on is recorded alongside th trace $CLI bundle deploy trace print_requests.py --dms //versions/1/operations +title "The plan is a no-op now and records DEPLOYMENT_METADATA_SERVICE as its backend" +trace $CLI bundle plan -o json | nostamp.py + title "The service lists both jobs, each recorded at version 1" deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-depends-on-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' +title "The raw recorded state the service holds for this deployment" +trace print_state.py + trace $CLI bundle destroy --auto-approve rm -f out.requests.txt diff --git a/acceptance/bundle/dms/deployment-metadata-change/output.txt b/acceptance/bundle/dms/deployment-metadata-change/output.txt index 0b0798eee7f..3422dde62d7 100644 --- a/acceptance/bundle/dms/deployment-metadata-change/output.txt +++ b/acceptance/bundle/dms/deployment-metadata-change/output.txt @@ -59,6 +59,50 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged } } +=== The plan records DEPLOYMENT_METADATA_SERVICE; the service holds the schema at version 1 +>>> [CLI] bundle plan -o json +{ + "plan_version": 3, + "cli_version": "[CLI_VERSION]", + "lineage": "[UUID]", + "serial": 1, + "storage_backend": "DEPLOYMENT_METADATA_SERVICE", + "plan": { + "resources.schemas.foo": { + "action": "skip", + "remote_state": { + "browse_only": false, + "catalog_name": "main", + "catalog_type": "MANAGED_CATALOG", + "created_at": [UNIX_TIME_MILLIS][0], + "created_by": "[USERNAME]", + "effective_predictive_optimization_flag": { + "inherited_from_name": "[METASTORE_NAME]", + "inherited_from_type": "METASTORE", + "value": "ENABLE" + }, + "enable_predictive_optimization": "INHERIT", + "full_name": "main.dms_metadata_schema_[UNIQUE_NAME]", + "metastore_id": "[UUID]", + "name": "dms_metadata_schema_[UNIQUE_NAME]", + "owner": "[USERNAME]", + "schema_id": "[UUID]", + "updated_at": [UNIX_TIME_MILLIS][0], + "updated_by": "[USERNAME]" + } + } + } +} + +>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[NUMID] -o json +[ + { + "last_action_type": "OPERATION_ACTION_TYPE_CREATE", + "last_version_id": "1", + "resource_key": "schemas.foo" + } +] + === Rename the bundle and switch target+mode. The state path is pinned, so this is the same deployment, updated in place - the mask carries every field that changed >>> update_file.py databricks.yml dms-metadata-before dms-metadata-after @@ -110,6 +154,56 @@ Resources: 0 created, 0 changed, 0 deleted, 1 unchanged } } +=== The metadata changed but the schema did not: still one deployment, recorded at version 1 +>>> [CLI] bundle plan -t prod -o json +Recommendation: target with 'mode: production' should set 'workspace.root_path' to make sure only one copy is deployed + +A common practice is to use a username or principal name in this path, i.e. use + + root_path: /Workspace/Users/[USERNAME]/.bundle/${bundle.name}/${bundle.target} + +{ + "plan_version": 3, + "cli_version": "[CLI_VERSION]", + "lineage": "[UUID]", + "serial": 1, + "storage_backend": "DEPLOYMENT_METADATA_SERVICE", + "plan": { + "resources.schemas.foo": { + "action": "skip", + "remote_state": { + "browse_only": false, + "catalog_name": "main", + "catalog_type": "MANAGED_CATALOG", + "created_at": [UNIX_TIME_MILLIS][0], + "created_by": "[USERNAME]", + "effective_predictive_optimization_flag": { + "inherited_from_name": "[METASTORE_NAME]", + "inherited_from_type": "METASTORE", + "value": "ENABLE" + }, + "enable_predictive_optimization": "INHERIT", + "full_name": "main.dms_metadata_schema_[UNIQUE_NAME]", + "metastore_id": "[UUID]", + "name": "dms_metadata_schema_[UNIQUE_NAME]", + "owner": "[USERNAME]", + "schema_id": "[UUID]", + "updated_at": [UNIX_TIME_MILLIS][0], + "updated_by": "[USERNAME]" + } + } + } +} + +>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[NUMID] -o json +[ + { + "last_action_type": "OPERATION_ACTION_TYPE_CREATE", + "last_version_id": "1", + "resource_key": "schemas.foo" + } +] + === Deploy again without touching anything: the record already says this, so no deployment update >>> [CLI] bundle deploy -t prod Recommendation: target with 'mode: production' should set 'workspace.root_path' to make sure only one copy is deployed @@ -142,3 +236,13 @@ Resources: 0 created, 0 changed, 0 deleted, 1 unchanged "completion_reason": "VERSION_COMPLETE_SUCCESS" } } + +=== The no-op redeploy leaves the schema at version 1 +>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[NUMID] -o json +[ + { + "last_action_type": "OPERATION_ACTION_TYPE_CREATE", + "last_version_id": "1", + "resource_key": "schemas.foo" + } +] diff --git a/acceptance/bundle/dms/deployment-metadata-change/script b/acceptance/bundle/dms/deployment-metadata-change/script index 03f0c593999..45de9533cab 100644 --- a/acceptance/bundle/dms/deployment-metadata-change/script +++ b/acceptance/bundle/dms/deployment-metadata-change/script @@ -4,11 +4,24 @@ title "Deploy: the deployment is created carrying this bundle's metadata" trace $CLI bundle deploy trace print_requests.py --dms //api/2.0/bundle +title "The plan records DEPLOYMENT_METADATA_SERVICE; the service holds the schema at version 1" +trace $CLI bundle plan -o json | nostamp.py +deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-metadata-pinned-${UNIQUE_NAME}/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') +trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' + title "Rename the bundle and switch target+mode. The state path is pinned, so this is the same deployment, updated in place - the mask carries every field that changed" trace update_file.py databricks.yml dms-metadata-before dms-metadata-after trace $CLI bundle deploy -t prod trace print_requests.py --dms //api/2.0/bundle +title "The metadata changed but the schema did not: still one deployment, recorded at version 1" +trace $CLI bundle plan -t prod -o json | nostamp.py +trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' + title "Deploy again without touching anything: the record already says this, so no deployment update" trace $CLI bundle deploy -t prod trace print_requests.py --dms //api/2.0/bundle + +title "The no-op redeploy leaves the schema at version 1" +trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' +rm -f out.requests.txt From eaa31f4dc21b55211fe8e7275bf826802b60849c Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Fri, 4 Sep 2026 13:10:45 +0000 Subject: [PATCH 16/46] acceptance/dms: record plan JSON, list-resources, and raw recorded state across the suite Extend no-resources, multiple-resources, record, successful-recreate, no-drift, stale-plan, emptied-resource and provenance to capture the serialized 'bundle plan -o json' (now carrying storage_backend), the DMS list-resources view, and the raw recorded state at each steady-state point. no-drift shows only the updated resource bumping to version 2; successful-recreate shows the recreate at version 2; provenance masks the git commit. Co-authored-by: Isaac --- .../bundle/dms/emptied-resource/output.txt | 56 ++ acceptance/bundle/dms/emptied-resource/script | 6 + .../bundle/dms/multiple-resources/output.txt | 692 ++++++++++++++++++ .../bundle/dms/multiple-resources/script | 22 + acceptance/bundle/dms/no-drift/output.txt | 276 +++++++ acceptance/bundle/dms/no-drift/script | 14 +- acceptance/bundle/dms/no-resources/output.txt | 46 +- acceptance/bundle/dms/no-resources/script | 22 +- acceptance/bundle/dms/provenance/output.txt | 100 +++ acceptance/bundle/dms/provenance/script | 9 + acceptance/bundle/dms/record/output.txt | 94 +++ acceptance/bundle/dms/record/script | 11 + acceptance/bundle/dms/stale-plan/output.txt | 66 ++ acceptance/bundle/dms/stale-plan/script | 11 +- .../bundle/dms/successful-recreate/output.txt | 136 ++++ .../bundle/dms/successful-recreate/script | 20 + 16 files changed, 1571 insertions(+), 10 deletions(-) diff --git a/acceptance/bundle/dms/emptied-resource/output.txt b/acceptance/bundle/dms/emptied-resource/output.txt index 7d0ad36a596..297b8e441db 100644 --- a/acceptance/bundle/dms/emptied-resource/output.txt +++ b/acceptance/bundle/dms/emptied-resource/output.txt @@ -130,6 +130,41 @@ Resources: 0 created, 1 changed, 0 deleted, 1 unchanged >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged +=== The plan is a no-op and records DEPLOYMENT_METADATA_SERVICE as its backend +>>> [CLI] bundle plan -o json +{ + "plan_version": 3, + "cli_version": "[CLI_VERSION]", + "lineage": "[UUID]", + "serial": 2, + "storage_backend": "DEPLOYMENT_METADATA_SERVICE", + "plan": { + "resources.schemas.foo": { + "action": "skip", + "remote_state": { + "browse_only": false, + "catalog_name": "main", + "catalog_type": "MANAGED_CATALOG", + "created_at": [UNIX_TIME_MILLIS][0], + "created_by": "[USERNAME]", + "effective_predictive_optimization_flag": { + "inherited_from_name": "[METASTORE_NAME]", + "inherited_from_type": "METASTORE", + "value": "ENABLE" + }, + "enable_predictive_optimization": "INHERIT", + "full_name": "main.dms_emptied_resource_[UNIQUE_NAME]", + "metastore_id": "[UUID]", + "name": "dms_emptied_resource_[UNIQUE_NAME]", + "owner": "[USERNAME]", + "schema_id": "[UUID]", + "updated_at": [UNIX_TIME_MILLIS][1], + "updated_by": "[USERNAME]" + } + } + } +} + === The grants node emptied out, so the service no longer lists it - only the schema remains >>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[NUMID] -o json [ @@ -140,6 +175,27 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged } ] +=== The raw recorded state the service holds for this deployment +>>> print_state.py +{ + "state_version": 2, + "cli_version": "[CLI_VERSION]", + "lineage": "[UUID]", + "serial": 2, + "features": { + "record_deployment_history": {} + }, + "state": { + "resources.schemas.foo": { + "__id__": "main.dms_emptied_resource_[UNIQUE_NAME]", + "state": { + "catalog_name": "main", + "name": "dms_emptied_resource_[UNIQUE_NAME]" + } + } + } +} + >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: delete resources.schemas.foo diff --git a/acceptance/bundle/dms/emptied-resource/script b/acceptance/bundle/dms/emptied-resource/script index 1e0606a31c0..6e5af06cc31 100644 --- a/acceptance/bundle/dms/emptied-resource/script +++ b/acceptance/bundle/dms/emptied-resource/script @@ -16,9 +16,15 @@ trace print_requests.py --dms //api/2.0/bundle title "Plan again: reading state back from the service works and reports no work" trace $CLI bundle plan | contains.py "Plan: 0 to add, 0 to change, 0 to delete" "!unexpected end of JSON input" +title "The plan is a no-op and records DEPLOYMENT_METADATA_SERVICE as its backend" +trace $CLI bundle plan -o json | nostamp.py + title "The grants node emptied out, so the service no longer lists it - only the schema remains" deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-emptied-resource-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' +title "The raw recorded state the service holds for this deployment" +trace print_state.py + trace $CLI bundle destroy --auto-approve rm -f out.requests.txt diff --git a/acceptance/bundle/dms/multiple-resources/output.txt b/acceptance/bundle/dms/multiple-resources/output.txt index ecc2092225a..4a737c37b6e 100644 --- a/acceptance/bundle/dms/multiple-resources/output.txt +++ b/acceptance/bundle/dms/multiple-resources/output.txt @@ -17,6 +17,352 @@ Resources: 5 created, 0 changed, 0 deleted, 0 unchanged {"method": "PATCH", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.three", "q": {"update_mask": "state,error_message,resource_id,status"}, "body": {"error_message": "", "resource_id": "[NUMID]", "sequence_id": "0", "status": "OPERATION_STATUS_SUCCEEDED"}} {"method": "PATCH", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.two", "q": {"update_mask": "state,error_message,resource_id,status"}, "body": {"error_message": "", "resource_id": "[NUMID]", "sequence_id": "0", "status": "OPERATION_STATUS_SUCCEEDED"}} +=== The plan records DEPLOYMENT_METADATA_SERVICE as its backend +>>> [CLI] bundle plan -o json +{ + "plan_version": 3, + "cli_version": "[CLI_VERSION]", + "lineage": "[UUID]", + "serial": 1, + "storage_backend": "DEPLOYMENT_METADATA_SERVICE", + "plan": { + "resources.jobs.five": { + "action": "skip", + "remote_state": { + "created_time": [UNIX_TIME_MILLIS][0], + "creator_user_name": "[USERNAME]", + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "email_notifications": {}, + "format": "MULTI_TASK", + "job_id": [NUMID], + "max_concurrent_runs": 1, + "name": "five", + "queue": { + "enabled": true + }, + "run_as_user_name": "[USERNAME]", + "timeout_seconds": 0, + "webhook_notifications": {} + }, + "changes": { + "email_notifications": { + "action": "skip", + "reason": "empty", + "remote": {} + }, + "timeout_seconds": { + "action": "skip", + "reason": "empty", + "remote": 0 + }, + "webhook_notifications": { + "action": "skip", + "reason": "empty", + "remote": {} + } + } + }, + "resources.jobs.four": { + "action": "skip", + "remote_state": { + "created_time": [UNIX_TIME_MILLIS][1], + "creator_user_name": "[USERNAME]", + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "email_notifications": {}, + "format": "MULTI_TASK", + "job_id": [NUMID], + "max_concurrent_runs": 1, + "name": "four", + "queue": { + "enabled": true + }, + "run_as_user_name": "[USERNAME]", + "timeout_seconds": 0, + "webhook_notifications": {} + }, + "changes": { + "email_notifications": { + "action": "skip", + "reason": "empty", + "remote": {} + }, + "timeout_seconds": { + "action": "skip", + "reason": "empty", + "remote": 0 + }, + "webhook_notifications": { + "action": "skip", + "reason": "empty", + "remote": {} + } + } + }, + "resources.jobs.one": { + "action": "skip", + "remote_state": { + "created_time": [UNIX_TIME_MILLIS][2], + "creator_user_name": "[USERNAME]", + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "email_notifications": {}, + "format": "MULTI_TASK", + "job_id": [NUMID], + "max_concurrent_runs": 1, + "name": "one", + "queue": { + "enabled": true + }, + "run_as_user_name": "[USERNAME]", + "timeout_seconds": 0, + "webhook_notifications": {} + }, + "changes": { + "email_notifications": { + "action": "skip", + "reason": "empty", + "remote": {} + }, + "timeout_seconds": { + "action": "skip", + "reason": "empty", + "remote": 0 + }, + "webhook_notifications": { + "action": "skip", + "reason": "empty", + "remote": {} + } + } + }, + "resources.jobs.three": { + "action": "skip", + "remote_state": { + "created_time": [UNIX_TIME_MILLIS][3], + "creator_user_name": "[USERNAME]", + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "email_notifications": {}, + "format": "MULTI_TASK", + "job_id": [NUMID], + "max_concurrent_runs": 1, + "name": "three", + "queue": { + "enabled": true + }, + "run_as_user_name": "[USERNAME]", + "timeout_seconds": 0, + "webhook_notifications": {} + }, + "changes": { + "email_notifications": { + "action": "skip", + "reason": "empty", + "remote": {} + }, + "timeout_seconds": { + "action": "skip", + "reason": "empty", + "remote": 0 + }, + "webhook_notifications": { + "action": "skip", + "reason": "empty", + "remote": {} + } + } + }, + "resources.jobs.two": { + "action": "skip", + "remote_state": { + "created_time": [UNIX_TIME_MILLIS][4], + "creator_user_name": "[USERNAME]", + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "email_notifications": {}, + "format": "MULTI_TASK", + "job_id": [NUMID], + "max_concurrent_runs": 1, + "name": "two", + "queue": { + "enabled": true + }, + "run_as_user_name": "[USERNAME]", + "timeout_seconds": 0, + "webhook_notifications": {} + }, + "changes": { + "email_notifications": { + "action": "skip", + "reason": "empty", + "remote": {} + }, + "timeout_seconds": { + "action": "skip", + "reason": "empty", + "remote": 0 + }, + "webhook_notifications": { + "action": "skip", + "reason": "empty", + "remote": {} + } + } + } + } +} + +=== The service lists all five resources at version 1 +>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[NUMID] -o json +[ + { + "last_action_type": "OPERATION_ACTION_TYPE_CREATE", + "last_version_id": "1", + "resource_key": "jobs.five" + }, + { + "last_action_type": "OPERATION_ACTION_TYPE_CREATE", + "last_version_id": "1", + "resource_key": "jobs.four" + }, + { + "last_action_type": "OPERATION_ACTION_TYPE_CREATE", + "last_version_id": "1", + "resource_key": "jobs.one" + }, + { + "last_action_type": "OPERATION_ACTION_TYPE_CREATE", + "last_version_id": "1", + "resource_key": "jobs.three" + }, + { + "last_action_type": "OPERATION_ACTION_TYPE_CREATE", + "last_version_id": "1", + "resource_key": "jobs.two" + } +] + +=== The raw recorded state the service holds for this deployment +>>> print_state.py +{ + "state_version": 2, + "cli_version": "[CLI_VERSION]", + "lineage": "[UUID]", + "serial": 1, + "features": { + "record_deployment_history": {} + }, + "state": { + "resources.jobs.five": { + "__id__": "[NUMID]", + "state": { + "deployment": { + "deployment_id": "[NUMID]", + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/state/metadata.json", + "version_id": "1" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "five", + "queue": { + "enabled": true + } + } + }, + "resources.jobs.four": { + "__id__": "[NUMID]", + "state": { + "deployment": { + "deployment_id": "[NUMID]", + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/state/metadata.json", + "version_id": "1" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "four", + "queue": { + "enabled": true + } + } + }, + "resources.jobs.one": { + "__id__": "[NUMID]", + "state": { + "deployment": { + "deployment_id": "[NUMID]", + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/state/metadata.json", + "version_id": "1" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "one", + "queue": { + "enabled": true + } + } + }, + "resources.jobs.three": { + "__id__": "[NUMID]", + "state": { + "deployment": { + "deployment_id": "[NUMID]", + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/state/metadata.json", + "version_id": "1" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "three", + "queue": { + "enabled": true + } + } + }, + "resources.jobs.two": { + "__id__": "[NUMID]", + "state": { + "deployment": { + "deployment_id": "[NUMID]", + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/state/metadata.json", + "version_id": "1" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "two", + "queue": { + "enabled": true + } + } + } + } +} + === Redeploy with no changes: nothing is applied, so no operations are recorded and only the version is opened and completed >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/files... @@ -26,3 +372,349 @@ Resources: 0 created, 0 changed, 0 deleted, 5 unchanged >>> print_requests.py --dms //api/2.0/bundle --sort --oneline {"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": {"version_id": "2"}, "body": {"cli_version": "[CLI_VERSION]", "version_type": "VERSION_TYPE_DEPLOY", "previous_version_id": "1"}} {"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/complete", "body": {"completion_reason": "VERSION_COMPLETE_SUCCESS"}} + +=== The plan records DEPLOYMENT_METADATA_SERVICE as its backend +>>> [CLI] bundle plan -o json +{ + "plan_version": 3, + "cli_version": "[CLI_VERSION]", + "lineage": "[UUID]", + "serial": 1, + "storage_backend": "DEPLOYMENT_METADATA_SERVICE", + "plan": { + "resources.jobs.five": { + "action": "skip", + "remote_state": { + "created_time": [UNIX_TIME_MILLIS][0], + "creator_user_name": "[USERNAME]", + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "email_notifications": {}, + "format": "MULTI_TASK", + "job_id": [NUMID], + "max_concurrent_runs": 1, + "name": "five", + "queue": { + "enabled": true + }, + "run_as_user_name": "[USERNAME]", + "timeout_seconds": 0, + "webhook_notifications": {} + }, + "changes": { + "email_notifications": { + "action": "skip", + "reason": "empty", + "remote": {} + }, + "timeout_seconds": { + "action": "skip", + "reason": "empty", + "remote": 0 + }, + "webhook_notifications": { + "action": "skip", + "reason": "empty", + "remote": {} + } + } + }, + "resources.jobs.four": { + "action": "skip", + "remote_state": { + "created_time": [UNIX_TIME_MILLIS][1], + "creator_user_name": "[USERNAME]", + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "email_notifications": {}, + "format": "MULTI_TASK", + "job_id": [NUMID], + "max_concurrent_runs": 1, + "name": "four", + "queue": { + "enabled": true + }, + "run_as_user_name": "[USERNAME]", + "timeout_seconds": 0, + "webhook_notifications": {} + }, + "changes": { + "email_notifications": { + "action": "skip", + "reason": "empty", + "remote": {} + }, + "timeout_seconds": { + "action": "skip", + "reason": "empty", + "remote": 0 + }, + "webhook_notifications": { + "action": "skip", + "reason": "empty", + "remote": {} + } + } + }, + "resources.jobs.one": { + "action": "skip", + "remote_state": { + "created_time": [UNIX_TIME_MILLIS][2], + "creator_user_name": "[USERNAME]", + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "email_notifications": {}, + "format": "MULTI_TASK", + "job_id": [NUMID], + "max_concurrent_runs": 1, + "name": "one", + "queue": { + "enabled": true + }, + "run_as_user_name": "[USERNAME]", + "timeout_seconds": 0, + "webhook_notifications": {} + }, + "changes": { + "email_notifications": { + "action": "skip", + "reason": "empty", + "remote": {} + }, + "timeout_seconds": { + "action": "skip", + "reason": "empty", + "remote": 0 + }, + "webhook_notifications": { + "action": "skip", + "reason": "empty", + "remote": {} + } + } + }, + "resources.jobs.three": { + "action": "skip", + "remote_state": { + "created_time": [UNIX_TIME_MILLIS][3], + "creator_user_name": "[USERNAME]", + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "email_notifications": {}, + "format": "MULTI_TASK", + "job_id": [NUMID], + "max_concurrent_runs": 1, + "name": "three", + "queue": { + "enabled": true + }, + "run_as_user_name": "[USERNAME]", + "timeout_seconds": 0, + "webhook_notifications": {} + }, + "changes": { + "email_notifications": { + "action": "skip", + "reason": "empty", + "remote": {} + }, + "timeout_seconds": { + "action": "skip", + "reason": "empty", + "remote": 0 + }, + "webhook_notifications": { + "action": "skip", + "reason": "empty", + "remote": {} + } + } + }, + "resources.jobs.two": { + "action": "skip", + "remote_state": { + "created_time": [UNIX_TIME_MILLIS][4], + "creator_user_name": "[USERNAME]", + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "email_notifications": {}, + "format": "MULTI_TASK", + "job_id": [NUMID], + "max_concurrent_runs": 1, + "name": "two", + "queue": { + "enabled": true + }, + "run_as_user_name": "[USERNAME]", + "timeout_seconds": 0, + "webhook_notifications": {} + }, + "changes": { + "email_notifications": { + "action": "skip", + "reason": "empty", + "remote": {} + }, + "timeout_seconds": { + "action": "skip", + "reason": "empty", + "remote": 0 + }, + "webhook_notifications": { + "action": "skip", + "reason": "empty", + "remote": {} + } + } + } + } +} + +=== The service still lists all five resources, now at version 2 +>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[NUMID] -o json +[ + { + "last_action_type": "OPERATION_ACTION_TYPE_CREATE", + "last_version_id": "1", + "resource_key": "jobs.five" + }, + { + "last_action_type": "OPERATION_ACTION_TYPE_CREATE", + "last_version_id": "1", + "resource_key": "jobs.four" + }, + { + "last_action_type": "OPERATION_ACTION_TYPE_CREATE", + "last_version_id": "1", + "resource_key": "jobs.one" + }, + { + "last_action_type": "OPERATION_ACTION_TYPE_CREATE", + "last_version_id": "1", + "resource_key": "jobs.three" + }, + { + "last_action_type": "OPERATION_ACTION_TYPE_CREATE", + "last_version_id": "1", + "resource_key": "jobs.two" + } +] + +=== The raw recorded state after redeploy +>>> print_state.py +{ + "state_version": 2, + "cli_version": "[CLI_VERSION]", + "lineage": "[UUID]", + "serial": 1, + "features": { + "record_deployment_history": {} + }, + "state": { + "resources.jobs.five": { + "__id__": "[NUMID]", + "state": { + "deployment": { + "deployment_id": "[NUMID]", + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/state/metadata.json", + "version_id": "1" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "five", + "queue": { + "enabled": true + } + } + }, + "resources.jobs.four": { + "__id__": "[NUMID]", + "state": { + "deployment": { + "deployment_id": "[NUMID]", + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/state/metadata.json", + "version_id": "1" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "four", + "queue": { + "enabled": true + } + } + }, + "resources.jobs.one": { + "__id__": "[NUMID]", + "state": { + "deployment": { + "deployment_id": "[NUMID]", + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/state/metadata.json", + "version_id": "1" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "one", + "queue": { + "enabled": true + } + } + }, + "resources.jobs.three": { + "__id__": "[NUMID]", + "state": { + "deployment": { + "deployment_id": "[NUMID]", + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/state/metadata.json", + "version_id": "1" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "three", + "queue": { + "enabled": true + } + } + }, + "resources.jobs.two": { + "__id__": "[NUMID]", + "state": { + "deployment": { + "deployment_id": "[NUMID]", + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/state/metadata.json", + "version_id": "1" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "two", + "queue": { + "enabled": true + } + } + } + } +} diff --git a/acceptance/bundle/dms/multiple-resources/script b/acceptance/bundle/dms/multiple-resources/script index 584423a4d5e..78ab3028390 100644 --- a/acceptance/bundle/dms/multiple-resources/script +++ b/acceptance/bundle/dms/multiple-resources/script @@ -4,6 +4,28 @@ title "Deploy several resources: operations are uploaded from background workers trace $CLI bundle deploy trace print_requests.py --dms //versions/1/operations --sort --del-body state --oneline +title "The plan records DEPLOYMENT_METADATA_SERVICE as its backend" +trace $CLI bundle plan -o json | nostamp.py + +title "The service lists all five resources at version 1" +deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-multiple-resources-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') +trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' + +title "The raw recorded state the service holds for this deployment" +trace print_state.py + title "Redeploy with no changes: nothing is applied, so no operations are recorded and only the version is opened and completed" trace $CLI bundle deploy trace print_requests.py --dms //api/2.0/bundle --sort --oneline + +title "The plan records DEPLOYMENT_METADATA_SERVICE as its backend" +trace $CLI bundle plan -o json | nostamp.py + +title "The service still lists all five resources, now at version 2" +deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-multiple-resources-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') +trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' + +title "The raw recorded state after redeploy" +trace print_state.py + +rm -f out.requests.txt diff --git a/acceptance/bundle/dms/no-drift/output.txt b/acceptance/bundle/dms/no-drift/output.txt index 952b5b2731e..87d254cf2e2 100644 --- a/acceptance/bundle/dms/no-drift/output.txt +++ b/acceptance/bundle/dms/no-drift/output.txt @@ -27,6 +27,143 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged "latest_version_id": "1" } +=== The plan is a no-op now and records DEPLOYMENT_METADATA_SERVICE as its backend +>>> [CLI] bundle plan -o json +{ + "plan_version": 3, + "cli_version": "[CLI_VERSION]", + "lineage": "[UUID]", + "serial": 1, + "storage_backend": "DEPLOYMENT_METADATA_SERVICE", + "plan": { + "resources.jobs.foo": { + "action": "skip", + "remote_state": { + "created_time": [UNIX_TIME_MILLIS][0], + "creator_user_name": "[USERNAME]", + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-no-drift-[UNIQUE_NAME]/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "email_notifications": {}, + "format": "MULTI_TASK", + "job_id": [NUMID], + "max_concurrent_runs": 1, + "name": "foo", + "queue": { + "enabled": true + }, + "run_as_user_name": "[USERNAME]", + "timeout_seconds": 0, + "webhook_notifications": {} + }, + "changes": { + "email_notifications": { + "action": "skip", + "reason": "empty", + "remote": {} + }, + "timeout_seconds": { + "action": "skip", + "reason": "empty", + "remote": 0 + }, + "webhook_notifications": { + "action": "skip", + "reason": "empty", + "remote": {} + } + } + }, + "resources.pipelines.bar": { + "action": "skip", + "remote_state": { + "catalog": "main", + "channel": "CURRENT", + "creator_user_name": "[USERNAME]", + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-no-drift-[UNIQUE_NAME]/default/state/metadata.json" + }, + "edition": "ADVANCED", + "effective_publishing_mode": "DEFAULT_PUBLISHING_MODE", + "id": "[UUID]", + "last_modified": [UNIX_TIME_MILLIS][1], + "libraries": [ + { + "file": { + "path": "/Workspace/Users/[USERNAME]/.bundle/dms-no-drift-[UNIQUE_NAME]/default/files/transform.py" + } + } + ], + "name": "bar-[UNIQUE_NAME]", + "pipeline_id": "[UUID]", + "run_as_user_name": "[USERNAME]", + "schema": "default", + "serverless": true, + "state": "IDLE" + } + } + } +} + +=== The raw recorded state the service holds for this deployment +>>> print_state.py +{ + "state_version": 2, + "cli_version": "[CLI_VERSION]", + "lineage": "[UUID]", + "serial": 1, + "features": { + "record_deployment_history": {} + }, + "state": { + "resources.jobs.foo": { + "__id__": "[NUMID]", + "state": { + "deployment": { + "deployment_id": "[NUMID]", + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-no-drift-[UNIQUE_NAME]/default/state/metadata.json", + "version_id": "1" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "foo", + "queue": { + "enabled": true + } + } + }, + "resources.pipelines.bar": { + "__id__": "[UUID]", + "state": { + "catalog": "main", + "channel": "CURRENT", + "deployment": { + "deployment_id": "[NUMID]", + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-no-drift-[UNIQUE_NAME]/default/state/metadata.json", + "version_id": "1" + }, + "edition": "ADVANCED", + "libraries": [ + { + "file": { + "path": "/Workspace/Users/[USERNAME]/.bundle/dms-no-drift-[UNIQUE_NAME]/default/files/transform.py" + } + } + ], + "name": "bar-[UNIQUE_NAME]", + "schema": "default", + "serverless": true + } + } + } +} + === Plan without touching anything: the deployment stamp must not show as drift >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged @@ -133,6 +270,145 @@ Resources: 0 created, 1 changed, 0 deleted, 1 unchanged "latest_version_id": "2" } +=== The plan shows version 2 for the updated pipeline and DEPLOYMENT_METADATA_SERVICE as backend +>>> [CLI] bundle plan -o json +{ + "plan_version": 3, + "cli_version": "[CLI_VERSION]", + "lineage": "[UUID]", + "serial": 2, + "storage_backend": "DEPLOYMENT_METADATA_SERVICE", + "plan": { + "resources.jobs.foo": { + "action": "skip", + "remote_state": { + "created_time": [UNIX_TIME_MILLIS][0], + "creator_user_name": "[USERNAME]", + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-no-drift-[UNIQUE_NAME]/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "email_notifications": {}, + "format": "MULTI_TASK", + "job_id": [NUMID], + "max_concurrent_runs": 1, + "name": "foo", + "queue": { + "enabled": true + }, + "run_as_user_name": "[USERNAME]", + "timeout_seconds": 0, + "webhook_notifications": {} + }, + "changes": { + "email_notifications": { + "action": "skip", + "reason": "empty", + "remote": {} + }, + "timeout_seconds": { + "action": "skip", + "reason": "empty", + "remote": 0 + }, + "webhook_notifications": { + "action": "skip", + "reason": "empty", + "remote": {} + } + } + }, + "resources.pipelines.bar": { + "action": "skip", + "remote_state": { + "catalog": "main", + "channel": "CURRENT", + "creator_user_name": "[USERNAME]", + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-no-drift-[UNIQUE_NAME]/default/state/metadata.json" + }, + "development": true, + "edition": "ADVANCED", + "effective_publishing_mode": "DEFAULT_PUBLISHING_MODE", + "id": "[UUID]", + "last_modified": [UNIX_TIME_MILLIS][1], + "libraries": [ + { + "file": { + "path": "/Workspace/Users/[USERNAME]/.bundle/dms-no-drift-[UNIQUE_NAME]/default/files/transform.py" + } + } + ], + "name": "bar-[UNIQUE_NAME]", + "pipeline_id": "[UUID]", + "run_as_user_name": "[USERNAME]", + "schema": "default", + "serverless": true, + "state": "IDLE" + } + } + } +} + +=== The raw recorded state the service holds after the update +>>> print_state.py +{ + "state_version": 2, + "cli_version": "[CLI_VERSION]", + "lineage": "[UUID]", + "serial": 2, + "features": { + "record_deployment_history": {} + }, + "state": { + "resources.jobs.foo": { + "__id__": "[NUMID]", + "state": { + "deployment": { + "deployment_id": "[NUMID]", + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-no-drift-[UNIQUE_NAME]/default/state/metadata.json", + "version_id": "1" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "foo", + "queue": { + "enabled": true + } + } + }, + "resources.pipelines.bar": { + "__id__": "[UUID]", + "state": { + "catalog": "main", + "channel": "CURRENT", + "deployment": { + "deployment_id": "[NUMID]", + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-no-drift-[UNIQUE_NAME]/default/state/metadata.json", + "version_id": "2" + }, + "development": true, + "edition": "ADVANCED", + "libraries": [ + { + "file": { + "path": "/Workspace/Users/[USERNAME]/.bundle/dms-no-drift-[UNIQUE_NAME]/default/files/transform.py" + } + } + ], + "name": "bar-[UNIQUE_NAME]", + "schema": "default", + "serverless": true + } + } + } +} + === Plan again: no residual drift, and the version stamps are ignored >>> [CLI] bundle plan Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged diff --git a/acceptance/bundle/dms/no-drift/script b/acceptance/bundle/dms/no-drift/script index dfcb0b55ef5..55bff445646 100644 --- a/acceptance/bundle/dms/no-drift/script +++ b/acceptance/bundle/dms/no-drift/script @@ -7,6 +7,12 @@ deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/$ trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' trace $CLI bundle summary -o json | jq .bundle.deployment.history +title "The plan is a no-op now and records DEPLOYMENT_METADATA_SERVICE as its backend" +trace $CLI bundle plan -o json | nostamp.py + +title "The raw recorded state the service holds for this deployment" +trace print_state.py + # Only deploy stamps deployment.deployment_id. Plan sees it in state but not in config, # so it must ignore it rather than report spurious changes. title "Plan without touching anything: the deployment stamp must not show as drift" @@ -20,9 +26,15 @@ trace print_requests.py --dms //api/2.2/jobs //api/2.0/pipelines --sort trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' trace $CLI bundle summary -o json | jq .bundle.deployment.history +title "The plan shows version 2 for the updated pipeline and DEPLOYMENT_METADATA_SERVICE as backend" +trace $CLI bundle plan -o json | nostamp.py + +title "The raw recorded state the service holds after the update" +trace print_state.py + title "Plan again: no residual drift, and the version stamps are ignored" trace $CLI bundle plan | contains.py "Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged" -# The list-resources and final plan above record requests we do not assert on; drop them so +# The list-resources and plan/state captures above record requests we do not assert on; drop them so # out.requests.txt does not become a nondeterministic golden. rm -f out.requests.txt diff --git a/acceptance/bundle/dms/no-resources/output.txt b/acceptance/bundle/dms/no-resources/output.txt index 5c80f26a912..f75ba46e2fa 100644 --- a/acceptance/bundle/dms/no-resources/output.txt +++ b/acceptance/bundle/dms/no-resources/output.txt @@ -44,6 +44,22 @@ Resources: 0 created, 0 changed, 0 deleted, 0 unchanged "path": "/Workspace/Users/[USERNAME]/.bundle/dms-no-resources-[UNIQUE_NAME]/default/state/resources.deployment.json" } +=== The plan records DEPLOYMENT_METADATA_SERVICE as its backend +>>> [CLI] bundle plan -o json +{ + "plan_version": 3, + "cli_version": "[CLI_VERSION]", + "storage_backend": "DEPLOYMENT_METADATA_SERVICE", + "plan": {} +} + +=== The service lists no resources for the deployment +>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[NUMID] -o json +[] + +=== The raw recorded state the service holds for this deployment +>>> print_state.py + === Redeploy: the deployment is resolved from that node, so no second deployment is created >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-no-resources-[UNIQUE_NAME]/default/files... @@ -59,6 +75,18 @@ Resources: 0 created, 0 changed, 0 deleted, 0 unchanged "method": "GET", "path": "/api/2.0/bundle/deployments/[NUMID]/resources" } +{ + "method": "GET", + "path": "/api/2.0/bundle/deployments/[NUMID]/resources" +} +{ + "method": "GET", + "path": "/api/2.0/bundle/deployments/[NUMID]" +} +{ + "method": "GET", + "path": "/api/2.0/bundle/deployments/[NUMID]/resources" +} { "method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", @@ -79,8 +107,18 @@ Resources: 0 created, 0 changed, 0 deleted, 0 unchanged } } -=== The service lists no resources for the deployment: nothing was recorded beyond the deployment itself +=== The plan records DEPLOYMENT_METADATA_SERVICE as its backend +>>> [CLI] bundle plan -o json +{ + "plan_version": 3, + "cli_version": "[CLI_VERSION]", + "storage_backend": "DEPLOYMENT_METADATA_SERVICE", + "plan": {} +} + +=== The service still lists no resources >>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[NUMID] -o json -[ - -] +[] + +=== The raw recorded state after redeploy +>>> print_state.py diff --git a/acceptance/bundle/dms/no-resources/script b/acceptance/bundle/dms/no-resources/script index 81444e00ccf..001d1e835f5 100644 --- a/acceptance/bundle/dms/no-resources/script +++ b/acceptance/bundle/dms/no-resources/script @@ -5,12 +5,28 @@ trace $CLI bundle deploy trace print_requests.py --dms //api/2.0/bundle --get trace MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-no-resources-${UNIQUE_NAME}/default/state/resources.deployment.json" | jq '{object_type,path}' +title "The plan records DEPLOYMENT_METADATA_SERVICE as its backend" +trace $CLI bundle plan -o json | nostamp.py + +title "The service lists no resources for the deployment" +deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-no-resources-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') +trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' + +title "The raw recorded state the service holds for this deployment" +trace print_state.py + title "Redeploy: the deployment is resolved from that node, so no second deployment is created" trace $CLI bundle deploy trace print_requests.py --dms //api/2.0/bundle --get -title "The service lists no resources for the deployment: nothing was recorded beyond the deployment itself" +title "The plan records DEPLOYMENT_METADATA_SERVICE as its backend" +trace $CLI bundle plan -o json | nostamp.py + +title "The service still lists no resources" deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-no-resources-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') -trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json -# Discard the get-status + list-resources reads above so out.requests.txt is not a golden. +trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' + +title "The raw recorded state after redeploy" +trace print_state.py + rm -f out.requests.txt diff --git a/acceptance/bundle/dms/provenance/output.txt b/acceptance/bundle/dms/provenance/output.txt index 2142dabb700..df026816182 100644 --- a/acceptance/bundle/dms/provenance/output.txt +++ b/acceptance/bundle/dms/provenance/output.txt @@ -51,6 +51,61 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged } } +=== The plan is a no-op and records DEPLOYMENT_METADATA_SERVICE as its backend +>>> [CLI] bundle plan -o json +{ + "plan_version": 3, + "cli_version": "[CLI_VERSION]", + "lineage": "[UUID]", + "serial": 1, + "storage_backend": "DEPLOYMENT_METADATA_SERVICE", + "plan": { + "resources.jobs.foo": { + "action": "skip", + "remote_state": { + "created_time": [UNIX_TIME_MILLIS], + "creator_user_name": "[USERNAME]", + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-provenance-[UNIQUE_NAME]/dev/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "email_notifications": {}, + "format": "MULTI_TASK", + "job_id": [NUMID], + "max_concurrent_runs": 4, + "name": "[dev [USERNAME]] foo", + "queue": { + "enabled": true + }, + "run_as_user_name": "[USERNAME]", + "tags": { + "dev": "[USERNAME]" + }, + "timeout_seconds": 0, + "webhook_notifications": {} + }, + "changes": { + "email_notifications": { + "action": "skip", + "reason": "empty", + "remote": {} + }, + "timeout_seconds": { + "action": "skip", + "reason": "empty", + "remote": 0 + }, + "webhook_notifications": { + "action": "skip", + "reason": "empty", + "remote": {} + } + } + } + } +} + === The service denormalizes them onto the deployment, so a reader gets the provenance of the latest version >>> MSYS_NO_PATHCONV=1 [CLI] api get /api/2.0/bundle/deployments/[NUMID] { @@ -66,3 +121,48 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-provenance-[UNIQUE_NAME]/dev" } } + +=== The service lists the deployed job +>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[NUMID] -o json +[ + { + "last_action_type": "OPERATION_ACTION_TYPE_CREATE", + "last_version_id": "1", + "resource_key": "jobs.foo" + } +] + +=== The raw recorded state the service holds for this deployment +>>> print_state.py -t dev +{ + "state_version": 2, + "cli_version": "[CLI_VERSION]", + "lineage": "[UUID]", + "serial": 1, + "features": { + "record_deployment_history": {} + }, + "state": { + "resources.jobs.foo": { + "__id__": "[NUMID]", + "state": { + "deployment": { + "deployment_id": "[NUMID]", + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-provenance-[UNIQUE_NAME]/dev/state/metadata.json", + "version_id": "1" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 4, + "name": "[dev [USERNAME]] foo", + "queue": { + "enabled": true + }, + "tags": { + "dev": "[USERNAME]" + } + } + } + } +} diff --git a/acceptance/bundle/dms/provenance/script b/acceptance/bundle/dms/provenance/script index 8b15822fa4b..1c6661c30d7 100644 --- a/acceptance/bundle/dms/provenance/script +++ b/acceptance/bundle/dms/provenance/script @@ -8,12 +8,21 @@ trace $CLI bundle deploy add_repl.py "$(git rev-parse HEAD)" COMMIT trace print_requests.py --dms //versions +title "The plan is a no-op and records DEPLOYMENT_METADATA_SERVICE as its backend" +trace $CLI bundle plan -o json | nostamp.py + title "The service denormalizes them onto the deployment, so a reader gets the provenance of the latest version" deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-provenance-${UNIQUE_NAME}/dev/state/resources.deployment.json" -o json | python3 -c 'import sys,json; print(json.load(sys.stdin)["object_id"])') # MSYS_NO_PATHCONV as above: Git Bash on Windows would rewrite the leading-'/' API # path into C:/Program Files/Git/api/2.0/... before the CLI saw it. trace MSYS_NO_PATHCONV=1 $CLI api get "/api/2.0/bundle/deployments/${deployment_id}" | jq '{target_name, deployment_mode, git_info, workspace_info}' +title "The service lists the deployed job" +trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' + +title "The raw recorded state the service holds for this deployment" +trace print_state.py -t dev + # The deploy uploads files in a nondeterministic order; only the requests above are # asserted. rm -f out.requests.txt diff --git a/acceptance/bundle/dms/record/output.txt b/acceptance/bundle/dms/record/output.txt index 7e63506bf79..7f62c55528d 100644 --- a/acceptance/bundle/dms/record/output.txt +++ b/acceptance/bundle/dms/record/output.txt @@ -82,3 +82,97 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "completion_reason": "VERSION_COMPLETE_SUCCESS" } } + +=== The plan is a no-op now and records DEPLOYMENT_METADATA_SERVICE as its backend +>>> [CLI] bundle plan -o json +{ + "plan_version": 3, + "cli_version": "[CLI_VERSION]", + "lineage": "[UUID]", + "serial": 3, + "storage_backend": "DEPLOYMENT_METADATA_SERVICE", + "plan": { + "resources.jobs.foo": { + "action": "skip", + "remote_state": { + "created_time": [UNIX_TIME_MILLIS], + "creator_user_name": "[USERNAME]", + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-record-[UNIQUE_NAME]/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "email_notifications": {}, + "format": "MULTI_TASK", + "job_id": [NUMID], + "max_concurrent_runs": 1, + "name": "foo", + "queue": { + "enabled": true + }, + "run_as_user_name": "[USERNAME]", + "timeout_seconds": 0, + "webhook_notifications": {} + }, + "changes": { + "email_notifications": { + "action": "skip", + "reason": "empty", + "remote": {} + }, + "timeout_seconds": { + "action": "skip", + "reason": "empty", + "remote": 0 + }, + "webhook_notifications": { + "action": "skip", + "reason": "empty", + "remote": {} + } + } + } + } +} + +=== The service lists the job, recorded at version 1 +>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[NUMID] -o json +[ + { + "last_action_type": "OPERATION_ACTION_TYPE_CREATE", + "last_version_id": "1", + "resource_key": "jobs.foo" + } +] + +=== The raw recorded state the service holds for this deployment +>>> print_state.py +{ + "state_version": 2, + "cli_version": "[CLI_VERSION]", + "lineage": "[UUID]", + "serial": 3, + "features": { + "record_deployment_history": {} + }, + "state": { + "resources.jobs.foo": { + "__id__": "[NUMID]", + "state": { + "deployment": { + "deployment_id": "[NUMID]", + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-record-[UNIQUE_NAME]/default/state/metadata.json", + "version_id": "1" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "foo", + "queue": { + "enabled": true + } + } + } + } +} diff --git a/acceptance/bundle/dms/record/script b/acceptance/bundle/dms/record/script index c521416b158..5c565b792d1 100644 --- a/acceptance/bundle/dms/record/script +++ b/acceptance/bundle/dms/record/script @@ -14,3 +14,14 @@ trace MSYS_NO_PATHCONV=1 musterr $CLI workspace get-status "/Workspace/Users/${C trace $CLI bundle deploy trace MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-record-${UNIQUE_NAME}/default/state/resources.deployment.json" | jq '{object_type,path}' trace print_requests.py --get --dms //api/2.0/bundle + +title "The plan is a no-op now and records DEPLOYMENT_METADATA_SERVICE as its backend" +trace $CLI bundle plan -o json | nostamp.py + +title "The service lists the job, recorded at version 1" +deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-record-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') +trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' + +title "The raw recorded state the service holds for this deployment" +trace print_state.py +rm -f out.requests.txt diff --git a/acceptance/bundle/dms/stale-plan/output.txt b/acceptance/bundle/dms/stale-plan/output.txt index f39b3049697..420d5225dd2 100644 --- a/acceptance/bundle/dms/stale-plan/output.txt +++ b/acceptance/bundle/dms/stale-plan/output.txt @@ -1,6 +1,31 @@ === Save a plan, then deploy it: records version 1 >>> [CLI] bundle plan -o json +{ + "plan_version": 3, + "cli_version": "[CLI_VERSION]", + "storage_backend": "DEPLOYMENT_METADATA_SERVICE", + "plan": { + "resources.jobs.foo": { + "action": "create", + "new_state": { + "value": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-stale-plan-[UNIQUE_NAME]/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "foo", + "queue": { + "enabled": true + } + } + } + } + } +} >>> [CLI] bundle deploy --plan tmp.plan.json Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-stale-plan-[UNIQUE_NAME]/default/files... @@ -8,6 +33,47 @@ Created jobs.foo Files: 6 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged +>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[NUMID] -o json +[ + { + "last_action_type": "OPERATION_ACTION_TYPE_CREATE", + "last_version_id": "1", + "resource_key": "jobs.foo" + } +] + +=== The raw recorded state the service holds for this deployment +>>> print_state.py +{ + "state_version": 2, + "cli_version": "[CLI_VERSION]", + "lineage": "[UUID]", + "serial": 1, + "features": { + "record_deployment_history": {} + }, + "state": { + "resources.jobs.foo": { + "__id__": "[NUMID]", + "state": { + "deployment": { + "deployment_id": "[NUMID]", + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-stale-plan-[UNIQUE_NAME]/default/state/metadata.json", + "version_id": "1" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "foo", + "queue": { + "enabled": true + } + } + } + } +} + === A saved plan carries the deployment id and the versions it targets (last 1, next 2) >>> [CLI] bundle plan -o json { diff --git a/acceptance/bundle/dms/stale-plan/script b/acceptance/bundle/dms/stale-plan/script index 09724d7d545..8a49c5fcd4f 100644 --- a/acceptance/bundle/dms/stale-plan/script +++ b/acceptance/bundle/dms/stale-plan/script @@ -1,9 +1,16 @@ envsubst < databricks.yml.tmpl > databricks.yml title "Save a plan, then deploy it: records version 1" -trace $CLI bundle plan -o json > tmp.plan.json +$CLI bundle plan -o json > tmp.plan.json +trace $CLI bundle plan -o json | nostamp.py trace $CLI bundle deploy --plan tmp.plan.json +deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-stale-plan-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') +trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' + +title "The raw recorded state the service holds for this deployment" +trace print_state.py + title "A saved plan carries the deployment id and the versions it targets (last 1, next 2)" trace $CLI bundle plan -o json | jq '{deployment_id, last_version_id, next_version_id}' @@ -13,4 +20,4 @@ trace musterr $CLI bundle deploy --plan tmp.plan.json title "Only the first deploy recorded a version; the rejected replay created none" trace print_requests.py --dms //api/2.0/bundle --oneline -rm -f tmp.plan.json +rm -f tmp.plan.json out.requests.txt diff --git a/acceptance/bundle/dms/successful-recreate/output.txt b/acceptance/bundle/dms/successful-recreate/output.txt index 4d497d81f2b..1fd34e383e4 100644 --- a/acceptance/bundle/dms/successful-recreate/output.txt +++ b/acceptance/bundle/dms/successful-recreate/output.txt @@ -59,6 +59,74 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged } } +=== The plan is a no-op now and records DEPLOYMENT_METADATA_SERVICE as its backend +>>> [CLI] bundle plan -o json +{ + "plan_version": 3, + "cli_version": "[CLI_VERSION]", + "lineage": "[UUID]", + "serial": 1, + "storage_backend": "DEPLOYMENT_METADATA_SERVICE", + "plan": { + "resources.schemas.foo": { + "action": "skip", + "remote_state": { + "browse_only": false, + "catalog_name": "main", + "catalog_type": "MANAGED_CATALOG", + "comment": "v1", + "created_at": [UNIX_TIME_MILLIS][0], + "created_by": "[USERNAME]", + "effective_predictive_optimization_flag": { + "inherited_from_name": "[METASTORE_NAME]", + "inherited_from_type": "METASTORE", + "value": "ENABLE" + }, + "enable_predictive_optimization": "INHERIT", + "full_name": "main.dms_successful_recreate_schema_[UNIQUE_NAME]", + "metastore_id": "[UUID]", + "name": "dms_successful_recreate_schema_[UNIQUE_NAME]", + "owner": "[USERNAME]", + "schema_id": "[UUID]", + "updated_at": [UNIX_TIME_MILLIS][0], + "updated_by": "[USERNAME]" + } + } + } +} + +=== The service lists the schema, recorded at version 1 +>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[NUMID] -o json +[ + { + "last_action_type": "OPERATION_ACTION_TYPE_CREATE", + "last_version_id": "1", + "resource_key": "schemas.foo" + } +] + +=== The raw recorded state the service holds for this deployment +>>> print_state.py +{ + "state_version": 2, + "cli_version": "[CLI_VERSION]", + "lineage": "[UUID]", + "serial": 1, + "features": { + "record_deployment_history": {} + }, + "state": { + "resources.schemas.foo": { + "__id__": "main.dms_successful_recreate_schema_[UNIQUE_NAME]", + "state": { + "catalog_name": "main", + "comment": "v1", + "name": "dms_successful_recreate_schema_[UNIQUE_NAME]" + } + } + } +} + === Recreate writes state twice - the entry is dropped, then the new resource is saved >>> update_file.py databricks.yml catalog_name: main catalog_name: dms_other_[UNIQUE_NAME] @@ -124,6 +192,74 @@ Resources: 1 created, 0 changed, 1 deleted, 0 unchanged } } +=== The plan now shows the recreate action +>>> [CLI] bundle plan -o json +{ + "plan_version": 3, + "cli_version": "[CLI_VERSION]", + "lineage": "[UUID]", + "serial": 2, + "storage_backend": "DEPLOYMENT_METADATA_SERVICE", + "plan": { + "resources.schemas.foo": { + "action": "skip", + "remote_state": { + "browse_only": false, + "catalog_name": "dms_other_[UNIQUE_NAME]", + "catalog_type": "MANAGED_CATALOG", + "comment": "v1", + "created_at": [UNIX_TIME_MILLIS][1], + "created_by": "[USERNAME]", + "effective_predictive_optimization_flag": { + "inherited_from_name": "[METASTORE_NAME]", + "inherited_from_type": "METASTORE", + "value": "ENABLE" + }, + "enable_predictive_optimization": "INHERIT", + "full_name": "dms_other_[UNIQUE_NAME].dms_successful_recreate_schema_[UNIQUE_NAME]", + "metastore_id": "[UUID]", + "name": "dms_successful_recreate_schema_[UNIQUE_NAME]", + "owner": "[USERNAME]", + "schema_id": "[UUID]", + "updated_at": [UNIX_TIME_MILLIS][1], + "updated_by": "[USERNAME]" + } + } + } +} + +=== The service lists the schema at version 2 with recreate action +>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[NUMID] -o json +[ + { + "last_action_type": "OPERATION_ACTION_TYPE_RECREATE", + "last_version_id": "2", + "resource_key": "schemas.foo" + } +] + +=== The raw recorded state after recreate +>>> print_state.py +{ + "state_version": 2, + "cli_version": "[CLI_VERSION]", + "lineage": "[UUID]", + "serial": 2, + "features": { + "record_deployment_history": {} + }, + "state": { + "resources.schemas.foo": { + "__id__": "dms_other_[UNIQUE_NAME].dms_successful_recreate_schema_[UNIQUE_NAME]", + "state": { + "catalog_name": "dms_other_[UNIQUE_NAME]", + "comment": "v1", + "name": "dms_successful_recreate_schema_[UNIQUE_NAME]" + } + } + } +} + === Destroy: the delete is recorded with the id and no state >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: diff --git a/acceptance/bundle/dms/successful-recreate/script b/acceptance/bundle/dms/successful-recreate/script index 43050cc7c8f..6a8e0fe8914 100644 --- a/acceptance/bundle/dms/successful-recreate/script +++ b/acceptance/bundle/dms/successful-recreate/script @@ -4,6 +4,16 @@ title "Deploy: the state write records the resource" trace $CLI bundle deploy trace print_requests.py --dms //api/2.0/bundle +title "The plan is a no-op now and records DEPLOYMENT_METADATA_SERVICE as its backend" +trace $CLI bundle plan -o json | nostamp.py + +title "The service lists the schema, recorded at version 1" +deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-successful-recreate-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') +trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' + +title "The raw recorded state the service holds for this deployment" +trace print_state.py + title "Recreate writes state twice - the entry is dropped, then the new resource is saved" # One operation per resource per version; both writes land on the same one. # The drop opens it IN_PROGRESS, the save patches it to SUCCEEDED. @@ -15,6 +25,16 @@ trace update_file.py databricks.yml "catalog_name: main" "catalog_name: ${other_ trace $CLI bundle deploy --auto-approve trace print_requests.py --dms //api/2.0/bundle +title "The plan now shows the recreate action" +trace $CLI bundle plan -o json | nostamp.py + +title "The service lists the schema at version 2 with recreate action" +trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' + +title "The raw recorded state after recreate" +trace print_state.py + title "Destroy: the delete is recorded with the id and no state" trace $CLI bundle destroy --auto-approve trace print_requests.py --dms //api/2.0/bundle +rm -f out.requests.txt From db6a166c6ec8f2f1b15b981db6cd043390697abc Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Fri, 4 Sep 2026 13:18:50 +0000 Subject: [PATCH 17/46] bundle/dms: guard StorageBackend() gate on direct engine; bump future-version golden - deploy.go: StorageBackend() asserts the state is open, but only the direct engine opens it, so gate the DMS deployment-create block on stateEngine.IsDirect() first. Fixes a panic ("DeploymentState must be opened first") on terraform deploys, which the direct-only dms tests did not exercise. - future_version: supported state version is now 3 (v3 is read-forward-compatible), so the too-new message reports 3. Co-authored-by: Isaac --- acceptance/bundle/state/future_version/output.txt | 2 +- bundle/phases/deploy.go | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/acceptance/bundle/state/future_version/output.txt b/acceptance/bundle/state/future_version/output.txt index 7cf98129ee9..0a16971f472 100644 --- a/acceptance/bundle/state/future_version/output.txt +++ b/acceptance/bundle/state/future_version/output.txt @@ -1,3 +1,3 @@ -state version 999 is newer than supported version 2; upgrade the CLI +state version 999 is newer than supported version 3; upgrade the CLI Exit code: 1 diff --git a/bundle/phases/deploy.go b/bundle/phases/deploy.go index b7dd3ea44d1..88245e107e1 100644 --- a/bundle/phases/deploy.go +++ b/bundle/phases/deploy.go @@ -331,7 +331,8 @@ func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHand // Create the deployment now that the plan is approved, so a declined deploy leaves none behind. // A first deploy's id did not exist at plan time - the version and any existing id were stamped // then - so stamp the one just created into the plan the apply reads. - if b.DeploymentBundle.StateDB.StorageBackend() == dstate.StorageBackendDeploymentMetadataService { + // IsDirect first: StorageBackend asserts the state is open, and only the direct engine opens it. + if stateEngine.IsDirect() && b.DeploymentBundle.StateDB.StorageBackend() == dstate.StorageBackendDeploymentMetadataService { existingID, _ := deploymentAndNextVersion(b) createOrUpdateDeployment(ctx, b, dmsDeployment) if logdiag.HasError(ctx) { From 0afcb1354c3bc1ccc687a8397fcd7025fcea416a Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Fri, 4 Sep 2026 13:41:34 +0000 Subject: [PATCH 18/46] bundle/dms: StateDB owns plan lineage; unbind-specific message; fail version on recording error - C11: StateDB holds DeploymentID/LatestVersionID (set at Open from the service); CalculatePlan stamps the plan's lineage from StateDB instead of the bundle config tree (config.History stays only for 'bundle summary'). Behavior-neutral. - unbind: Open returns a sentinel ErrUnsettingRecording; DeploymentBundle.Unbind maps it to an unbind-specific message instead of the deploy/destroy-flavored guard text. - CompleteVersion now completes as Failure when the drained buffer holds a recording error, even if the caller passed success=true, so a destroy whose uploads failed keeps its deployment record. Co-authored-by: Isaac --- bundle/direct/bind.go | 4 ++++ bundle/direct/bundle_plan.go | 21 ++++++++++----------- bundle/direct/dstate/state.go | 27 +++++++++++++++++++++++---- cmd/bundle/utils/process.go | 3 +++ 4 files changed, 40 insertions(+), 15 deletions(-) diff --git a/bundle/direct/bind.go b/bundle/direct/bind.go index a65cacd7eb4..4605f34be66 100644 --- a/bundle/direct/bind.go +++ b/bundle/direct/bind.go @@ -2,6 +2,7 @@ package direct import ( "context" + "errors" "fmt" "os" "strings" @@ -219,6 +220,9 @@ func (result *BindResult) Cancel() { func (b *DeploymentBundle) Unbind(ctx context.Context, statePath, resourceKey string) error { err := b.StateDB.Open(ctx, statePath, dstate.WithRecovery(true), dstate.WithWrite(true), dstate.WithDeploymentHistory(false), "") if err != nil { + if errors.Is(err, dstate.ErrUnsettingRecording) { + return errors.New("unbind is not supported for a bundle that records deployment history") + } return err } diff --git a/bundle/direct/bundle_plan.go b/bundle/direct/bundle_plan.go index 3b6b389bc57..775103e6ad9 100644 --- a/bundle/direct/bundle_plan.go +++ b/bundle/direct/bundle_plan.go @@ -168,24 +168,23 @@ func (b *DeploymentBundle) CalculatePlan(ctx context.Context, client *databricks return nil, fmt.Errorf("reading config: %w", err) } - // Record the DMS deployment and version this plan targets. A saved plan carries them so - // deploy --plan can reject a plan the deployment has moved on from, and pass last_version_id - // as previous_version_id. History is set only while recording, so other plans leave these empty. - // configRoot is nil for destroy, which records its version separately. // The plan records where its state lives so deploy --plan can reject a plan whose target has - // since switched backends. Set from StateDB (the source of truth) so a first deploy - which has - // no History yet - still carries it. + // since switched backends. StateDB is the source of truth, so a first deploy - which has no + // recorded deployment yet - still carries the backend. if b.StateDB.StorageBackend() == dstate.StorageBackendDeploymentMetadataService { plan.StorageBackend = string(dstate.StorageBackendDeploymentMetadataService) } - if configRoot != nil && configRoot.Bundle.Deployment.History != nil { - h := configRoot.Bundle.Deployment.History - next, err := dms.NextVersion(h.LatestVersionID) + // Record the DMS deployment and version this plan targets, from StateDB (not the config tree). + // A saved plan carries them so deploy --plan can reject a plan the deployment has moved on from, + // and passes last_version_id as previous_version_id. Empty for a first deploy, which has no + // recorded deployment yet - the deploy phase stamps the created id. + if b.StateDB.DeploymentID != "" { + next, err := dms.NextVersion(b.StateDB.LatestVersionID) if err != nil { return nil, fmt.Errorf("computing next deployment version: %w", err) } - plan.DeploymentId = h.DeploymentID - plan.LastVersionId = h.LatestVersionID + plan.DeploymentId = b.StateDB.DeploymentID + plan.LastVersionId = b.StateDB.LatestVersionID plan.NextVersionId = strconv.FormatInt(next, 10) } diff --git a/bundle/direct/dstate/state.go b/bundle/direct/dstate/state.go index a248f8328fb..17c4eb9d4e7 100644 --- a/bundle/direct/dstate/state.go +++ b/bundle/direct/dstate/state.go @@ -100,6 +100,13 @@ func checkStateFeatures(features map[string]struct{}) error { // The caller should delete the stale WAL and proceed normally. var errStaleWAL = errors.New("stale WAL") +// ErrUnsettingRecording is returned by Open when a recorded state is opened without recording - the +// config turned the feature off, or an operation that never records (unbind) reached it. Callers +// present an operation-appropriate message via errors.Is. +var ErrUnsettingRecording = errors.New(`unsetting experimental.record_deployment_history is not supported + +This deployment's resources are recorded with the deployment metadata service. Set experimental.record_deployment_history: true to deploy or destroy this bundle`) + type DeploymentState struct { Path string Data Database @@ -130,6 +137,12 @@ type DeploymentState struct { // versionCompleted makes CompleteVersion a no-op after the first call, so a deferred safety-net // completion after an explicit one does nothing. versionCompleted bool + + // DeploymentID and LatestVersionID are the recorded deployment's id and its most recent version + // at Open time, from the service. CalculatePlan reads them (rather than the bundle config tree) + // to stamp the plan's lineage. Both empty for a first deploy, whose id the deploy phase creates. + DeploymentID string + LatestVersionID string } type Header struct { @@ -204,7 +217,8 @@ func (db *DeploymentState) RecordingError() error { // survive Finalize's reset and asserts nothing, so it is safe to call after the state is closed. func (db *DeploymentState) CompleteVersion(ctx context.Context, success bool) (bool, error) { db.mu.Lock() - if db.operationBuffer == nil || db.versionCompleted { + buf := db.operationBuffer + if buf == nil || db.versionCompleted { db.mu.Unlock() return false, nil } @@ -212,6 +226,13 @@ func (db *DeploymentState) CompleteVersion(ctx context.Context, success bool) (b deploymentID, versionID, client := db.deploymentID, db.versionID, db.dmsClient db.mu.Unlock() + // A recording failure fails the version even when the caller counted the deploy a success: the + // service does not then hold everything the WAL does. Finalize already drained and surfaced it; + // this reads the drained buffer's error so a destroy whose uploads failed keeps its record. + if buf.Err() != nil { + success = false + } + reason := bundledeployments.VersionCompleteVersionCompleteSuccess if !success { reason = bundledeployments.VersionCompleteVersionCompleteFailure @@ -552,9 +573,7 @@ To record this bundle's history, start it over as a new deployment: 2. run "databricks bundle destroy" to delete the existing resources 3. add experimental.record_deployment_history back and deploy again`) case !recording && recorded: - return errors.New(`unsetting experimental.record_deployment_history is not supported - -This deployment's resources are recorded with the deployment metadata service. Set experimental.record_deployment_history: true to deploy or destroy this bundle`) + return ErrUnsettingRecording } db.storageBackend = StorageBackendWorkspaceFilesystem diff --git a/cmd/bundle/utils/process.go b/cmd/bundle/utils/process.go index 74909c20997..dc11d95306e 100644 --- a/cmd/bundle/utils/process.go +++ b/cmd/bundle/utils/process.go @@ -293,6 +293,9 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle logdiag.LogError(ctx, err) return b, stateDesc, root.ErrAlreadyPrinted } + // StateDB owns the plan's lineage: CalculatePlan reads these, not the config tree. + b.DeploymentBundle.StateDB.DeploymentID = dmsDeploymentID + b.DeploymentBundle.StateDB.LatestVersionID = lastVersionID } else { if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WithRecovery(true), dstate.WithWrite(false), dstate.WithDeploymentHistory(false), ""); err != nil { logdiag.LogError(ctx, err) From 0f92c144c4350e7f4556eb5daed33f33b6a5261d Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Fri, 4 Sep 2026 13:46:51 +0000 Subject: [PATCH 19/46] acceptance/dms: drop dedicated readplan test; cover deploy --plan via the readplan option The dedicated readplan test duplicated stale-plan, which already deploys from a saved raw plan under recording. Delete it and add the standard READPLAN matrix to stale-plan so its initial deploy runs both from the saved plan and recomputed - the deploy is untraced so the variant-dependent command stays out of the golden. Recorded operations still carry the deployment_id/version_id lineage (asserted in depends-on and others). Co-authored-by: Isaac --- .../bundle/dms/readplan/databricks.yml.tmpl | 19 ---------------- acceptance/bundle/dms/readplan/out.test.toml | 3 --- acceptance/bundle/dms/readplan/output.txt | 22 ------------------- acceptance/bundle/dms/readplan/script | 13 ----------- acceptance/bundle/dms/readplan/transform.py | 1 - .../bundle/dms/stale-plan/out.test.toml | 1 + acceptance/bundle/dms/stale-plan/output.txt | 4 +--- acceptance/bundle/dms/stale-plan/script | 4 +++- acceptance/bundle/dms/stale-plan/test.toml | 4 ++++ 9 files changed, 9 insertions(+), 62 deletions(-) delete mode 100644 acceptance/bundle/dms/readplan/databricks.yml.tmpl delete mode 100644 acceptance/bundle/dms/readplan/out.test.toml delete mode 100644 acceptance/bundle/dms/readplan/output.txt delete mode 100644 acceptance/bundle/dms/readplan/script delete mode 100644 acceptance/bundle/dms/readplan/transform.py create mode 100644 acceptance/bundle/dms/stale-plan/test.toml diff --git a/acceptance/bundle/dms/readplan/databricks.yml.tmpl b/acceptance/bundle/dms/readplan/databricks.yml.tmpl deleted file mode 100644 index d11621ab1e5..00000000000 --- a/acceptance/bundle/dms/readplan/databricks.yml.tmpl +++ /dev/null @@ -1,19 +0,0 @@ -bundle: - name: dms-readplan-$UNIQUE_NAME -experimental: - record_deployment_history: true - -resources: - jobs: - foo: - name: foo - pipelines: - bar: - # Unique: the Pipelines API refuses a duplicate name across overlapping runs. - name: bar-$UNIQUE_NAME - catalog: main - schema: default - serverless: true - libraries: - - file: - path: ./transform.py diff --git a/acceptance/bundle/dms/readplan/out.test.toml b/acceptance/bundle/dms/readplan/out.test.toml deleted file mode 100644 index 23c07f70dca..00000000000 --- a/acceptance/bundle/dms/readplan/out.test.toml +++ /dev/null @@ -1,3 +0,0 @@ -Cloud = true -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -EnvMatrix.DMS = ["true"] diff --git a/acceptance/bundle/dms/readplan/output.txt b/acceptance/bundle/dms/readplan/output.txt deleted file mode 100644 index 371855dbcc0..00000000000 --- a/acceptance/bundle/dms/readplan/output.txt +++ /dev/null @@ -1,22 +0,0 @@ - -=== Save a plan, then deploy from it: the deploy stamps the id and version onto the job and pipeline the plan predates ->>> [CLI] bundle plan -o json - ->>> [CLI] bundle deploy --plan tmp.plan.json -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-readplan-[UNIQUE_NAME]/default/files... -Created jobs.foo -Created pipelines.bar -Files: 7 uploaded, 0 deleted -Resources: 2 created, 0 changed, 0 deleted, 0 unchanged - -=== Re-plan: no drift, the deployed job and pipeline carry the deployment stamp ->>> [CLI] bundle plan -Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged - -=== Both recorded operations carry deployment_id and version_id (lineage), not just kind ->>> print_requests.py --dms //api/2.0/bundle --sort --oneline -{"method": "PATCH", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.foo", "q": {"update_mask": "state,error_message,resource_id,status"}, "body": {"error_message": "", "resource_id": "[NUMID]", "sequence_id": "0", "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-readplan-[UNIQUE_NAME]/default/state/metadata.json\",\"version_id\":\"1\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"foo\",\"queue\":{\"enabled\":true}}}", "status": "OPERATION_STATUS_SUCCEEDED"}} -{"method": "PATCH", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/pipelines.bar", "q": {"update_mask": "state,error_message,resource_id,status"}, "body": {"error_message": "", "resource_id": "[UUID]", "sequence_id": "0", "state": "{\"state\":{\"catalog\":\"main\",\"channel\":\"CURRENT\",\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-readplan-[UNIQUE_NAME]/default/state/metadata.json\",\"version_id\":\"1\"},\"edition\":\"ADVANCED\",\"libraries\":[{\"file\":{\"path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-readplan-[UNIQUE_NAME]/default/files/transform.py\"}}],\"name\":\"bar-[UNIQUE_NAME]\",\"schema\":\"default\",\"serverless\":true}}", "status": "OPERATION_STATUS_SUCCEEDED"}} -{"method": "POST", "path": "/api/2.0/bundle/deployments", "body": {"display_name": "dms-readplan-[UNIQUE_NAME]", "initial_parent_path": "/Workspace/Users/[USERNAME]/.bundle/dms-readplan-[UNIQUE_NAME]/default/state", "target_name": "default", "workspace_info": {"file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-readplan-[UNIQUE_NAME]/default/files", "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-readplan-[UNIQUE_NAME]/default"}}} -{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": {"version_id": "1"}, "body": {"cli_version": "[CLI_VERSION]", "version_type": "VERSION_TYPE_DEPLOY", "operations": [{"resource_key": "jobs.foo", "action_type": "OPERATION_ACTION_TYPE_CREATE"}, {"resource_key": "pipelines.bar", "action_type": "OPERATION_ACTION_TYPE_CREATE"}]}} -{"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/complete", "body": {"completion_reason": "VERSION_COMPLETE_SUCCESS"}} diff --git a/acceptance/bundle/dms/readplan/script b/acceptance/bundle/dms/readplan/script deleted file mode 100644 index 40bd110bf00..00000000000 --- a/acceptance/bundle/dms/readplan/script +++ /dev/null @@ -1,13 +0,0 @@ -envsubst < databricks.yml.tmpl > databricks.yml - -title "Save a plan, then deploy from it: the deploy stamps the id and version onto the job and pipeline the plan predates" -trace $CLI bundle plan -o json > tmp.plan.json -trace $CLI bundle deploy --plan tmp.plan.json - -title "Re-plan: no drift, the deployed job and pipeline carry the deployment stamp" -trace $CLI bundle plan - -title "Both recorded operations carry deployment_id and version_id (lineage), not just kind" -trace print_requests.py --dms //api/2.0/bundle --sort --oneline | contains.py deployment_id version_id - -rm -f tmp.plan.json diff --git a/acceptance/bundle/dms/readplan/transform.py b/acceptance/bundle/dms/readplan/transform.py deleted file mode 100644 index c07007592a6..00000000000 --- a/acceptance/bundle/dms/readplan/transform.py +++ /dev/null @@ -1 +0,0 @@ -# Minimal source so the pipeline has a library, which the API requires. diff --git a/acceptance/bundle/dms/stale-plan/out.test.toml b/acceptance/bundle/dms/stale-plan/out.test.toml index 23c07f70dca..f7ae4fc905f 100644 --- a/acceptance/bundle/dms/stale-plan/out.test.toml +++ b/acceptance/bundle/dms/stale-plan/out.test.toml @@ -1,3 +1,4 @@ Cloud = true EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] EnvMatrix.DMS = ["true"] +EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/dms/stale-plan/output.txt b/acceptance/bundle/dms/stale-plan/output.txt index 420d5225dd2..3b89e18875a 100644 --- a/acceptance/bundle/dms/stale-plan/output.txt +++ b/acceptance/bundle/dms/stale-plan/output.txt @@ -26,11 +26,9 @@ } } } - ->>> [CLI] bundle deploy --plan tmp.plan.json Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-stale-plan-[UNIQUE_NAME]/default/files... Created jobs.foo -Files: 6 uploaded, 0 deleted +Files: 7 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged >>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[NUMID] -o json diff --git a/acceptance/bundle/dms/stale-plan/script b/acceptance/bundle/dms/stale-plan/script index 8a49c5fcd4f..f50dae91193 100644 --- a/acceptance/bundle/dms/stale-plan/script +++ b/acceptance/bundle/dms/stale-plan/script @@ -3,7 +3,9 @@ envsubst < databricks.yml.tmpl > databricks.yml title "Save a plan, then deploy it: records version 1" $CLI bundle plan -o json > tmp.plan.json trace $CLI bundle plan -o json | nostamp.py -trace $CLI bundle deploy --plan tmp.plan.json +# READPLAN deploys from the saved raw plan; without it the plan is recomputed. Untraced so the +# variant-dependent command line stays out of the golden (the deploy output is identical either way). +$CLI bundle deploy $(readplanarg tmp.plan.json) deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-stale-plan-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' diff --git a/acceptance/bundle/dms/stale-plan/test.toml b/acceptance/bundle/dms/stale-plan/test.toml new file mode 100644 index 00000000000..4c51e8879d6 --- /dev/null +++ b/acceptance/bundle/dms/stale-plan/test.toml @@ -0,0 +1,4 @@ +# Also deploy from the saved plan (not just compute it), so recorded deploy --plan is covered by the +# standard readplan option rather than a dedicated test. The saved plan is raw (not nostamp.py'd) so +# it carries the DMS fields the replay needs. +EnvMatrix.READPLAN = ["", "1"] From 5f108058d68416b1b570afd05a5cf1cfc79ec8d9 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Fri, 4 Sep 2026 13:56:34 +0000 Subject: [PATCH 20/46] bundle/dms: trim redundant comment on the cmdctx workspace-client bridge Co-authored-by: Isaac --- cmd/bundle/utils/process.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cmd/bundle/utils/process.go b/cmd/bundle/utils/process.go index dc11d95306e..f223abc82d4 100644 --- a/cmd/bundle/utils/process.go +++ b/cmd/bundle/utils/process.go @@ -284,8 +284,7 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle if logdiag.HasError(ctx) { return b, stateDesc, root.ErrAlreadyPrinted } - // StateDB.Open builds the DMS client from the workspace client on the context. A bundle - // command resolves that client lazily through b, so it is not otherwise on the context. + // StateDB.Open builds the DMS client from the workspace client on the context. if !cmdctx.HasWorkspaceClient(ctx) { ctx = cmdctx.SetWorkspaceClient(ctx, b.WorkspaceClient(ctx)) } From abd73573045604c03f76f4052e50aa96bed1f6db Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Fri, 4 Sep 2026 14:06:18 +0000 Subject: [PATCH 21/46] acceptance/dms: record plan/list-resources/state for the failure tests Extend the failure-injection tests to show how DMS records a failed or refused deploy: failed-delete/recreate/update, failed-update-permissions, operation-upload-fails-once, record-failure, version-never-created (and a WORKSPACE_FILESYSTEM plan on existing-state's first deploy). Each captures the healthy recorded state and the recorded result of the failure. Co-authored-by: Isaac --- .../bundle/dms/existing-state/output.txt | 50 +++++ acceptance/bundle/dms/existing-state/script | 1 + .../bundle/dms/failed-delete/output.txt | 105 +++++++++ acceptance/bundle/dms/failed-delete/script | 18 +- .../bundle/dms/failed-recreate/output.txt | 138 +++++++++++- acceptance/bundle/dms/failed-recreate/script | 23 +- .../dms/failed-update-permissions/output.txt | 212 ++++++++++++++++++ .../dms/failed-update-permissions/script | 18 ++ .../bundle/dms/failed-update/output.txt | 161 ++++++++++++- acceptance/bundle/dms/failed-update/script | 23 +- .../operation-upload-fails-once/output.txt | 100 +++++++++ .../dms/operation-upload-fails-once/script | 18 ++ .../bundle/dms/record-failure/output.txt | 45 ++++ acceptance/bundle/dms/record-failure/script | 5 + .../dms/version-never-created/output.txt | 33 +++ .../bundle/dms/version-never-created/script | 4 + 16 files changed, 925 insertions(+), 29 deletions(-) diff --git a/acceptance/bundle/dms/existing-state/output.txt b/acceptance/bundle/dms/existing-state/output.txt index 9a7439eef8a..95352e1dcb0 100644 --- a/acceptance/bundle/dms/existing-state/output.txt +++ b/acceptance/bundle/dms/existing-state/output.txt @@ -6,6 +6,56 @@ Created jobs.one Files: 6 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged +>>> [CLI] bundle plan -o json +{ + "plan_version": 3, + "cli_version": "[CLI_VERSION]", + "lineage": "[UUID]", + "serial": 1, + "plan": { + "resources.jobs.one": { + "action": "skip", + "remote_state": { + "created_time": [UNIX_TIME_MILLIS], + "creator_user_name": "[USERNAME]", + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "email_notifications": {}, + "format": "MULTI_TASK", + "job_id": [NUMID], + "max_concurrent_runs": 1, + "name": "one", + "queue": { + "enabled": true + }, + "run_as_user_name": "[USERNAME]", + "timeout_seconds": 0, + "webhook_notifications": {} + }, + "changes": { + "email_notifications": { + "action": "skip", + "reason": "empty", + "remote": {} + }, + "timeout_seconds": { + "action": "skip", + "reason": "empty", + "remote": 0 + }, + "webhook_notifications": { + "action": "skip", + "reason": "empty", + "remote": {} + } + } + } + } +} + >>> print_requests.py --dms //api/2.0/bundle --oneline >>> jq {state_version, features} .databricks/bundle/default/resources.json diff --git a/acceptance/bundle/dms/existing-state/script b/acceptance/bundle/dms/existing-state/script index e0784bb406b..5ab20285e56 100644 --- a/acceptance/bundle/dms/existing-state/script +++ b/acceptance/bundle/dms/existing-state/script @@ -2,6 +2,7 @@ envsubst < databricks.yml.tmpl > databricks.yml title "Deploy without recording: ordinary direct-engine state tracking one job, unknown to DMS" trace $CLI bundle deploy +trace $CLI bundle plan -o json | nostamp.py trace print_requests.py --dms //api/2.0/bundle --oneline trace jq '{state_version, features}' .databricks/bundle/default/resources.json diff --git a/acceptance/bundle/dms/failed-delete/output.txt b/acceptance/bundle/dms/failed-delete/output.txt index 701b2051c14..17d98e0a4a3 100644 --- a/acceptance/bundle/dms/failed-delete/output.txt +++ b/acceptance/bundle/dms/failed-delete/output.txt @@ -59,6 +59,100 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged } } +=== The plan is a no-op now and records DEPLOYMENT_METADATA_SERVICE as its backend +>>> [CLI] bundle plan -o json +{ + "plan_version": 3, + "cli_version": "[CLI_VERSION]", + "lineage": "[UUID]", + "serial": 1, + "storage_backend": "DEPLOYMENT_METADATA_SERVICE", + "plan": { + "resources.jobs.stuck": { + "action": "skip", + "remote_state": { + "created_time": [UNIX_TIME_MILLIS], + "creator_user_name": "[USERNAME]", + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-failed-delete-[UNIQUE_NAME]/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "email_notifications": {}, + "format": "MULTI_TASK", + "job_id": [NUMID], + "max_concurrent_runs": 1, + "name": "stuck", + "queue": { + "enabled": true + }, + "run_as_user_name": "[USERNAME]", + "timeout_seconds": 0, + "webhook_notifications": {} + }, + "changes": { + "email_notifications": { + "action": "skip", + "reason": "empty", + "remote": {} + }, + "timeout_seconds": { + "action": "skip", + "reason": "empty", + "remote": 0 + }, + "webhook_notifications": { + "action": "skip", + "reason": "empty", + "remote": {} + } + } + } + } +} + +=== After deploy, the service lists the resource at version 1 +>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[NUMID] -o json +[ + { + "last_action_type": "OPERATION_ACTION_TYPE_CREATE", + "last_version_id": "1", + "resource_key": "jobs.stuck" + } +] + +=== The raw state after successful deploy +>>> print_state.py +{ + "state_version": 2, + "cli_version": "[CLI_VERSION]", + "lineage": "[UUID]", + "serial": 1, + "features": { + "record_deployment_history": {} + }, + "state": { + "resources.jobs.stuck": { + "__id__": "[NUMID]", + "state": { + "deployment": { + "deployment_id": "[NUMID]", + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-failed-delete-[UNIQUE_NAME]/default/state/metadata.json", + "version_id": "1" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "stuck", + "queue": { + "enabled": true + } + } + } + } +} + === A resource that fails to delete is recorded as a failed operation carrying the error, so the history says why rather than showing the resource gone >>> fault.py POST /api/2.2/jobs/delete 400 0 1 INVALID_PARAMETER_VALUE @@ -117,6 +211,17 @@ API message: Fault injected by test. } } +=== After failure, the service shows the operation failed +>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[NUMID] -o json +[ + { + "last_action_type": "OPERATION_ACTION_TYPE_DELETE", + "last_version_id": "2", + "resource_key": "jobs.stuck" + } +] + +=== The state shows the resource still exists but the operation is marked failed >>> print_state.py { "state_version": 2, diff --git a/acceptance/bundle/dms/failed-delete/script b/acceptance/bundle/dms/failed-delete/script index fff56460c21..ed544b27d7f 100644 --- a/acceptance/bundle/dms/failed-delete/script +++ b/acceptance/bundle/dms/failed-delete/script @@ -4,13 +4,29 @@ title "Deploy: the job is recorded" trace $CLI bundle deploy trace print_requests.py --dms //api/2.0/bundle +title "The plan is a no-op now and records DEPLOYMENT_METADATA_SERVICE as its backend" +trace $CLI bundle plan -o json | nostamp.py + +title "After deploy, the service lists the resource at version 1" +deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-failed-delete-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') +trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' + +title "The raw state after successful deploy" +trace print_state.py + title "A resource that fails to delete is recorded as a failed operation carrying the error, so the history says why rather than showing the resource gone" # The state entry survives too, so the next deploy still knows about the resource. trace fault.py "POST /api/2.2/jobs/delete" 400 0 1 INVALID_PARAMETER_VALUE trace musterr $CLI bundle destroy --auto-approve trace print_requests.py --dms //api/2.0/bundle + +title "After failure, the service shows the operation failed" +deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-failed-delete-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') +trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' + +title "The state shows the resource still exists but the operation is marked failed" trace print_state.py # print_state.py asks the service for the resources while recording, so drain those reads instead # of leaving them behind for the harness to diff. -print_requests.py //api/2.0 > /dev/null +rm -f out.requests.txt diff --git a/acceptance/bundle/dms/failed-recreate/output.txt b/acceptance/bundle/dms/failed-recreate/output.txt index 718b8108771..a85298826b2 100644 --- a/acceptance/bundle/dms/failed-recreate/output.txt +++ b/acceptance/bundle/dms/failed-recreate/output.txt @@ -6,6 +6,127 @@ Created schemas.foo Files: 6 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged +>>> print_requests.py --dms //api/2.0/bundle +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments", + "body": { + "display_name": "dms-failed-recreate-[UNIQUE_NAME]", + "initial_parent_path": "/Workspace/Users/[USERNAME]/.bundle/dms-failed-recreate-[UNIQUE_NAME]/default/state", + "target_name": "default", + "workspace_info": { + "file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-failed-recreate-[UNIQUE_NAME]/default/files", + "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-failed-recreate-[UNIQUE_NAME]/default" + } + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions", + "q": { + "version_id": "1" + }, + "body": { + "cli_version": "[CLI_VERSION]", + "version_type": "VERSION_TYPE_DEPLOY", + "operations": [ + { + "resource_key": "schemas.foo", + "action_type": "OPERATION_ACTION_TYPE_CREATE" + } + ] + } +} +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/schemas.foo", + "q": { + "update_mask": "state,error_message,resource_id,status" + }, + "body": { + "error_message": "", + "resource_id": "main.dms_failed_recreate_schema_[UNIQUE_NAME]", + "sequence_id": "0", + "state": "{\"state\":{\"catalog_name\":\"main\",\"comment\":\"v1\",\"name\":\"dms_failed_recreate_schema_[UNIQUE_NAME]\"}}", + "status": "OPERATION_STATUS_SUCCEEDED" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/complete", + "body": { + "completion_reason": "VERSION_COMPLETE_SUCCESS" + } +} + +=== The plan is a no-op and records DEPLOYMENT_METADATA_SERVICE as its backend +>>> [CLI] bundle plan -o json +{ + "plan_version": 3, + "cli_version": "[CLI_VERSION]", + "lineage": "[UUID]", + "serial": 1, + "storage_backend": "DEPLOYMENT_METADATA_SERVICE", + "plan": { + "resources.schemas.foo": { + "action": "skip", + "remote_state": { + "browse_only": false, + "catalog_name": "main", + "catalog_type": "MANAGED_CATALOG", + "comment": "v1", + "created_at": [UNIX_TIME_MILLIS][0], + "created_by": "[USERNAME]", + "effective_predictive_optimization_flag": { + "inherited_from_name": "[METASTORE_NAME]", + "inherited_from_type": "METASTORE", + "value": "ENABLE" + }, + "enable_predictive_optimization": "INHERIT", + "full_name": "main.dms_failed_recreate_schema_[UNIQUE_NAME]", + "metastore_id": "[UUID]", + "name": "dms_failed_recreate_schema_[UNIQUE_NAME]", + "owner": "[USERNAME]", + "schema_id": "[UUID]", + "updated_at": [UNIX_TIME_MILLIS][0], + "updated_by": "[USERNAME]" + } + } + } +} + +=== After deploy, the service lists the resource at version 1 +>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[NUMID] -o json +[ + { + "last_action_type": "OPERATION_ACTION_TYPE_CREATE", + "last_version_id": "1", + "resource_key": "schemas.foo" + } +] + +=== The raw state after successful deploy +>>> print_state.py +{ + "state_version": 2, + "cli_version": "[CLI_VERSION]", + "lineage": "[UUID]", + "serial": 1, + "features": { + "record_deployment_history": {} + }, + "state": { + "resources.schemas.foo": { + "__id__": "main.dms_failed_recreate_schema_[UNIQUE_NAME]", + "state": { + "catalog_name": "main", + "comment": "v1", + "name": "dms_failed_recreate_schema_[UNIQUE_NAME]" + } + } + } +} + === A recreate deletes and then fails to create. The delete went through, so the resource really is gone: the failed operation clears its state (self-contained, not assuming the delete's own write reached the service), rather than describing the resource as it was before the deploy >>> update_file.py databricks.yml catalog_name: main catalog_name: dms_other_[UNIQUE_NAME] @@ -77,8 +198,21 @@ Files: 4 uploaded, 0 deleted } === The resource is not listed: state is what projects a resource, and the failed recreate left none ->>> MSYS_NO_PATHCONV=1 [CLI] api get /api/2.0/bundle/deployments/[NUMID]/resources -{} +>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[NUMID] -o json +[] + +=== The raw state after failed recreate shows no resource +>>> print_state.py +{ + "state_version": 2, + "cli_version": "[CLI_VERSION]", + "lineage": "[UUID]", + "serial": 2, + "features": { + "record_deployment_history": {} + }, + "state": {} +} === Planning from DMS state alone creates the schema, which is what has to happen: it no longer exists >>> [CLI] bundle plan diff --git a/acceptance/bundle/dms/failed-recreate/script b/acceptance/bundle/dms/failed-recreate/script index e2d4a68f508..6c0b82a1a69 100644 --- a/acceptance/bundle/dms/failed-recreate/script +++ b/acceptance/bundle/dms/failed-recreate/script @@ -2,7 +2,17 @@ envsubst < databricks.yml.tmpl > databricks.yml title "Deploy the schema, so there is a resource to recreate" trace $CLI bundle deploy -rm -f out.requests.txt +trace print_requests.py --dms //api/2.0/bundle + +title "The plan is a no-op and records DEPLOYMENT_METADATA_SERVICE as its backend" +trace $CLI bundle plan -o json | nostamp.py + +title "After deploy, the service lists the resource at version 1" +deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-failed-recreate-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') +trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' + +title "The raw state after successful deploy" +trace print_state.py title "A recreate deletes and then fails to create. The delete went through, so the resource really is gone: the failed operation clears its state (self-contained, not assuming the delete's own write reached the service), rather than describing the resource as it was before the deploy" # Unity Catalog has no catalog called "other", so make one to move the schema into. @@ -17,12 +27,11 @@ trace musterr $CLI bundle deploy --auto-approve trace print_requests.py --dms //api/2.0/bundle title "The resource is not listed: state is what projects a resource, and the failed recreate left none" -# The deployment ID is the workspace node's ID; read it back the way the CLI does. -# Extracted with python, not jq: the ID exceeds 2^53 and jq 1.6 rounds it. -deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-failed-recreate-${UNIQUE_NAME}/default/state/resources.deployment.json" -o json | python3 -c 'import sys,json; print(json.load(sys.stdin)["object_id"])') -# MSYS_NO_PATHCONV as above: Git Bash on Windows would rewrite the leading-'/' API -# path into C:/Program Files/Git/api/2.0/... before the CLI saw it. -trace MSYS_NO_PATHCONV=1 $CLI api get "/api/2.0/bundle/deployments/${deployment_id}/resources" +deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-failed-recreate-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') +trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' + +title "The raw state after failed recreate shows no resource" +trace print_state.py title "Planning from DMS state alone creates the schema, which is what has to happen: it no longer exists" rm -rf .databricks diff --git a/acceptance/bundle/dms/failed-update-permissions/output.txt b/acceptance/bundle/dms/failed-update-permissions/output.txt index 080f4b6c8be..ac97f5ce901 100644 --- a/acceptance/bundle/dms/failed-update-permissions/output.txt +++ b/acceptance/bundle/dms/failed-update-permissions/output.txt @@ -7,6 +7,149 @@ Created jobs.foo.permissions Files: 6 uploaded, 0 deleted Resources: 2 created, 0 changed, 0 deleted, 0 unchanged +=== The plan records DEPLOYMENT_METADATA_SERVICE backend and the job +>>> [CLI] bundle plan -o json +{ + "plan_version": 3, + "cli_version": "[CLI_VERSION]", + "lineage": "[UUID]", + "serial": 1, + "storage_backend": "DEPLOYMENT_METADATA_SERVICE", + "plan": { + "resources.jobs.foo": { + "action": "skip", + "remote_state": { + "created_time": [UNIX_TIME_MILLIS], + "creator_user_name": "[USERNAME]", + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-failed-update-permissions-[UNIQUE_NAME]/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "email_notifications": {}, + "format": "MULTI_TASK", + "job_id": [NUMID], + "max_concurrent_runs": 1, + "name": "foo", + "queue": { + "enabled": true + }, + "run_as_user_name": "[USERNAME]", + "timeout_seconds": 0, + "webhook_notifications": {} + }, + "changes": { + "email_notifications": { + "action": "skip", + "reason": "empty", + "remote": {} + }, + "timeout_seconds": { + "action": "skip", + "reason": "empty", + "remote": 0 + }, + "webhook_notifications": { + "action": "skip", + "reason": "empty", + "remote": {} + } + } + }, + "resources.jobs.foo.permissions": { + "depends_on": [ + { + "node": "resources.jobs.foo", + "label": "${resources.jobs.foo.id}" + } + ], + "action": "skip", + "remote_state": { + "object_id": "/jobs/[NUMID]", + "__embed__": [ + { + "level": "CAN_VIEW", + "user_name": "viewer@example.com" + }, + { + "level": "IS_OWNER", + "user_name": "[USERNAME]" + } + ] + } + } + } +} + +=== The service lists the job at version 1 +>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[NUMID] -o json +[ + { + "last_action_type": "OPERATION_ACTION_TYPE_CREATE", + "last_version_id": "1", + "resource_key": "jobs.foo" + }, + { + "last_action_type": "OPERATION_ACTION_TYPE_CREATE", + "last_version_id": "1", + "resource_key": "jobs.foo.permissions" + } +] + +=== The raw recorded state after deploy +>>> print_state.py +{ + "state_version": 2, + "cli_version": "[CLI_VERSION]", + "lineage": "[UUID]", + "serial": 1, + "features": { + "record_deployment_history": {} + }, + "state": { + "resources.jobs.foo": { + "__id__": "[NUMID]", + "state": { + "deployment": { + "deployment_id": "[NUMID]", + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-failed-update-permissions-[UNIQUE_NAME]/default/state/metadata.json", + "version_id": "1" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "foo", + "queue": { + "enabled": true + } + } + }, + "resources.jobs.foo.permissions": { + "__id__": "/jobs/[NUMID]", + "state": { + "object_id": "/jobs/[NUMID]", + "__embed__": [ + { + "level": "CAN_VIEW", + "user_name": "viewer@example.com" + }, + { + "level": "IS_OWNER", + "user_name": "[USERNAME]" + } + ] + }, + "depends_on": [ + { + "node": "resources.jobs.foo", + "label": "${resources.jobs.foo.id}" + } + ] + } + } +} + === A permissions update that fails records the failure with the permissions it still has. The resource ID here is a path rather than a number, and the state is the embedded permission list, so this covers a shape the resource tests do not >>> update_file.py databricks.yml CAN_VIEW CAN_MANAGE @@ -23,6 +166,75 @@ API message: Fault injected by test. Files: 3 uploaded, 0 deleted +=== The service lists the job with the failure +>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[NUMID] -o json +[ + { + "last_action_type": "OPERATION_ACTION_TYPE_CREATE", + "last_version_id": "1", + "resource_key": "jobs.foo" + }, + { + "last_action_type": "OPERATION_ACTION_TYPE_UPDATE", + "last_version_id": "2", + "resource_key": "jobs.foo.permissions" + } +] + +=== The raw recorded state after the failure +>>> print_state.py +{ + "state_version": 2, + "cli_version": "[CLI_VERSION]", + "lineage": "[UUID]", + "serial": 1, + "features": { + "record_deployment_history": {} + }, + "state": { + "resources.jobs.foo": { + "__id__": "[NUMID]", + "state": { + "deployment": { + "deployment_id": "[NUMID]", + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-failed-update-permissions-[UNIQUE_NAME]/default/state/metadata.json", + "version_id": "1" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "foo", + "queue": { + "enabled": true + } + } + }, + "resources.jobs.foo.permissions": { + "__id__": "/jobs/[NUMID]", + "state": { + "object_id": "/jobs/[NUMID]", + "__embed__": [ + { + "level": "CAN_VIEW", + "user_name": "viewer@example.com" + }, + { + "level": "IS_OWNER", + "user_name": "[USERNAME]" + } + ] + }, + "depends_on": [ + { + "node": "resources.jobs.foo", + "label": "${resources.jobs.foo.id}" + } + ] + } + } +} + >>> print_requests.py --dms //api/2.0/bundle --oneline {"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": {"version_id": "2"}, "body": {"cli_version": "[CLI_VERSION]", "version_type": "VERSION_TYPE_DEPLOY", "previous_version_id": "1", "operations": [{"resource_key": "jobs.foo.permissions", "action_type": "OPERATION_ACTION_TYPE_UPDATE"}]}} {"method": "PATCH", "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations/jobs.foo.permissions", "q": {"update_mask": "state,error_message,resource_id,status"}, "body": {"error_message": "updating id=/jobs/[NUMID]: Fault injected by test. (403 PERMISSION_DENIED)", "resource_id": "/jobs/[NUMID]", "sequence_id": "0", "state": "{\"state\":{\"object_id\":\"/jobs/[NUMID]\",\"__embed__\":[{\"level\":\"CAN_VIEW\",\"user_name\":\"viewer@example.com\"},{\"level\":\"IS_OWNER\",\"user_name\":\"[USERNAME]\"}]},\"depends_on\":[{\"node\":\"resources.jobs.foo\",\"label\":\"${resources.jobs.foo.id}\"}]}", "status": "OPERATION_STATUS_FAILED"}} diff --git a/acceptance/bundle/dms/failed-update-permissions/script b/acceptance/bundle/dms/failed-update-permissions/script index 56141a30f4c..6883d8f3775 100644 --- a/acceptance/bundle/dms/failed-update-permissions/script +++ b/acceptance/bundle/dms/failed-update-permissions/script @@ -4,9 +4,27 @@ title "Deploy the job and its permissions" trace $CLI bundle deploy rm -f out.requests.txt +title "The plan records DEPLOYMENT_METADATA_SERVICE backend and the job" +trace $CLI bundle plan -o json | nostamp.py + +title "The service lists the job at version 1" +deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-failed-update-permissions-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') +trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' + +title "The raw recorded state after deploy" +trace print_state.py + title "A permissions update that fails records the failure with the permissions it still has. The resource ID here is a path rather than a number, and the state is the embedded permission list, so this covers a shape the resource tests do not" trace update_file.py databricks.yml "CAN_VIEW" "CAN_MANAGE" # A 403 is what the API returns when the caller cannot change a job's owner. trace fault.py "PUT /api/2.0/permissions/jobs/*" 403 0 1 PERMISSION_DENIED trace musterr $CLI bundle deploy --auto-approve + +title "The service lists the job with the failure" +deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-failed-update-permissions-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') +trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' + +title "The raw recorded state after the failure" +trace print_state.py + trace print_requests.py --dms //api/2.0/bundle --oneline diff --git a/acceptance/bundle/dms/failed-update/output.txt b/acceptance/bundle/dms/failed-update/output.txt index 125c16608b0..2cb6c855d5f 100644 --- a/acceptance/bundle/dms/failed-update/output.txt +++ b/acceptance/bundle/dms/failed-update/output.txt @@ -6,6 +6,127 @@ Created schemas.foo Files: 6 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged +>>> print_requests.py --dms //api/2.0/bundle +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments", + "body": { + "display_name": "dms-failed-update-[UNIQUE_NAME]", + "initial_parent_path": "/Workspace/Users/[USERNAME]/.bundle/dms-failed-update-[UNIQUE_NAME]/default/state", + "target_name": "default", + "workspace_info": { + "file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-failed-update-[UNIQUE_NAME]/default/files", + "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-failed-update-[UNIQUE_NAME]/default" + } + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions", + "q": { + "version_id": "1" + }, + "body": { + "cli_version": "[CLI_VERSION]", + "version_type": "VERSION_TYPE_DEPLOY", + "operations": [ + { + "resource_key": "schemas.foo", + "action_type": "OPERATION_ACTION_TYPE_CREATE" + } + ] + } +} +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/schemas.foo", + "q": { + "update_mask": "state,error_message,resource_id,status" + }, + "body": { + "error_message": "", + "resource_id": "main.dms_failed_update_schema_[UNIQUE_NAME]", + "sequence_id": "0", + "state": "{\"state\":{\"catalog_name\":\"main\",\"comment\":\"v1\",\"name\":\"dms_failed_update_schema_[UNIQUE_NAME]\"}}", + "status": "OPERATION_STATUS_SUCCEEDED" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/complete", + "body": { + "completion_reason": "VERSION_COMPLETE_SUCCESS" + } +} + +=== The plan is a no-op and records DEPLOYMENT_METADATA_SERVICE as its backend +>>> [CLI] bundle plan -o json +{ + "plan_version": 3, + "cli_version": "[CLI_VERSION]", + "lineage": "[UUID]", + "serial": 1, + "storage_backend": "DEPLOYMENT_METADATA_SERVICE", + "plan": { + "resources.schemas.foo": { + "action": "skip", + "remote_state": { + "browse_only": false, + "catalog_name": "main", + "catalog_type": "MANAGED_CATALOG", + "comment": "v1", + "created_at": [UNIX_TIME_MILLIS][0], + "created_by": "[USERNAME]", + "effective_predictive_optimization_flag": { + "inherited_from_name": "[METASTORE_NAME]", + "inherited_from_type": "METASTORE", + "value": "ENABLE" + }, + "enable_predictive_optimization": "INHERIT", + "full_name": "main.dms_failed_update_schema_[UNIQUE_NAME]", + "metastore_id": "[UUID]", + "name": "dms_failed_update_schema_[UNIQUE_NAME]", + "owner": "[USERNAME]", + "schema_id": "[UUID]", + "updated_at": [UNIX_TIME_MILLIS][0], + "updated_by": "[USERNAME]" + } + } + } +} + +=== After deploy, the service lists the resource at version 1 +>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[NUMID] -o json +[ + { + "last_action_type": "OPERATION_ACTION_TYPE_CREATE", + "last_version_id": "1", + "resource_key": "schemas.foo" + } +] + +=== The raw state after successful deploy +>>> print_state.py +{ + "state_version": 2, + "cli_version": "[CLI_VERSION]", + "lineage": "[UUID]", + "serial": 1, + "features": { + "record_deployment_history": {} + }, + "state": { + "resources.schemas.foo": { + "__id__": "main.dms_failed_update_schema_[UNIQUE_NAME]", + "state": { + "catalog_name": "main", + "comment": "v1", + "name": "dms_failed_update_schema_[UNIQUE_NAME]" + } + } + } +} + === An update that fails before writing any state only marks its operation failed. It names no state, so the deployment keeps describing the schema the last successful version recorded - which is right, because nothing touched it >>> update_file.py databricks.yml comment: v1 comment: v2 @@ -64,19 +185,35 @@ Files: 3 uploaded, 0 deleted } === The schema is still listed as the previous version left it, so the next plan updates it rather than creating a second one ->>> MSYS_NO_PATHCONV=1 [CLI] api get /api/2.0/bundle/deployments/[NUMID]/resources +>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[NUMID] -o json +[ + { + "last_action_type": "OPERATION_ACTION_TYPE_UPDATE", + "last_version_id": "2", + "resource_key": "schemas.foo" + } +] + +=== The raw state after failed update shows the previous version +>>> print_state.py { - "resources": [ - { - "last_action_type": "OPERATION_ACTION_TYPE_UPDATE", - "last_version_id": "2", - "name": "deployments/[NUMID]/resources/schemas.foo", - "resource_id": "main.dms_failed_update_schema_[UNIQUE_NAME]", - "resource_key": "schemas.foo", - "resource_type": "", - "state": "{\"state\":{\"catalog_name\":\"main\",\"comment\":\"v1\",\"name\":\"dms_failed_update_schema_[UNIQUE_NAME]\"}}" - } - ] + "state_version": 2, + "cli_version": "[CLI_VERSION]", + "lineage": "[UUID]", + "serial": 1, + "features": { + "record_deployment_history": {} + }, + "state": { + "resources.schemas.foo": { + "__id__": "main.dms_failed_update_schema_[UNIQUE_NAME]", + "state": { + "catalog_name": "main", + "comment": "v1", + "name": "dms_failed_update_schema_[UNIQUE_NAME]" + } + } + } } === Planning from DMS state alone updates the schema rather than creating a second one, which is what recording the prior state buys diff --git a/acceptance/bundle/dms/failed-update/script b/acceptance/bundle/dms/failed-update/script index 91640bd7631..7d223af223b 100644 --- a/acceptance/bundle/dms/failed-update/script +++ b/acceptance/bundle/dms/failed-update/script @@ -2,7 +2,17 @@ envsubst < databricks.yml.tmpl > databricks.yml title "Deploy the schema, so there is an existing resource to update" trace $CLI bundle deploy -rm -f out.requests.txt +trace print_requests.py --dms //api/2.0/bundle + +title "The plan is a no-op and records DEPLOYMENT_METADATA_SERVICE as its backend" +trace $CLI bundle plan -o json | nostamp.py + +title "After deploy, the service lists the resource at version 1" +deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-failed-update-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') +trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' + +title "The raw state after successful deploy" +trace print_state.py title "An update that fails before writing any state only marks its operation failed. It names no state, so the deployment keeps describing the schema the last successful version recorded - which is right, because nothing touched it" trace update_file.py databricks.yml "comment: v1" "comment: v2" @@ -13,12 +23,11 @@ trace musterr $CLI bundle deploy --auto-approve trace print_requests.py --dms //api/2.0/bundle title "The schema is still listed as the previous version left it, so the next plan updates it rather than creating a second one" -# The deployment ID is the workspace node's ID; read it back the way the CLI does. -# Extracted with python, not jq: the ID exceeds 2^53 and jq 1.6 rounds it. -deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-failed-update-${UNIQUE_NAME}/default/state/resources.deployment.json" -o json | python3 -c 'import sys,json; print(json.load(sys.stdin)["object_id"])') -# MSYS_NO_PATHCONV as above: Git Bash on Windows would rewrite the leading-'/' API -# path into C:/Program Files/Git/api/2.0/... before the CLI saw it. -trace MSYS_NO_PATHCONV=1 $CLI api get "/api/2.0/bundle/deployments/${deployment_id}/resources" +deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-failed-update-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') +trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' + +title "The raw state after failed update shows the previous version" +trace print_state.py title "Planning from DMS state alone updates the schema rather than creating a second one, which is what recording the prior state buys" rm -rf .databricks diff --git a/acceptance/bundle/dms/operation-upload-fails-once/output.txt b/acceptance/bundle/dms/operation-upload-fails-once/output.txt index abdeedeb959..8e74957520c 100644 --- a/acceptance/bundle/dms/operation-upload-fails-once/output.txt +++ b/acceptance/bundle/dms/operation-upload-fails-once/output.txt @@ -6,6 +6,74 @@ Created schemas.foo Files: 6 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged +=== The plan records DEPLOYMENT_METADATA_SERVICE backend and the schema +>>> [CLI] bundle plan -o json +{ + "plan_version": 3, + "cli_version": "[CLI_VERSION]", + "lineage": "[UUID]", + "serial": 1, + "storage_backend": "DEPLOYMENT_METADATA_SERVICE", + "plan": { + "resources.schemas.foo": { + "action": "skip", + "remote_state": { + "browse_only": false, + "catalog_name": "main", + "catalog_type": "MANAGED_CATALOG", + "comment": "v1", + "created_at": [UNIX_TIME_MILLIS][0], + "created_by": "[USERNAME]", + "effective_predictive_optimization_flag": { + "inherited_from_name": "[METASTORE_NAME]", + "inherited_from_type": "METASTORE", + "value": "ENABLE" + }, + "enable_predictive_optimization": "INHERIT", + "full_name": "main.dms_upload_fails_once_schema_[UNIQUE_NAME]", + "metastore_id": "[UUID]", + "name": "dms_upload_fails_once_schema_[UNIQUE_NAME]", + "owner": "[USERNAME]", + "schema_id": "[UUID]", + "updated_at": [UNIX_TIME_MILLIS][0], + "updated_by": "[USERNAME]" + } + } + } +} + +=== The service lists the schema at version 1 +>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[NUMID] -o json +[ + { + "last_action_type": "OPERATION_ACTION_TYPE_CREATE", + "last_version_id": "1", + "resource_key": "schemas.foo" + } +] + +=== The raw recorded state after deploy +>>> print_state.py +{ + "state_version": 2, + "cli_version": "[CLI_VERSION]", + "lineage": "[UUID]", + "serial": 1, + "features": { + "record_deployment_history": {} + }, + "state": { + "resources.schemas.foo": { + "__id__": "main.dms_upload_fails_once_schema_[UNIQUE_NAME]", + "state": { + "catalog_name": "main", + "comment": "v1", + "name": "dms_upload_fails_once_schema_[UNIQUE_NAME]" + } + } + } +} + === A recreate records twice, and the first one fails: the second must still carry the sequence id staging left, because a rejected update earned no new one >>> fault.py PATCH /api/2.0/bundle/* 500 0 1 @@ -25,6 +93,38 @@ API message: Fault injected by test. Files: 4 uploaded, 0 deleted +=== The service lists the schema with the failure +>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[NUMID] -o json +[ + { + "last_action_type": "OPERATION_ACTION_TYPE_RECREATE", + "last_version_id": "2", + "resource_key": "schemas.foo" + } +] + +=== The raw recorded state after the failure +>>> print_state.py +{ + "state_version": 2, + "cli_version": "[CLI_VERSION]", + "lineage": "[UUID]", + "serial": 2, + "features": { + "record_deployment_history": {} + }, + "state": { + "resources.schemas.foo": { + "__id__": "dms_other_[UNIQUE_NAME].dms_upload_fails_once_schema_[UNIQUE_NAME]", + "state": { + "catalog_name": "dms_other_[UNIQUE_NAME]", + "comment": "v1", + "name": "dms_upload_fails_once_schema_[UNIQUE_NAME]" + } + } + } +} + >>> print_requests.py --dms //api/2.0/bundle { "method": "POST", diff --git a/acceptance/bundle/dms/operation-upload-fails-once/script b/acceptance/bundle/dms/operation-upload-fails-once/script index 5b609b1b8ee..d69268d8f87 100644 --- a/acceptance/bundle/dms/operation-upload-fails-once/script +++ b/acceptance/bundle/dms/operation-upload-fails-once/script @@ -4,6 +4,16 @@ title "Deploy so there is a recorded resource to recreate" trace $CLI bundle deploy rm -f out.requests.txt +title "The plan records DEPLOYMENT_METADATA_SERVICE backend and the schema" +trace $CLI bundle plan -o json | nostamp.py + +title "The service lists the schema at version 1" +deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-upload-fails-once-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') +trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' + +title "The raw recorded state after deploy" +trace print_state.py + title "A recreate records twice, and the first one fails: the second must still carry the sequence id staging left, because a rejected update earned no new one" trace fault.py "PATCH /api/2.0/bundle/*" 500 0 1 # Unity Catalog has no catalog called "other", so make one to move the schema into. @@ -12,4 +22,12 @@ $CLI catalogs create "$other_catalog" &> LOG.catalog trap '$CLI catalogs delete "$other_catalog" --force >> LOG.catalog 2>&1' EXIT trace update_file.py databricks.yml "catalog_name: main" "catalog_name: ${other_catalog}" trace musterr $CLI bundle deploy --auto-approve + +title "The service lists the schema with the failure" +deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-upload-fails-once-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') +trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' + +title "The raw recorded state after the failure" +trace print_state.py + trace print_requests.py --dms //api/2.0/bundle diff --git a/acceptance/bundle/dms/record-failure/output.txt b/acceptance/bundle/dms/record-failure/output.txt index 819004844d7..a08dd0df054 100644 --- a/acceptance/bundle/dms/record-failure/output.txt +++ b/acceptance/bundle/dms/record-failure/output.txt @@ -11,6 +11,46 @@ API message: Node type nonexistent-node-type is not supported. Supported node ty Files: 6 uploaded, 0 deleted +>>> [CLI] bundle plan -o json +{ + "plan_version": 3, + "cli_version": "[CLI_VERSION]", + "storage_backend": "DEPLOYMENT_METADATA_SERVICE", + "plan": { + "resources.jobs.doomed": { + "action": "create", + "new_state": { + "value": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-record-failure-[UNIQUE_NAME]/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "doomed", + "queue": { + "enabled": true + }, + "tasks": [ + { + "new_cluster": { + "node_type_id": "nonexistent-node-type", + "num_workers": 1, + "spark_version": "15.4.x-scala2.12" + }, + "notebook_task": { + "notebook_path": "/nonexistent" + }, + "task_key": "doomed" + } + ] + } + } + } + } +} + >>> print_requests.py --dms //api/2.0/bundle { "method": "POST", @@ -66,6 +106,11 @@ Files: 6 uploaded, 0 deleted >>> MSYS_NO_PATHCONV=1 [CLI] api get /api/2.0/bundle/deployments/[NUMID]/resources {} +>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[NUMID] -o json +[] + +>>> print_state.py + === Redeploying from DMS state alone still creates the failed resource: it was never applied, so it must not be skipped as unchanged >>> [CLI] bundle plan create jobs.doomed diff --git a/acceptance/bundle/dms/record-failure/script b/acceptance/bundle/dms/record-failure/script index 8b629dfd6be..8fc543ff20d 100644 --- a/acceptance/bundle/dms/record-failure/script +++ b/acceptance/bundle/dms/record-failure/script @@ -2,6 +2,7 @@ envsubst < databricks.yml.tmpl > databricks.yml title "A resource that fails to apply is recorded as a failed operation carrying the error, so the deployment history says why rather than omitting the resource" trace musterr $CLI bundle deploy +trace $CLI bundle plan -o json | nostamp.py trace print_requests.py --dms //api/2.0/bundle title "The failed resource is not listed at all: state is what projects a resource, and a failed create records none, so a later deploy plans to create it rather than treating it as already deployed" @@ -12,6 +13,10 @@ deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/$ # path into C:/Program Files/Git/api/2.0/... before the CLI saw it. trace MSYS_NO_PATHCONV=1 $CLI api get "/api/2.0/bundle/deployments/${deployment_id}/resources" +trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' + +trace print_state.py + title "Redeploying from DMS state alone still creates the failed resource: it was never applied, so it must not be skipped as unchanged" rm -rf .databricks trace $CLI bundle plan diff --git a/acceptance/bundle/dms/version-never-created/output.txt b/acceptance/bundle/dms/version-never-created/output.txt index 8a7e5fe8506..db21f2a000a 100644 --- a/acceptance/bundle/dms/version-never-created/output.txt +++ b/acceptance/bundle/dms/version-never-created/output.txt @@ -11,6 +11,33 @@ API message: Internal error Files: 5 uploaded, 0 deleted +>>> [CLI] bundle plan -o json +{ + "plan_version": 3, + "cli_version": "[CLI_VERSION]", + "storage_backend": "DEPLOYMENT_METADATA_SERVICE", + "plan": { + "resources.jobs.foo": { + "action": "create", + "new_state": { + "value": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-version-never-created/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "foo", + "queue": { + "enabled": true + } + } + } + } + } +} + >>> MSYS_NO_PATHCONV=1 [CLI] workspace get-status /Workspace/Users/[USERNAME]/.bundle/dms-version-never-created/default/state/resources.deployment.json { "object_type": "BUNDLE_DEPLOYMENT" @@ -19,6 +46,11 @@ Files: 5 uploaded, 0 deleted >>> print_requests.py --dms //api/2.0/bundle --get --oneline {"method": "POST", "path": "/api/2.0/bundle/deployments", "body": {"display_name": "dms-version-never-created", "initial_parent_path": "/Workspace/Users/[USERNAME]/.bundle/dms-version-never-created/default/state", "target_name": "default", "workspace_info": {"file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-version-never-created/default/files", "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-version-never-created/default"}}} {"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": {"version_id": "1"}, "body": {"cli_version": "[CLI_VERSION]", "version_type": "VERSION_TYPE_DEPLOY", "operations": [{"resource_key": "jobs.foo", "action_type": "OPERATION_ACTION_TYPE_CREATE"}]}} +{"method": "GET", "path": "/api/2.0/bundle/deployments/[NUMID]"} +{"method": "GET", "path": "/api/2.0/bundle/deployments/[NUMID]/resources"} + +>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[NUMID] -o json +[] === The next deploy reuses the ID that node names and retries version 1, rather than creating a second deployment >>> musterr [CLI] bundle deploy @@ -33,6 +65,7 @@ API message: Internal error Files: 2 uploaded, 0 deleted >>> print_requests.py --dms //api/2.0/bundle --get --oneline +{"method": "GET", "path": "/api/2.0/bundle/deployments/[NUMID]/resources"} {"method": "GET", "path": "/api/2.0/bundle/deployments/[NUMID]"} {"method": "GET", "path": "/api/2.0/bundle/deployments/[NUMID]/resources"} {"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": {"version_id": "1"}, "body": {"cli_version": "[CLI_VERSION]", "version_type": "VERSION_TYPE_DEPLOY", "operations": [{"resource_key": "jobs.foo", "action_type": "OPERATION_ACTION_TYPE_CREATE"}]}} diff --git a/acceptance/bundle/dms/version-never-created/script b/acceptance/bundle/dms/version-never-created/script index 58178feaf53..aca7b030713 100644 --- a/acceptance/bundle/dms/version-never-created/script +++ b/acceptance/bundle/dms/version-never-created/script @@ -1,8 +1,12 @@ title "The first version fails, so no deployment record exists - only the node naming its ID" trace musterr $CLI bundle deploy +trace $CLI bundle plan -o json | nostamp.py trace MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-version-never-created/default/state/resources.deployment.json" | jq '{object_type}' trace print_requests.py --dms //api/2.0/bundle --get --oneline +deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-version-never-created/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') +trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' + title "The next deploy reuses the ID that node names and retries version 1, rather than creating a second deployment" trace musterr $CLI bundle deploy trace print_requests.py --dms //api/2.0/bundle --get --oneline From b4d23d20072be2329e499e995a2999c426c7e87b Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Fri, 4 Sep 2026 14:10:55 +0000 Subject: [PATCH 22/46] bundle/dms: review fixes - naming, single lineage fields, user-facing wording - rename StartRecording -> InitializeOperationBuffer (it opens the buffer; kept separate from Open because a first deploy's id does not exist until after approval) - collapse the duplicated deployment/version fields to one public pair (DeploymentID/LatestVersionID); InitializeOperationBuffer records the first-deploy id, CompleteVersion derives the version from it - RecordingError is now a no-op unless the backend is the deployment metadata service, keeping the non-recording path cleanly separate (like RecordFailure) - stop surfacing the internal "deployment metadata service" name in user-facing errors; say "deployment history feature enabled" / "the deployment history service" - drop the "in place of nil client checks" comment and restore the original featureStateVersion wording (minimal C12 diff) Co-authored-by: Isaac --- .../bundle/dms/existing-state/output.txt | 6 +- bundle/direct/bundle_apply.go | 2 +- bundle/direct/dstate/state.go | 79 +++++++++++-------- bundle/phases/dms.go | 2 +- libs/dms/client.go | 2 +- libs/dms/resources.go | 2 +- 6 files changed, 53 insertions(+), 40 deletions(-) diff --git a/acceptance/bundle/dms/existing-state/output.txt b/acceptance/bundle/dms/existing-state/output.txt index 95352e1dcb0..0d35577a43c 100644 --- a/acceptance/bundle/dms/existing-state/output.txt +++ b/acceptance/bundle/dms/existing-state/output.txt @@ -66,7 +66,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged === Enabling recording is refused: this deployment already exists but the service has no record of it, and treating the service as authoritative would create its resources a second time >>> DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY=true [CLI] bundle deploy -Error: this deployment already exists and is not recorded with the deployment metadata service, so it cannot be recorded without redeploying its resources +Error: this deployment already exists and is not recorded with the deployment history feature enabled, so it cannot be recorded without redeploying its resources To record this bundle's history, start it over as a new deployment: 1. remove experimental.record_deployment_history from your bundle configuration @@ -78,7 +78,7 @@ To record this bundle's history, start it over as a new deployment: === The check is against the remote state, not the local cache: wiping .databricks still refuses, because the deploy pulls the state back from the workspace >>> DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY=true [CLI] bundle deploy -Error: this deployment already exists and is not recorded with the deployment metadata service, so it cannot be recorded without redeploying its resources +Error: this deployment already exists and is not recorded with the deployment history feature enabled, so it cannot be recorded without redeploying its resources To record this bundle's history, start it over as a new deployment: 1. remove experimental.record_deployment_history from your bundle configuration @@ -88,7 +88,7 @@ To record this bundle's history, start it over as a new deployment: === A leftover local WAL is irrelevant too: under recording it is discarded rather than replayed, so the refusal stands >>> DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY=true [CLI] bundle deploy -Error: this deployment already exists and is not recorded with the deployment metadata service, so it cannot be recorded without redeploying its resources +Error: this deployment already exists and is not recorded with the deployment history feature enabled, so it cannot be recorded without redeploying its resources To record this bundle's history, start it over as a new deployment: 1. remove experimental.record_deployment_history from your bundle configuration diff --git a/bundle/direct/bundle_apply.go b/bundle/direct/bundle_apply.go index 8c2de1f29ee..74bfa226cbf 100644 --- a/bundle/direct/bundle_apply.go +++ b/bundle/direct/bundle_apply.go @@ -43,7 +43,7 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa return } - // The state DB records every write with DMS from here on (via the buffer StartRecording opened + // The state DB records every write with DMS from here on (via the buffer InitializeOperationBuffer opened // in the deploy phase), so the service mirrors the WAL. Writes go out on one background // goroutine, off the apply path, and are drained below once every worker has finished recording. g.Run(defaultParallelism, func(resourceKey string, failedDependency *string) bool { diff --git a/bundle/direct/dstate/state.go b/bundle/direct/dstate/state.go index 17c4eb9d4e7..ef67ccfc559 100644 --- a/bundle/direct/dstate/state.go +++ b/bundle/direct/dstate/state.go @@ -33,18 +33,27 @@ const ( maxWalEntrySize = 10 * 1024 * 1024 walSuffix = ".wal" - // featureStateVersion is the schema version a future CLI will write once it records feature flags - // in the state version itself. This CLI does not write it, but reads such states (see - // migrateState): featureStateVersion with no features is equivalent to currentStateVersion and is - // accepted as-is (the on-disk version is left at featureStateVersion, not flipped down); - // featureStateVersion with any feature depends on capabilities this CLI lacks, so it is refused. - // Forward-compat scaffolding so a later release can write featureStateVersion + features without - // this CLI rejecting them outright. Always 3. + // featureStateVersion is the schema version a future CLI will write once it + // records deployment state "feature flags" (see Header.Features). This CLI does + // not write it and records no features; it exists now only so this CLI reads + // such states correctly (see migrateState): + // - featureStateVersion with no features -> accept and leave the version as-is + // - featureStateVersion with any feature -> refuse, tell the user to upgrade + // + // A featureStateVersion state with no features is equivalent to + // currentStateVersion, but we deliberately do not flip the on-disk version down + // to currentStateVersion: a state written at featureStateVersion stays at + // featureStateVersion. This is forward-compat scaffolding so that a later release + // can start writing featureStateVersion + features without older CLIs (with this + // change) either mishandling a feature they lack or rejecting a featureless state + // outright. featureStateVersion is always 3. featureStateVersion = 3 - // supportedStateVersion is the highest schema version this CLI can read. Normally equal to - // currentStateVersion - the version it reads is the version it writes - but this CLI can also - // read (never write) featureStateVersion. A state newer than this is rejected as too new. + // supportedStateVersion is the highest schema version this CLI can read. It is + // normally equal to currentStateVersion — the version this CLI reads is the + // version it writes — and exceeds it only during a two-phase version bump like + // the current feature-flag scaffolding, where this CLI reads (but does not + // write) featureStateVersion. A state newer than this is rejected as too new. supportedStateVersion = featureStateVersion ) @@ -105,7 +114,7 @@ var errStaleWAL = errors.New("stale WAL") // present an operation-appropriate message via errors.Is. var ErrUnsettingRecording = errors.New(`unsetting experimental.record_deployment_history is not supported -This deployment's resources are recorded with the deployment metadata service. Set experimental.record_deployment_history: true to deploy or destroy this bundle`) +This deployment's resources are recorded with the deployment history feature enabled. Set experimental.record_deployment_history: true to deploy or destroy this bundle`) type DeploymentState struct { Path string @@ -117,7 +126,7 @@ type DeploymentState struct { stateIDs map[string]string // operationBuffer records each state write with DMS. Nil unless the bundle records deployment - // history, in which case StartRecording installs it once the version exists. + // history, in which case InitializeOperationBuffer installs it once the version exists. operationBuffer *dms.OperationBuffer // dmsClient talks to the deployment metadata service. Open builds it from the workspace @@ -125,22 +134,18 @@ type DeploymentState struct { dmsClient *dms.Client // storageBackend is where this deployment's state lives, set by Open from the feature marker. - // It is the source of truth for gating DMS behavior after Open, in place of nil client checks. + // It is the source of truth for gating DMS behavior after Open. storageBackend StorageBackend - // deploymentID and versionID identify the recorded deployment and the version this run writes, - // stored by StartRecording. CompleteVersion reads them after Finalize's reset has cleared Path, - // Data and stateIDs, so - like operationBuffer and dmsClient - they must survive reset. - deploymentID string - versionID int64 - // versionCompleted makes CompleteVersion a no-op after the first call, so a deferred safety-net // completion after an explicit one does nothing. versionCompleted bool - // DeploymentID and LatestVersionID are the recorded deployment's id and its most recent version - // at Open time, from the service. CalculatePlan reads them (rather than the bundle config tree) - // to stamp the plan's lineage. Both empty for a first deploy, whose id the deploy phase creates. + // DeploymentID and LatestVersionID are the recorded deployment's id and its most recent version, + // from the service at Open (InitializeOperationBuffer updates DeploymentID to the id a first + // deploy creates, which Open cannot know). CalculatePlan reads them to stamp the plan's lineage; + // CompleteVersion reads them after Finalize's reset - so, like operationBuffer and dmsClient, they + // must survive reset. DeploymentID string LatestVersionID string } @@ -183,24 +188,27 @@ type WALEntry struct { Value *ResourceEntry `json:"v,omitempty"` // nil means delete } -// StartRecording opens the operation buffer that records each subsequent state write with DMS -// under deploymentID and versionID, so what the service holds mirrors the WAL. It uses the client -// Open built, so it is called only when the deployment records history, and once the version -// exists (after approval) - which is why it is not an Open option. -func (db *DeploymentState) StartRecording(ctx context.Context, deploymentID string, versionID int64) { +// InitializeOperationBuffer opens the operation buffer that records each subsequent state write with +// DMS under deploymentID and versionID, so what the service holds mirrors the WAL. It uses the client +// Open built, so it is called only when the deployment records history, and once the version exists +// (after approval) - which is why it is not an Open option. It also records the id a first deploy +// just created, which Open could not know, so CompleteVersion later has it. +func (db *DeploymentState) InitializeOperationBuffer(ctx context.Context, deploymentID string, versionID int64) { buf := dms.StartOperationBuffer(ctx, db.dmsClient, deploymentID, versionID) db.mu.Lock() defer db.mu.Unlock() db.operationBuffer = buf - db.deploymentID = deploymentID - db.versionID = versionID + db.DeploymentID = deploymentID } // RecordingError reports whether recording state writes to the service has failed, so the apply // stops touching resources once the service is no longer keeping up. Nil when the bundle does not // record deployment history or recording is healthy. func (db *DeploymentState) RecordingError() error { + if db.StorageBackend() != StorageBackendDeploymentMetadataService { + return nil + } db.mu.Lock() buf := db.operationBuffer db.mu.Unlock() @@ -223,9 +231,14 @@ func (db *DeploymentState) CompleteVersion(ctx context.Context, success bool) (b return false, nil } db.versionCompleted = true - deploymentID, versionID, client := db.deploymentID, db.versionID, db.dmsClient + deploymentID, latestVersionID, client := db.DeploymentID, db.LatestVersionID, db.dmsClient db.mu.Unlock() + versionID, err := dms.NextVersion(latestVersionID) + if err != nil { + return false, err + } + // A recording failure fails the version even when the caller counted the deploy a success: the // service does not then hold everything the WAL does. Finalize already drained and surfaced it; // this reads the drained buffer's error so a destroy whose uploads failed keeps its record. @@ -266,7 +279,7 @@ func (db *DeploymentState) RecordFailure(resourceKey, resourceID string, cause e } } - // A failure is recorded only while a version is open (after StartRecording), so the buffer is set. + // A failure is recorded only while a version is open (after InitializeOperationBuffer), so the buffer is set. db.mu.Lock() buf := db.operationBuffer db.mu.Unlock() @@ -463,7 +476,7 @@ type ( // When withDeploymentHistory is set it builds a DMS client from wsClient and reads resources from // the service instead, with dmsDeploymentID the id the service holds (empty before the first // recorded deploy); lineage and serial still come from the file, since that is what the write path -// increments. Open only reads through the client - StartRecording installs the write path once a +// increments. Open only reads through the client - InitializeOperationBuffer installs the write path once a // version exists. func (db *DeploymentState) Open(ctx context.Context, path string, withRecovery WithRecovery, withWrite WithWrite, withDeploymentHistory WithDeploymentHistory, dmsDeploymentID string) error { db.mu.Lock() @@ -566,7 +579,7 @@ func (db *DeploymentState) unlockedOpen(ctx context.Context, path string, withRe db.Data.Features[featureRecordDeploymentHistory] = struct{}{} recorded = true case recording && !recorded: - return errors.New(`this deployment already exists and is not recorded with the deployment metadata service, so it cannot be recorded without redeploying its resources + return errors.New(`this deployment already exists and is not recorded with the deployment history feature enabled, so it cannot be recorded without redeploying its resources To record this bundle's history, start it over as a new deployment: 1. remove experimental.record_deployment_history from your bundle configuration diff --git a/bundle/phases/dms.go b/bundle/phases/dms.go index 2db8bf57dca..fa9a0f65a14 100644 --- a/bundle/phases/dms.go +++ b/bundle/phases/dms.go @@ -153,7 +153,7 @@ func startVersion(ctx context.Context, b *bundle.Bundle, versionType dms.Version } log.Infof(ctx, "Created deployment version: deployment=%s version=%s", deploymentID, version.VersionId) - db.StateDB.StartRecording(ctx, deploymentID, versionID) + db.StateDB.InitializeOperationBuffer(ctx, deploymentID, versionID) return nil } diff --git a/libs/dms/client.go b/libs/dms/client.go index 611bdf9c11b..c4f592dec11 100644 --- a/libs/dms/client.go +++ b/libs/dms/client.go @@ -105,7 +105,7 @@ func (c *Client) UpdateOperation(ctx context.Context, deploymentID string, versi func deploymentIDFromName(name string) (string, error) { id, ok := strings.CutPrefix(name, DeploymentName("")) if !ok || id == "" { - return "", fmt.Errorf("unexpected deployment name %q from deployment metadata service", name) + return "", fmt.Errorf("unexpected deployment name %q from the deployment history service", name) } return id, nil } diff --git a/libs/dms/resources.go b/libs/dms/resources.go index 565571415f6..11321161c15 100644 --- a/libs/dms/resources.go +++ b/libs/dms/resources.go @@ -29,7 +29,7 @@ func (c *Client) ListResources(ctx context.Context, deploymentID string) ([]Reso for it.HasNext(ctx) { res, err := it.Next(ctx) if err != nil { - return nil, fmt.Errorf("listing resources from deployment metadata service: %w", err) + return nil, fmt.Errorf("listing resources from the deployment history service: %w", err) } out = append(out, Resource{ Key: statePrefix + res.ResourceKey, From 97ba52add1ba5f877847d5042e48e1070b2180cf Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 7 Sep 2026 00:21:45 +0200 Subject: [PATCH 23/46] hand edits --- bundle/direct/apply.go | 5 +- bundle/direct/bundle_apply.go | 1 - bundle/direct/bundle_plan.go | 35 +++---- bundle/direct/dstate/state.go | 171 +++++++++++++--------------------- bundle/phases/bind.go | 3 - bundle/phases/deploy.go | 42 ++++----- bundle/phases/destroy.go | 44 +++++---- 7 files changed, 128 insertions(+), 173 deletions(-) diff --git a/bundle/direct/apply.go b/bundle/direct/apply.go index af2a7277c77..f04b56e8b72 100644 --- a/bundle/direct/apply.go +++ b/bundle/direct/apply.go @@ -259,9 +259,8 @@ func (d *DeploymentUnit) Delete(ctx context.Context, db *dstate.DeploymentState, log.Warnf(ctx, "Ignoring permission error when deleting %s id=%s: %s", d.ResourceKey, oldID, err) } else if d.deleteConfirmedGone(ctx, oldID) { log.Warnf(ctx, "Treating %s id=%s as already deleted despite delete error: %s", d.ResourceKey, oldID, err) - } else { - // The resource is still there, so the history says why rather than leaving its - // operation pending. The state write below is what records a delete that worked. + } else if db.StorageBackend() == dstate.StorageBackendDeploymentMetadataService { + // When using the deployment metadata service, record the error. err = fmt.Errorf("deleting id=%s: %w", oldID, err) db.RecordFailure(d.ResourceKey, oldID, err) return err diff --git a/bundle/direct/bundle_apply.go b/bundle/direct/bundle_apply.go index 74bfa226cbf..db0a7ab4b3c 100644 --- a/bundle/direct/bundle_apply.go +++ b/bundle/direct/bundle_apply.go @@ -178,7 +178,6 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa return true }) - } func (b *DeploymentBundle) LookupReferencePostDeploy(ctx context.Context, path *structpath.PathNode) (any, error) { diff --git a/bundle/direct/bundle_plan.go b/bundle/direct/bundle_plan.go index 775103e6ad9..a2232b13088 100644 --- a/bundle/direct/bundle_plan.go +++ b/bundle/direct/bundle_plan.go @@ -61,10 +61,8 @@ func ValidatePlanAgainstState(stateDB *dstate.DeploymentState, plan *deployplan. return nil } -// InitForApply initializes the DeploymentBundle for applying a pre-computed plan: it parses the -// plan's entries (loaded as JSON when the plan came from a file) into the typed state cache the -// apply reads. A first deployment's id does not exist until the deploy creates it after approval; -// StampDeploymentIdForFirstVersion fills it in then. StateDB must already be open for write before calling this. +// InitForApply initializes the DeploymentBundle for applying a pre-computed plan. +// StateDB must already be open for write before calling this function. func (b *DeploymentBundle) InitForApply(ctx context.Context, client *databricks.WorkspaceClient, plan *deployplan.Plan) error { b.StateDB.AssertOpenedForWrite() @@ -120,8 +118,9 @@ func (b *DeploymentBundle) InitForApply(ctx context.Context, client *databricks. // StampDeploymentIdForFirstVersion fills deploymentID into the plan's jobs and pipelines that still lack one, in // both the state cache the apply reads and the plan JSON. It is called after approval creates the -// deployment, for a first deploy whose id did not exist at plan time; the version and any -// pre-existing id were stamped then. +// deployment, for a first deploy whose deployment_id did not exist at plan time. +// +// For subsequent deployments, the deployment_id is already stamped in the plan. func (b *DeploymentBundle) StampDeploymentIdForFirstVersion(deploymentID string) error { for resourceKey, entry := range b.Plan.Plan { if entry.NewState == nil || len(entry.NewState.Value) == 0 { @@ -169,23 +168,19 @@ func (b *DeploymentBundle) CalculatePlan(ctx context.Context, client *databricks } // The plan records where its state lives so deploy --plan can reject a plan whose target has - // since switched backends. StateDB is the source of truth, so a first deploy - which has no - // recorded deployment yet - still carries the backend. + // since switched backends or has an outdated version or a wrong deployment_id. if b.StateDB.StorageBackend() == dstate.StorageBackendDeploymentMetadataService { plan.StorageBackend = string(dstate.StorageBackendDeploymentMetadataService) - } - // Record the DMS deployment and version this plan targets, from StateDB (not the config tree). - // A saved plan carries them so deploy --plan can reject a plan the deployment has moved on from, - // and passes last_version_id as previous_version_id. Empty for a first deploy, which has no - // recorded deployment yet - the deploy phase stamps the created id. - if b.StateDB.DeploymentID != "" { - next, err := dms.NextVersion(b.StateDB.LatestVersionID) - if err != nil { - return nil, fmt.Errorf("computing next deployment version: %w", err) + // Not the first deployment, stamp the deployment_id and version_id into the plan. + if b.StateDB.DeploymentID != "" { + next, err := dms.NextVersion(b.StateDB.LatestVersionID) + if err != nil { + return nil, fmt.Errorf("computing next deployment version: %w", err) + } + plan.DeploymentId = b.StateDB.DeploymentID + plan.LastVersionId = b.StateDB.LatestVersionID + plan.NextVersionId = strconv.FormatInt(next, 10) } - plan.DeploymentId = b.StateDB.DeploymentID - plan.LastVersionId = b.StateDB.LatestVersionID - plan.NextVersionId = strconv.FormatInt(next, 10) } b.Plan = plan diff --git a/bundle/direct/dstate/state.go b/bundle/direct/dstate/state.go index ef67ccfc559..79d59bd39ee 100644 --- a/bundle/direct/dstate/state.go +++ b/bundle/direct/dstate/state.go @@ -64,7 +64,7 @@ const ( // // The marker is sticky: once a deployment is recorded, the service holds resources that a // CLI which is not recording must not touch. So turning recording off does not clear it, and -// deploying such a state without recording is refused (see RequiresDeploymentHistory). +// deploying such a state without recording is refused. const featureRecordDeploymentHistory = "record_deployment_history" // featuresDocURL explains the state-features mechanism in the error shown when this CLI refuses a @@ -88,10 +88,8 @@ var recognizedFeatures = map[string]struct{}{ featureRecordDeploymentHistory: {}, } -// checkStateFeatures refuses a state that depends on a feature this CLI does not recognize: keeping -// the state and the service in step needs every reader to understand the features in play, so an -// unknown one belongs to a newer CLI and this one refuses rather than clobber it. -func checkStateFeatures(features map[string]struct{}) error { +// assertNoUnsupportedFeatures refuses a state that depends on a feature this CLI does not recognize. +func assertNoUnsupportedFeatures(features map[string]struct{}) error { var unsupported []string for feature := range features { if _, ok := recognizedFeatures[feature]; !ok { @@ -224,6 +222,10 @@ func (db *DeploymentState) RecordingError() error { // one does nothing. Finalize has already drained the buffered operations. It reads only fields that // survive Finalize's reset and asserts nothing, so it is safe to call after the state is closed. func (db *DeploymentState) CompleteVersion(ctx context.Context, success bool) (bool, error) { + if db.StorageBackend() != StorageBackendDeploymentMetadataService { + return false, nil + } + db.mu.Lock() buf := db.operationBuffer if buf == nil || db.versionCompleted { @@ -301,6 +303,12 @@ func NewDatabase(lineage string, serial int) Database { // SaveState records the resource's state after an operation was applied to it. func (db *DeploymentState) SaveState(ctx context.Context, key, newID string, state any, dependsOn []deployplan.DependsOnEntry) error { db.AssertOpenedForWrite() + db.mu.Lock() + defer db.mu.Unlock() + + if db.Data.State == nil { + db.Data.State = make(map[string]ResourceEntry) + } jsonMessage, err := json.Marshal(state) if err != nil { @@ -312,33 +320,23 @@ func (db *DeploymentState) SaveState(ctx context.Context, key, newID string, sta DependsOn: dependsOn, } - db.mu.Lock() - if db.Data.State == nil { - db.Data.State = make(map[string]ResourceEntry) - } err = appendJSONLine(db.walFile, WALEntry{Key: key, Value: &entry}) if err == nil { db.stateIDs[key] = newID } - buf := db.operationBuffer - db.mu.Unlock() - if err != nil { return err } - if buf == nil { - return nil - } - // Recorded after the WAL write, so DMS never reports state the deploy failed to persist, - // and outside db.mu because recording waits when the service is behind - waiting under the - // lock would hold up every other resource's write. - recorded, err := json.Marshal(RecordedState{State: entry.State, DependsOn: dependsOn}) - if err != nil { - return err + // Record operation in DMS. + buf := db.operationBuffer + if db.StorageBackend() == StorageBackendDeploymentMetadataService && buf != nil { + recorded, err := json.Marshal(RecordedState{State: entry.State, DependsOn: dependsOn}) + if err != nil { + return err + } + buf.RecordOperation(ctx, key, false, newID, recorded) } - buf.RecordOperation(ctx, key, false, newID, recorded) - return nil } @@ -347,10 +345,10 @@ func (db *DeploymentState) SaveState(ctx context.Context, key, newID string, sta // deploy must not leave the resource described as finished. func (db *DeploymentState) DeleteState(ctx context.Context, key string, inProgress bool) error { db.AssertOpenedForWrite() - db.mu.Lock() + defer db.mu.Unlock() + if db.Data.State == nil { - db.mu.Unlock() return nil } // Read before the delete: DMS needs the id to say which resource went away. @@ -359,19 +357,14 @@ func (db *DeploymentState) DeleteState(ctx context.Context, key string, inProgre if err == nil { delete(db.stateIDs, key) } - buf := db.operationBuffer - db.mu.Unlock() - if err != nil { return err } - // State is nil: the resource no longer exists. Recorded outside the lock for the - // same reason as SaveState. - if buf != nil { + buf := db.operationBuffer + if db.StorageBackend() == StorageBackendDeploymentMetadataService && buf != nil { buf.RecordOperation(ctx, key, inProgress, deletedID, nil) } - return nil } @@ -410,24 +403,9 @@ func (db *DeploymentState) StateCLIVersion() string { return db.Data.CLIVersion } -// RequiresDeploymentHistory reports whether the state depends on the deployment metadata -// service recording it. Deploying such a state without recording would leave the service -// holding a deployment that no longer matches, so the caller refuses instead. -func (db *DeploymentState) RequiresDeploymentHistory() bool { - db.AssertOpenedForReadOrWrite() - db.mu.Lock() - defer db.mu.Unlock() - - _, ok := db.Data.Features[featureRecordDeploymentHistory] - return ok -} - // StorageBackend reports where this deployment's state lives - the source of truth for whether the // bundle records deployment history. Set by Open; valid only after the state is opened. func (db *DeploymentState) StorageBackend() StorageBackend { - db.AssertOpenedForReadOrWrite() - db.mu.Lock() - defer db.mu.Unlock() return db.storageBackend } @@ -512,7 +490,7 @@ func (db *DeploymentState) unlockedOpen(ctx context.Context, path string, withRe db.Path = path // The state file is the source of truth for whether this deployment records history: read it - // first, then reconcile the config against the marker it carries. + // first. data, err := os.ReadFile(db.Path) if err != nil { if errors.Is(err, fs.ErrNotExist) { @@ -530,68 +508,52 @@ func (db *DeploymentState) unlockedOpen(ctx context.Context, path string, withRe return fmt.Errorf("migrating state %s: %w", path, err) } - if err := checkStateFeatures(db.Data.Features); err != nil { + // TODO: We can remove and move this assertion to migrateState once we do the state + // version bump to 3 for this CLI. + if err := assertNoUnsupportedFeatures(db.Data.Features); err != nil { return err } - recording := bool(withDeploymentHistory) - db.stateIDs = make(map[string]string) for key, entry := range db.Data.State { db.stateIDs[key] = entry.ID } walPath := db.Path + walSuffix - _, statErr := os.Stat(walPath) - walExists := statErr == nil - if statErr != nil && !errors.Is(statErr, fs.ErrNotExist) { - return fmt.Errorf("failed to stat WAL file %s: %w", walPath, statErr) - } - - // A recording open discards a leftover WAL - the service owns the resources, so a local log from - // a crash or a declined deploy is irrelevant, even when recording is refused below. Without - // recording, replay the WAL as usual. - if walExists { - switch { - case recording: - if err := os.Remove(walPath); err != nil { - return fmt.Errorf("removing WAL file %s: %w", walPath, err) - } - case bool(withRecovery): + _, err = os.Stat(walPath) + switch { + case errors.Is(err, fs.ErrNotExist): + // no WAL, nothing to do + case err != nil: + return fmt.Errorf("failed to stat WAL file %s: %w", walPath, err) + default: // WAL exists + if withRecovery { if err := db.replayWAL(ctx); err != nil { return fmt.Errorf("reading state from %s: %w", path, err) } - default: + } else if withDeploymentHistory { + // Ignore the WAL file. That is not read for deployments that record history. + } else { return fmt.Errorf("unexpected WAL file found at %s", walPath) } } - // Reconcile the config against the state. A brand-new deployment is where recording begins, so - // the config bootstraps the marker there. On an existing deployment the state is authoritative: - // the config can neither start recording one that was not (its resources would be created a - // second time) nor stop recording one that is (the service still holds it). - _, recorded := db.Data.Features[featureRecordDeploymentHistory] - switch { - case recording && !recorded && len(db.Data.State) == 0: - if db.Data.Features == nil { - db.Data.Features = make(map[string]struct{}, 1) - } - db.Data.Features[featureRecordDeploymentHistory] = struct{}{} - recorded = true - case recording && !recorded: + _, stateHasRecordDeploymentHistory := db.Data.Features[featureRecordDeploymentHistory] + if !stateHasRecordDeploymentHistory && bool(withDeploymentHistory) { return errors.New(`this deployment already exists and is not recorded with the deployment history feature enabled, so it cannot be recorded without redeploying its resources -To record this bundle's history, start it over as a new deployment: - 1. remove experimental.record_deployment_history from your bundle configuration - 2. run "databricks bundle destroy" to delete the existing resources - 3. add experimental.record_deployment_history back and deploy again`) - case !recording && recorded: - return ErrUnsettingRecording + To record this bundle's history, start it over as a new deployment: + 1. remove experimental.record_deployment_history from your bundle configuration + 2. run "databricks bundle destroy" to delete the existing resources + 3. add experimental.record_deployment_history back and deploy again`) } db.storageBackend = StorageBackendWorkspaceFilesystem - if recorded { + if stateHasRecordDeploymentHistory { db.storageBackend = StorageBackendDeploymentMetadataService + if !bool(withDeploymentHistory) { + return ErrUnsettingRecording + } // The service is the source of truth for a recorded deployment; the file is a tombstone // carrying only the marker, and applyDMSState loads the resources the service holds. @@ -716,7 +678,6 @@ func (db *DeploymentState) mergeWalIntoState(ctx context.Context) (bool, error) if header.Serial > expectedSerial { return false, fmt.Errorf("WAL serial (%d) is ahead of expected (%d), state may be corrupted", header.Serial, expectedSerial) } - newSerial = header.Serial newCLIVersion = header.CLIVersion } else { @@ -796,10 +757,8 @@ func (db *DeploymentState) Finalize(ctx context.Context) (resourcestate.Exported err = db.replayWAL(ctx) } - // A recorded deployment uploads each write to the service on a background goroutine; block until - // they land or fail so the version completes only over state the service actually holds. A nil - // buffer means the deployment does not record history. - if db.operationBuffer != nil { + // Wait until all operations are recorded in the service. + if db.StorageBackend() == StorageBackendDeploymentMetadataService && db.operationBuffer != nil { if drainErr := db.operationBuffer.Drain(); drainErr != nil { err = errors.Join(err, drainErr) } @@ -925,7 +884,7 @@ func (db *DeploymentState) ExportState(ctx context.Context) resourcestate.Export // the WAL. A torn write would therefore leave a state file that Open rejects // next to an intact WAL it never reads. func (db *DeploymentState) unlockedSave() error { - data, err := json.MarshalIndent(db.dataToPersist(), "", " ") + data, err := json.MarshalIndent(db.dataForFile(), "", " ") if err != nil { return err } @@ -961,18 +920,22 @@ func (db *DeploymentState) unlockedSave() error { return nil } -// dataToPersist returns what the state file should hold: db.Data itself, unless the deployment -// records its history, in which case it is the header alone. The service holds those resources and -// is where they are read back from, so writing them here too would be a second, never-read copy. -// Returns a copy, leaving the in-memory state - what the rest of the deploy works from - untouched. -func (db *DeploymentState) dataToPersist() Database { - data := db.Data - if _, recorded := data.Features[featureRecordDeploymentHistory]; !recorded { - return data +// Data to persist in the remote resources.json file. +func (db *DeploymentState) dataForFile() Database { + if db.StorageBackend() == StorageBackendDeploymentMetadataService { + header := db.Data.Header + if header.Features == nil { + header.Features = make(map[string]struct{}) + } + header.Features[featureRecordDeploymentHistory] = struct{}{} + + return Database{ + Header: header, + State: map[string]ResourceEntry{}, + } } - data.State = map[string]ResourceEntry{} - return data + return db.Data } func appendJSONLine(file *os.File, obj any) error { diff --git a/bundle/phases/bind.go b/bundle/phases/bind.go index fc36122d1d8..86c9cc80da3 100644 --- a/bundle/phases/bind.go +++ b/bundle/phases/bind.go @@ -135,9 +135,6 @@ func Unbind(ctx context.Context, b *bundle.Bundle, bundleType, tfResourceType, r }() if engine.IsDirect() { - // A recorded deployment cannot be unbound: DeploymentBundle.Unbind opens the state without - // recording, so the state guard refuses it when the marker is set. State is the source of - // truth, so a config flag that disagrees with the marker cannot slip an unbind through. groupName, ok := terraform.TerraformToGroupName[tfResourceType] if !ok { groupName = tfResourceType diff --git a/bundle/phases/deploy.go b/bundle/phases/deploy.go index 88245e107e1..531270bfc76 100644 --- a/bundle/phases/deploy.go +++ b/bundle/phases/deploy.go @@ -205,9 +205,8 @@ func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHand // The version is created only after approval; CompleteVersion is deferred before // lock.Release and no-ops until then. defer func() { - // Skip entirely when not recording; CompleteVersion runs after Finalize has closed the state, - // so it is gated on the config rather than the (reset) StateDB. - if b.ConfiguresDeploymentHistory(ctx) { + // Close the version when backend type is deployment metadata service. + if b.DeploymentBundle.StateDB.StorageBackend() == dstate.StorageBackendDeploymentMetadataService { if _, err := b.DeploymentBundle.StateDB.CompleteVersion(ctx, !logdiag.HasError(ctx)); err != nil { logdiag.LogError(ctx, err) } @@ -299,8 +298,7 @@ func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHand } if planFromFile { - // Parse the loaded plan into the state cache the apply reads. A first deploy's deployment id - // is stamped later, after approval creates it (see below). + // Initialize DeploymentBundle for applying the loaded plan err := b.DeploymentBundle.InitForApply(ctx, b.WorkspaceClient(ctx), plan) if err != nil { logdiag.LogError(ctx, err) @@ -315,13 +313,13 @@ func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHand return } - haveApproval, approvalErr := approvalForDeploy(ctx, b, plan) + haveApproval, err := approvalForDeploy(ctx, b, plan) if !haveApproval { // No version was created, so the deferred CompleteVersion is a no-op and the version // number is left for the next deploy. Both the user declining and a console that // cannot prompt land here. - if approvalErr != nil { - logdiag.LogError(ctx, approvalErr) + if err != nil { + logdiag.LogError(ctx, err) return } cmdio.LogString(ctx, "Deployment cancelled!") @@ -345,22 +343,22 @@ func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHand return } } - } - // Create the version the plan was stamped with, staging an operation for every resource - // it touches. Doing it here rather than before the prompt means a declined deploy never - // claims a version number. - staged, err := stagedOperations(plan) - if err != nil { - logdiag.LogError(ctx, err) - return - } - if err := startVersion(ctx, b, dms.VersionTypeDeploy, staged); err != nil { - logdiag.LogError(ctx, err) - return + // Create the version the plan was stamped with, staging an operation for every resource + // it touches. Doing it here rather than before the prompt means a declined deploy never + // claims a version number. + staged, err := stagedOperations(plan) + if err != nil { + logdiag.LogError(ctx, err) + return + } + if err := startVersion(ctx, b, dms.VersionTypeDeploy, staged); err != nil { + logdiag.LogError(ctx, err) + return + } + deploymentID, versionID := deploymentAndNextVersion(b) + logDeploymentVersion(ctx, b, deploymentID, versionID) } - deploymentID, versionID := deploymentAndNextVersion(b) - logDeploymentVersion(ctx, b, deploymentID, versionID) deployCore(ctx, b, plan, stateEngine) diff --git a/bundle/phases/destroy.go b/bundle/phases/destroy.go index 309adb120da..69523ee7579 100644 --- a/bundle/phases/destroy.go +++ b/bundle/phases/destroy.go @@ -14,6 +14,7 @@ import ( "github.com/databricks/cli/bundle/deploy/lock" "github.com/databricks/cli/bundle/deploy/terraform" "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/cli/bundle/direct/dstate" "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dms" @@ -159,19 +160,21 @@ func destroyCore(ctx context.Context, b *bundle.Bundle, plan *deployplan.Plan, e return } - // Complete version before deleting remote files; the deployment node is under statePath. - completed, err := b.DeploymentBundle.StateDB.CompleteVersion(ctx, true) - if err != nil { - logdiag.LogError(ctx, err) - return - } - // A completed destroy's resources are gone, so its deployment record is deleted too. - if completed { - deploymentID, _ := deploymentAndNextVersion(b) - if err := b.DeploymentBundle.StateDB.DmsClient().DeleteDeployment(ctx, deploymentID); err != nil { - logdiag.LogError(ctx, fmt.Errorf("failed to delete deployment: %w", err)) + if engine.IsDirect() && b.DeploymentBundle.StateDB.StorageBackend() == dstate.StorageBackendDeploymentMetadataService { + // Complete version before deleting remote files; the deployment node is under statePath. + completed, err := b.DeploymentBundle.StateDB.CompleteVersion(ctx, true) + if err != nil { + logdiag.LogError(ctx, err) return } + // A completed destroy's resources are gone, so its deployment record is deleted too. + if completed { + deploymentID, _ := deploymentAndNextVersion(b) + if err := b.DeploymentBundle.StateDB.DmsClient().DeleteDeployment(ctx, deploymentID); err != nil { + logdiag.LogError(ctx, fmt.Errorf("failed to delete deployment: %w", err)) + return + } + } } bundle.ApplyContext(ctx, b, files.Delete()) @@ -216,14 +219,15 @@ func Destroy(ctx context.Context, b *bundle.Bundle, engine engine.EngineType) { // destroy records nothing. Deferred before lock.Release to hold the lock; a no-op once // destroyCore has completed the version. defer func() { - completed, err := b.DeploymentBundle.StateDB.CompleteVersion(ctx, !logdiag.HasError(ctx)) - if err != nil { - logdiag.LogError(ctx, err) - } else if completed { - // A completed destroy's resources are gone, so its deployment record is deleted too. - deploymentID, _ := deploymentAndNextVersion(b) - if err := b.DeploymentBundle.StateDB.DmsClient().DeleteDeployment(ctx, deploymentID); err != nil { - logdiag.LogError(ctx, fmt.Errorf("failed to delete deployment: %w", err)) + if engine.IsDirect() && b.DeploymentBundle.StateDB.StorageBackend() == dstate.StorageBackendDeploymentMetadataService { + completed, err := b.DeploymentBundle.StateDB.CompleteVersion(ctx, !logdiag.HasError(ctx)) + if err != nil { + logdiag.LogError(ctx, err) + } else if completed { + deploymentID, _ := deploymentAndNextVersion(b) + if err := b.DeploymentBundle.StateDB.DmsClient().DeleteDeployment(ctx, deploymentID); err != nil { + logdiag.LogError(ctx, fmt.Errorf("failed to delete deployment: %w", err)) + } } } bundle.ApplyContext(ctx, b, lock.Release(lock.GoalDestroy)) @@ -288,7 +292,7 @@ func Destroy(ctx context.Context, b *bundle.Bundle, engine engine.EngineType) { // destroy's recording: it opens the operation buffer that destroyCore drains and completes, // so the drain and the deployment delete follow only when a version was started here. // Destroy never creates or updates the deployment - it is about to be deleted. - if engine.IsDirect() && b.DeploymentBundle.StateDB.RequiresDeploymentHistory() { + if engine.IsDirect() && b.DeploymentBundle.StateDB.StorageBackend() == dstate.StorageBackendDeploymentMetadataService { staged, err := stagedOperations(plan) if err != nil { logdiag.LogError(ctx, err) From dad29930a31f7d6453c4ea16f7e1cb823fc4ab38 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 7 Sep 2026 00:37:00 +0200 Subject: [PATCH 24/46] bundle/dms: fix hand-edit regressions in unlockedOpen and declined-deploy test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three bugs introduced by the previous hand-edit commit: 1. unlockedOpen: the 3-way switch reconciling config vs state was replaced with a simple `if !stateHasRecordDeploymentHistory && withDeploymentHistory`, which errored even on brand-new deployments (no state file → no feature flag → error). Restored the original logic including the bootstrap case for `recording && !recorded && len(db.Data.State) == 0`. 2. unlockedOpen: recording opens must delete a leftover WAL (the service owns the resources; a local log from a crash is irrelevant). The hand edit changed this to "ignore the WAL", breaking version-never-created and existing-state. Restored os.Remove on recording opens. 3. Error message indentation: the multi-line error had tab-indented continuation lines instead of no leading whitespace, mismatching the existing-state golden. Also: declined-deploy/script - add --get to the second print_requests call so the GET requests made during the declined deploy (GetDeployment + ListResources) appear in the output. Move out.requests.txt cleanup into the EXIT trap so the catalog delete's request file is removed and no out.requests.txt is left behind. Co-authored-by: Isaac --- .../dms/declined-deploy/out.requests.txt | 7 --- .../bundle/dms/declined-deploy/output.txt | 24 +++++++++- acceptance/bundle/dms/declined-deploy/script | 7 ++- bundle/direct/dstate/state.go | 45 +++++++++++++------ 4 files changed, 56 insertions(+), 27 deletions(-) delete mode 100644 acceptance/bundle/dms/declined-deploy/out.requests.txt diff --git a/acceptance/bundle/dms/declined-deploy/out.requests.txt b/acceptance/bundle/dms/declined-deploy/out.requests.txt deleted file mode 100644 index eb169f26a0c..00000000000 --- a/acceptance/bundle/dms/declined-deploy/out.requests.txt +++ /dev/null @@ -1,7 +0,0 @@ -{ - "method": "DELETE", - "path": "/api/2.1/unity-catalog/catalogs/dms_other_[UNIQUE_NAME]", - "q": { - "force": "true" - } -} diff --git a/acceptance/bundle/dms/declined-deploy/output.txt b/acceptance/bundle/dms/declined-deploy/output.txt index d0ce19a2389..890e58e72b7 100644 --- a/acceptance/bundle/dms/declined-deploy/output.txt +++ b/acceptance/bundle/dms/declined-deploy/output.txt @@ -6,7 +6,7 @@ Created schemas.foo Files: 5 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged ->>> print_requests.py --dms //api/2.0/bundle +>>> print_requests.py --dms --get //api/2.0/bundle { "method": "POST", "path": "/api/2.0/bundle/deployments", @@ -129,7 +129,27 @@ To proceed, use --auto-approve after reviewing the plan above. Files: 4 uploaded, 0 deleted === Nothing was recorded for the declined deploy - no version, so none to abort ->>> print_requests.py --dms //api/2.0/bundle +>>> print_requests.py --dms --get //api/2.0/bundle +{ + "method": "GET", + "path": "/api/2.0/bundle/deployments/[NUMID]" +} +{ + "method": "GET", + "path": "/api/2.0/bundle/deployments/[NUMID]/resources" +} +{ + "method": "GET", + "path": "/api/2.0/bundle/deployments/[NUMID]/resources" +} +{ + "method": "GET", + "path": "/api/2.0/bundle/deployments/[NUMID]" +} +{ + "method": "GET", + "path": "/api/2.0/bundle/deployments/[NUMID]/resources" +} === The recorded resources are unchanged by the decline: the schema is still at version 1 >>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[NUMID] -o json diff --git a/acceptance/bundle/dms/declined-deploy/script b/acceptance/bundle/dms/declined-deploy/script index cc69f672e63..d49fcfbfb53 100644 --- a/acceptance/bundle/dms/declined-deploy/script +++ b/acceptance/bundle/dms/declined-deploy/script @@ -2,7 +2,7 @@ envsubst < databricks.yml.tmpl > databricks.yml title "Deploy a schema, so the deployment and its first version exist" trace $CLI bundle deploy -trace print_requests.py --dms //api/2.0/bundle +trace print_requests.py --dms --get //api/2.0/bundle title "The plan is a no-op now and records DEPLOYMENT_METADATA_SERVICE; the service holds the schema at version 1" trace $CLI bundle plan -o json | nostamp.py @@ -13,14 +13,14 @@ title "A destructive change without --auto-approve is declined: this console can # Unity Catalog has no catalog called "other", so make one to move the schema into. other_catalog="dms_other_${UNIQUE_NAME}" $CLI catalogs create "$other_catalog" &> LOG.catalog -trap '$CLI catalogs delete "$other_catalog" --force >> LOG.catalog 2>&1' EXIT +trap '$CLI catalogs delete "$other_catalog" --force >> LOG.catalog 2>&1; rm -f out.requests.txt' EXIT trace update_file.py databricks.yml "catalog_name: main" "catalog_name: ${other_catalog}" trace musterr $CLI bundle deploy title "Nothing was recorded for the declined deploy - no version, so none to abort" # The version number it would have used is left for the next deploy to take, so the # history has no entry that reads like a deploy which failed or did nothing. -trace print_requests.py --dms //api/2.0/bundle +trace print_requests.py --dms --get //api/2.0/bundle title "The recorded resources are unchanged by the decline: the schema is still at version 1" deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-declined-deploy-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') @@ -29,4 +29,3 @@ trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${d trace $CLI bundle destroy --auto-approve title "After the destroy the deployment is gone, so the summary reports no history rather than a dangling id" trace $CLI bundle summary -o json | jq '.bundle.deployment | has("history")' -rm -f out.requests.txt diff --git a/bundle/direct/dstate/state.go b/bundle/direct/dstate/state.go index 79d59bd39ee..f43503c7a05 100644 --- a/bundle/direct/dstate/state.go +++ b/bundle/direct/dstate/state.go @@ -519,6 +519,8 @@ func (db *DeploymentState) unlockedOpen(ctx context.Context, path string, withRe db.stateIDs[key] = entry.ID } + recording := bool(withDeploymentHistory) + walPath := db.Path + walSuffix _, err = os.Stat(walPath) switch { @@ -527,33 +529,48 @@ func (db *DeploymentState) unlockedOpen(ctx context.Context, path string, withRe case err != nil: return fmt.Errorf("failed to stat WAL file %s: %w", walPath, err) default: // WAL exists - if withRecovery { + switch { + case recording: + // A recording open discards a leftover WAL - the service owns the resources, so a local + // log from a crash or a declined deploy is irrelevant, even when recording is refused below. + if err := os.Remove(walPath); err != nil { + return fmt.Errorf("removing WAL file %s: %w", walPath, err) + } + case bool(withRecovery): if err := db.replayWAL(ctx); err != nil { return fmt.Errorf("reading state from %s: %w", path, err) } - } else if withDeploymentHistory { - // Ignore the WAL file. That is not read for deployments that record history. - } else { + default: return fmt.Errorf("unexpected WAL file found at %s", walPath) } } - _, stateHasRecordDeploymentHistory := db.Data.Features[featureRecordDeploymentHistory] - if !stateHasRecordDeploymentHistory && bool(withDeploymentHistory) { + // Reconcile the config against the state. A brand-new deployment is where recording begins, so + // the config bootstraps the marker there. On an existing deployment the state is authoritative: + // the config can neither start recording one that was not (its resources would be created a + // second time) nor stop recording one that is (the service still holds it). + _, recorded := db.Data.Features[featureRecordDeploymentHistory] + switch { + case recording && !recorded && len(db.Data.State) == 0: + if db.Data.Features == nil { + db.Data.Features = make(map[string]struct{}, 1) + } + db.Data.Features[featureRecordDeploymentHistory] = struct{}{} + recorded = true + case recording && !recorded: return errors.New(`this deployment already exists and is not recorded with the deployment history feature enabled, so it cannot be recorded without redeploying its resources - To record this bundle's history, start it over as a new deployment: - 1. remove experimental.record_deployment_history from your bundle configuration - 2. run "databricks bundle destroy" to delete the existing resources - 3. add experimental.record_deployment_history back and deploy again`) +To record this bundle's history, start it over as a new deployment: + 1. remove experimental.record_deployment_history from your bundle configuration + 2. run "databricks bundle destroy" to delete the existing resources + 3. add experimental.record_deployment_history back and deploy again`) + case !recording && recorded: + return ErrUnsettingRecording } db.storageBackend = StorageBackendWorkspaceFilesystem - if stateHasRecordDeploymentHistory { + if recorded { db.storageBackend = StorageBackendDeploymentMetadataService - if !bool(withDeploymentHistory) { - return ErrUnsettingRecording - } // The service is the source of truth for a recorded deployment; the file is a tombstone // carrying only the marker, and applyDMSState loads the resources the service holds. From bb5abcdae431ad641df1c9b4b32805fe0dace9bc Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 7 Sep 2026 01:02:23 +0200 Subject: [PATCH 25/46] acceptance/dms: serialize deployment_id and version_id in plan output MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Remove | nostamp.py from all bundle/dms test scripts. nostamp was stripping deployment_id, next_version_id, and last_version_id from plan output (its purpose is to make non-DMS tests agnostic to recording mode). DMS-specific tests should assert these fields directly — they are the feature under test. Regenerate all goldens. Plans now show deployment_id/next_version_id/ last_version_id after the first deploy, confirming the fields are serialized. Co-authored-by: Isaac --- .../bundle/dms/declined-deploy/output.txt | 3 + acceptance/bundle/dms/declined-deploy/script | 2 +- acceptance/bundle/dms/depends-on/output.txt | 25 +++- acceptance/bundle/dms/depends-on/script | 2 +- .../dms/deployment-metadata-change/output.txt | 6 + .../dms/deployment-metadata-change/script | 4 +- .../bundle/dms/emptied-resource/output.txt | 3 + acceptance/bundle/dms/emptied-resource/script | 2 +- acceptance/bundle/dms/existing-state/script | 2 +- .../bundle/dms/failed-delete/output.txt | 14 ++- acceptance/bundle/dms/failed-delete/script | 2 +- .../bundle/dms/failed-recreate/output.txt | 3 + acceptance/bundle/dms/failed-recreate/script | 2 +- .../dms/failed-update-permissions/output.txt | 14 ++- .../dms/failed-update-permissions/script | 2 +- .../bundle/dms/failed-update/output.txt | 3 + acceptance/bundle/dms/failed-update/script | 2 +- .../bundle/dms/multiple-resources/output.txt | 116 ++++++++++++++++-- .../bundle/dms/multiple-resources/script | 4 +- acceptance/bundle/dms/no-drift/output.txt | 54 +++++++- acceptance/bundle/dms/no-drift/script | 4 +- acceptance/bundle/dms/no-resources/output.txt | 6 + acceptance/bundle/dms/no-resources/script | 4 +- .../operation-upload-fails-once/output.txt | 3 + .../dms/operation-upload-fails-once/script | 2 +- acceptance/bundle/dms/provenance/output.txt | 14 ++- acceptance/bundle/dms/provenance/script | 2 +- .../bundle/dms/record-failure/output.txt | 7 +- acceptance/bundle/dms/record-failure/script | 2 +- acceptance/bundle/dms/record/output.txt | 14 ++- acceptance/bundle/dms/record/script | 2 +- acceptance/bundle/dms/stale-plan/output.txt | 3 +- acceptance/bundle/dms/stale-plan/script | 2 +- .../bundle/dms/successful-recreate/output.txt | 6 + .../bundle/dms/successful-recreate/script | 4 +- .../dms/version-never-created/output.txt | 6 +- .../bundle/dms/version-never-created/script | 2 +- 37 files changed, 301 insertions(+), 47 deletions(-) diff --git a/acceptance/bundle/dms/declined-deploy/output.txt b/acceptance/bundle/dms/declined-deploy/output.txt index 890e58e72b7..552567f1cab 100644 --- a/acceptance/bundle/dms/declined-deploy/output.txt +++ b/acceptance/bundle/dms/declined-deploy/output.txt @@ -66,6 +66,9 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, + "deployment_id": "[NUMID]", + "next_version_id": "2", + "last_version_id": "1", "storage_backend": "DEPLOYMENT_METADATA_SERVICE", "plan": { "resources.schemas.foo": { diff --git a/acceptance/bundle/dms/declined-deploy/script b/acceptance/bundle/dms/declined-deploy/script index d49fcfbfb53..d31b65e6129 100644 --- a/acceptance/bundle/dms/declined-deploy/script +++ b/acceptance/bundle/dms/declined-deploy/script @@ -5,7 +5,7 @@ trace $CLI bundle deploy trace print_requests.py --dms --get //api/2.0/bundle title "The plan is a no-op now and records DEPLOYMENT_METADATA_SERVICE; the service holds the schema at version 1" -trace $CLI bundle plan -o json | nostamp.py +trace $CLI bundle plan -o json trace print_state.py title "A destructive change without --auto-approve is declined: this console cannot prompt" diff --git a/acceptance/bundle/dms/depends-on/output.txt b/acceptance/bundle/dms/depends-on/output.txt index 4fb3a8caab5..83d71a1e2ad 100644 --- a/acceptance/bundle/dms/depends-on/output.txt +++ b/acceptance/bundle/dms/depends-on/output.txt @@ -44,6 +44,9 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, + "deployment_id": "[NUMID]", + "next_version_id": "2", + "last_version_id": "1", "storage_backend": "DEPLOYMENT_METADATA_SERVICE", "plan": { "resources.jobs.child": { @@ -58,8 +61,10 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged "created_time": [UNIX_TIME_MILLIS][0], "creator_user_name": "[USERNAME]", "deployment": { + "deployment_id": "[NUMID]", "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-depends-on-[UNIQUE_NAME]/default/state/metadata.json" + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-depends-on-[UNIQUE_NAME]/default/state/metadata.json", + "version_id": "1" }, "description": "depends on [NUMID]", "edit_mode": "UI_LOCKED", @@ -76,6 +81,13 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged "webhook_notifications": {} }, "changes": { + "deployment.version_id": { + "action": "skip", + "reason": "auto", + "old": "1", + "new": "2", + "remote": "1" + }, "email_notifications": { "action": "skip", "reason": "empty", @@ -99,8 +111,10 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged "created_time": [UNIX_TIME_MILLIS][1], "creator_user_name": "[USERNAME]", "deployment": { + "deployment_id": "[NUMID]", "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-depends-on-[UNIQUE_NAME]/default/state/metadata.json" + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-depends-on-[UNIQUE_NAME]/default/state/metadata.json", + "version_id": "1" }, "edit_mode": "UI_LOCKED", "email_notifications": {}, @@ -116,6 +130,13 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged "webhook_notifications": {} }, "changes": { + "deployment.version_id": { + "action": "skip", + "reason": "auto", + "old": "1", + "new": "2", + "remote": "1" + }, "email_notifications": { "action": "skip", "reason": "empty", diff --git a/acceptance/bundle/dms/depends-on/script b/acceptance/bundle/dms/depends-on/script index 022f8e3dd11..5b721e24c6b 100644 --- a/acceptance/bundle/dms/depends-on/script +++ b/acceptance/bundle/dms/depends-on/script @@ -5,7 +5,7 @@ trace $CLI bundle deploy trace print_requests.py --dms //versions/1/operations title "The plan is a no-op now and records DEPLOYMENT_METADATA_SERVICE as its backend" -trace $CLI bundle plan -o json | nostamp.py +trace $CLI bundle plan -o json title "The service lists both jobs, each recorded at version 1" deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-depends-on-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') diff --git a/acceptance/bundle/dms/deployment-metadata-change/output.txt b/acceptance/bundle/dms/deployment-metadata-change/output.txt index 3422dde62d7..99c1c51f301 100644 --- a/acceptance/bundle/dms/deployment-metadata-change/output.txt +++ b/acceptance/bundle/dms/deployment-metadata-change/output.txt @@ -66,6 +66,9 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, + "deployment_id": "[NUMID]", + "next_version_id": "2", + "last_version_id": "1", "storage_backend": "DEPLOYMENT_METADATA_SERVICE", "plan": { "resources.schemas.foo": { @@ -167,6 +170,9 @@ A common practice is to use a username or principal name in this path, i.e. use "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, + "deployment_id": "[NUMID]", + "next_version_id": "3", + "last_version_id": "2", "storage_backend": "DEPLOYMENT_METADATA_SERVICE", "plan": { "resources.schemas.foo": { diff --git a/acceptance/bundle/dms/deployment-metadata-change/script b/acceptance/bundle/dms/deployment-metadata-change/script index 45de9533cab..5d650aaa341 100644 --- a/acceptance/bundle/dms/deployment-metadata-change/script +++ b/acceptance/bundle/dms/deployment-metadata-change/script @@ -5,7 +5,7 @@ trace $CLI bundle deploy trace print_requests.py --dms //api/2.0/bundle title "The plan records DEPLOYMENT_METADATA_SERVICE; the service holds the schema at version 1" -trace $CLI bundle plan -o json | nostamp.py +trace $CLI bundle plan -o json deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-metadata-pinned-${UNIQUE_NAME}/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' @@ -15,7 +15,7 @@ trace $CLI bundle deploy -t prod trace print_requests.py --dms //api/2.0/bundle title "The metadata changed but the schema did not: still one deployment, recorded at version 1" -trace $CLI bundle plan -t prod -o json | nostamp.py +trace $CLI bundle plan -t prod -o json trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' title "Deploy again without touching anything: the record already says this, so no deployment update" diff --git a/acceptance/bundle/dms/emptied-resource/output.txt b/acceptance/bundle/dms/emptied-resource/output.txt index 297b8e441db..0b00307c1e3 100644 --- a/acceptance/bundle/dms/emptied-resource/output.txt +++ b/acceptance/bundle/dms/emptied-resource/output.txt @@ -137,6 +137,9 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 2, + "deployment_id": "[NUMID]", + "next_version_id": "3", + "last_version_id": "2", "storage_backend": "DEPLOYMENT_METADATA_SERVICE", "plan": { "resources.schemas.foo": { diff --git a/acceptance/bundle/dms/emptied-resource/script b/acceptance/bundle/dms/emptied-resource/script index 6e5af06cc31..8d49ae09cdd 100644 --- a/acceptance/bundle/dms/emptied-resource/script +++ b/acceptance/bundle/dms/emptied-resource/script @@ -17,7 +17,7 @@ title "Plan again: reading state back from the service works and reports no work trace $CLI bundle plan | contains.py "Plan: 0 to add, 0 to change, 0 to delete" "!unexpected end of JSON input" title "The plan is a no-op and records DEPLOYMENT_METADATA_SERVICE as its backend" -trace $CLI bundle plan -o json | nostamp.py +trace $CLI bundle plan -o json title "The grants node emptied out, so the service no longer lists it - only the schema remains" deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-emptied-resource-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') diff --git a/acceptance/bundle/dms/existing-state/script b/acceptance/bundle/dms/existing-state/script index 5ab20285e56..1b1a9401965 100644 --- a/acceptance/bundle/dms/existing-state/script +++ b/acceptance/bundle/dms/existing-state/script @@ -2,7 +2,7 @@ envsubst < databricks.yml.tmpl > databricks.yml title "Deploy without recording: ordinary direct-engine state tracking one job, unknown to DMS" trace $CLI bundle deploy -trace $CLI bundle plan -o json | nostamp.py +trace $CLI bundle plan -o json trace print_requests.py --dms //api/2.0/bundle --oneline trace jq '{state_version, features}' .databricks/bundle/default/resources.json diff --git a/acceptance/bundle/dms/failed-delete/output.txt b/acceptance/bundle/dms/failed-delete/output.txt index 17d98e0a4a3..4585105b85a 100644 --- a/acceptance/bundle/dms/failed-delete/output.txt +++ b/acceptance/bundle/dms/failed-delete/output.txt @@ -66,6 +66,9 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, + "deployment_id": "[NUMID]", + "next_version_id": "2", + "last_version_id": "1", "storage_backend": "DEPLOYMENT_METADATA_SERVICE", "plan": { "resources.jobs.stuck": { @@ -74,8 +77,10 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "created_time": [UNIX_TIME_MILLIS], "creator_user_name": "[USERNAME]", "deployment": { + "deployment_id": "[NUMID]", "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-failed-delete-[UNIQUE_NAME]/default/state/metadata.json" + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-failed-delete-[UNIQUE_NAME]/default/state/metadata.json", + "version_id": "1" }, "edit_mode": "UI_LOCKED", "email_notifications": {}, @@ -91,6 +96,13 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "webhook_notifications": {} }, "changes": { + "deployment.version_id": { + "action": "skip", + "reason": "auto", + "old": "1", + "new": "2", + "remote": "1" + }, "email_notifications": { "action": "skip", "reason": "empty", diff --git a/acceptance/bundle/dms/failed-delete/script b/acceptance/bundle/dms/failed-delete/script index ed544b27d7f..9f3881f78bd 100644 --- a/acceptance/bundle/dms/failed-delete/script +++ b/acceptance/bundle/dms/failed-delete/script @@ -5,7 +5,7 @@ trace $CLI bundle deploy trace print_requests.py --dms //api/2.0/bundle title "The plan is a no-op now and records DEPLOYMENT_METADATA_SERVICE as its backend" -trace $CLI bundle plan -o json | nostamp.py +trace $CLI bundle plan -o json title "After deploy, the service lists the resource at version 1" deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-failed-delete-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') diff --git a/acceptance/bundle/dms/failed-recreate/output.txt b/acceptance/bundle/dms/failed-recreate/output.txt index a85298826b2..5f8bfa55121 100644 --- a/acceptance/bundle/dms/failed-recreate/output.txt +++ b/acceptance/bundle/dms/failed-recreate/output.txt @@ -66,6 +66,9 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, + "deployment_id": "[NUMID]", + "next_version_id": "2", + "last_version_id": "1", "storage_backend": "DEPLOYMENT_METADATA_SERVICE", "plan": { "resources.schemas.foo": { diff --git a/acceptance/bundle/dms/failed-recreate/script b/acceptance/bundle/dms/failed-recreate/script index 6c0b82a1a69..77b72b94669 100644 --- a/acceptance/bundle/dms/failed-recreate/script +++ b/acceptance/bundle/dms/failed-recreate/script @@ -5,7 +5,7 @@ trace $CLI bundle deploy trace print_requests.py --dms //api/2.0/bundle title "The plan is a no-op and records DEPLOYMENT_METADATA_SERVICE as its backend" -trace $CLI bundle plan -o json | nostamp.py +trace $CLI bundle plan -o json title "After deploy, the service lists the resource at version 1" deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-failed-recreate-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') diff --git a/acceptance/bundle/dms/failed-update-permissions/output.txt b/acceptance/bundle/dms/failed-update-permissions/output.txt index ac97f5ce901..345f048464a 100644 --- a/acceptance/bundle/dms/failed-update-permissions/output.txt +++ b/acceptance/bundle/dms/failed-update-permissions/output.txt @@ -14,6 +14,9 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, + "deployment_id": "[NUMID]", + "next_version_id": "2", + "last_version_id": "1", "storage_backend": "DEPLOYMENT_METADATA_SERVICE", "plan": { "resources.jobs.foo": { @@ -22,8 +25,10 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged "created_time": [UNIX_TIME_MILLIS], "creator_user_name": "[USERNAME]", "deployment": { + "deployment_id": "[NUMID]", "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-failed-update-permissions-[UNIQUE_NAME]/default/state/metadata.json" + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-failed-update-permissions-[UNIQUE_NAME]/default/state/metadata.json", + "version_id": "1" }, "edit_mode": "UI_LOCKED", "email_notifications": {}, @@ -39,6 +44,13 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged "webhook_notifications": {} }, "changes": { + "deployment.version_id": { + "action": "skip", + "reason": "auto", + "old": "1", + "new": "2", + "remote": "1" + }, "email_notifications": { "action": "skip", "reason": "empty", diff --git a/acceptance/bundle/dms/failed-update-permissions/script b/acceptance/bundle/dms/failed-update-permissions/script index 6883d8f3775..8b251b08b7c 100644 --- a/acceptance/bundle/dms/failed-update-permissions/script +++ b/acceptance/bundle/dms/failed-update-permissions/script @@ -5,7 +5,7 @@ trace $CLI bundle deploy rm -f out.requests.txt title "The plan records DEPLOYMENT_METADATA_SERVICE backend and the job" -trace $CLI bundle plan -o json | nostamp.py +trace $CLI bundle plan -o json title "The service lists the job at version 1" deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-failed-update-permissions-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') diff --git a/acceptance/bundle/dms/failed-update/output.txt b/acceptance/bundle/dms/failed-update/output.txt index 2cb6c855d5f..54358b62e95 100644 --- a/acceptance/bundle/dms/failed-update/output.txt +++ b/acceptance/bundle/dms/failed-update/output.txt @@ -66,6 +66,9 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, + "deployment_id": "[NUMID]", + "next_version_id": "2", + "last_version_id": "1", "storage_backend": "DEPLOYMENT_METADATA_SERVICE", "plan": { "resources.schemas.foo": { diff --git a/acceptance/bundle/dms/failed-update/script b/acceptance/bundle/dms/failed-update/script index 7d223af223b..6e1acfe26e2 100644 --- a/acceptance/bundle/dms/failed-update/script +++ b/acceptance/bundle/dms/failed-update/script @@ -5,7 +5,7 @@ trace $CLI bundle deploy trace print_requests.py --dms //api/2.0/bundle title "The plan is a no-op and records DEPLOYMENT_METADATA_SERVICE as its backend" -trace $CLI bundle plan -o json | nostamp.py +trace $CLI bundle plan -o json title "After deploy, the service lists the resource at version 1" deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-failed-update-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') diff --git a/acceptance/bundle/dms/multiple-resources/output.txt b/acceptance/bundle/dms/multiple-resources/output.txt index 4a737c37b6e..79370c3d51d 100644 --- a/acceptance/bundle/dms/multiple-resources/output.txt +++ b/acceptance/bundle/dms/multiple-resources/output.txt @@ -24,6 +24,9 @@ Resources: 5 created, 0 changed, 0 deleted, 0 unchanged "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, + "deployment_id": "[NUMID]", + "next_version_id": "2", + "last_version_id": "1", "storage_backend": "DEPLOYMENT_METADATA_SERVICE", "plan": { "resources.jobs.five": { @@ -32,8 +35,10 @@ Resources: 5 created, 0 changed, 0 deleted, 0 unchanged "created_time": [UNIX_TIME_MILLIS][0], "creator_user_name": "[USERNAME]", "deployment": { + "deployment_id": "[NUMID]", "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/state/metadata.json" + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/state/metadata.json", + "version_id": "1" }, "edit_mode": "UI_LOCKED", "email_notifications": {}, @@ -49,6 +54,13 @@ Resources: 5 created, 0 changed, 0 deleted, 0 unchanged "webhook_notifications": {} }, "changes": { + "deployment.version_id": { + "action": "skip", + "reason": "auto", + "old": "1", + "new": "2", + "remote": "1" + }, "email_notifications": { "action": "skip", "reason": "empty", @@ -72,8 +84,10 @@ Resources: 5 created, 0 changed, 0 deleted, 0 unchanged "created_time": [UNIX_TIME_MILLIS][1], "creator_user_name": "[USERNAME]", "deployment": { + "deployment_id": "[NUMID]", "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/state/metadata.json" + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/state/metadata.json", + "version_id": "1" }, "edit_mode": "UI_LOCKED", "email_notifications": {}, @@ -89,6 +103,13 @@ Resources: 5 created, 0 changed, 0 deleted, 0 unchanged "webhook_notifications": {} }, "changes": { + "deployment.version_id": { + "action": "skip", + "reason": "auto", + "old": "1", + "new": "2", + "remote": "1" + }, "email_notifications": { "action": "skip", "reason": "empty", @@ -112,8 +133,10 @@ Resources: 5 created, 0 changed, 0 deleted, 0 unchanged "created_time": [UNIX_TIME_MILLIS][2], "creator_user_name": "[USERNAME]", "deployment": { + "deployment_id": "[NUMID]", "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/state/metadata.json" + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/state/metadata.json", + "version_id": "1" }, "edit_mode": "UI_LOCKED", "email_notifications": {}, @@ -129,6 +152,13 @@ Resources: 5 created, 0 changed, 0 deleted, 0 unchanged "webhook_notifications": {} }, "changes": { + "deployment.version_id": { + "action": "skip", + "reason": "auto", + "old": "1", + "new": "2", + "remote": "1" + }, "email_notifications": { "action": "skip", "reason": "empty", @@ -152,8 +182,10 @@ Resources: 5 created, 0 changed, 0 deleted, 0 unchanged "created_time": [UNIX_TIME_MILLIS][3], "creator_user_name": "[USERNAME]", "deployment": { + "deployment_id": "[NUMID]", "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/state/metadata.json" + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/state/metadata.json", + "version_id": "1" }, "edit_mode": "UI_LOCKED", "email_notifications": {}, @@ -169,6 +201,13 @@ Resources: 5 created, 0 changed, 0 deleted, 0 unchanged "webhook_notifications": {} }, "changes": { + "deployment.version_id": { + "action": "skip", + "reason": "auto", + "old": "1", + "new": "2", + "remote": "1" + }, "email_notifications": { "action": "skip", "reason": "empty", @@ -192,8 +231,10 @@ Resources: 5 created, 0 changed, 0 deleted, 0 unchanged "created_time": [UNIX_TIME_MILLIS][4], "creator_user_name": "[USERNAME]", "deployment": { + "deployment_id": "[NUMID]", "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/state/metadata.json" + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/state/metadata.json", + "version_id": "1" }, "edit_mode": "UI_LOCKED", "email_notifications": {}, @@ -209,6 +250,13 @@ Resources: 5 created, 0 changed, 0 deleted, 0 unchanged "webhook_notifications": {} }, "changes": { + "deployment.version_id": { + "action": "skip", + "reason": "auto", + "old": "1", + "new": "2", + "remote": "1" + }, "email_notifications": { "action": "skip", "reason": "empty", @@ -380,6 +428,9 @@ Resources: 0 created, 0 changed, 0 deleted, 5 unchanged "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, + "deployment_id": "[NUMID]", + "next_version_id": "3", + "last_version_id": "2", "storage_backend": "DEPLOYMENT_METADATA_SERVICE", "plan": { "resources.jobs.five": { @@ -388,8 +439,10 @@ Resources: 0 created, 0 changed, 0 deleted, 5 unchanged "created_time": [UNIX_TIME_MILLIS][0], "creator_user_name": "[USERNAME]", "deployment": { + "deployment_id": "[NUMID]", "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/state/metadata.json" + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/state/metadata.json", + "version_id": "1" }, "edit_mode": "UI_LOCKED", "email_notifications": {}, @@ -405,6 +458,13 @@ Resources: 0 created, 0 changed, 0 deleted, 5 unchanged "webhook_notifications": {} }, "changes": { + "deployment.version_id": { + "action": "skip", + "reason": "auto", + "old": "1", + "new": "3", + "remote": "1" + }, "email_notifications": { "action": "skip", "reason": "empty", @@ -428,8 +488,10 @@ Resources: 0 created, 0 changed, 0 deleted, 5 unchanged "created_time": [UNIX_TIME_MILLIS][1], "creator_user_name": "[USERNAME]", "deployment": { + "deployment_id": "[NUMID]", "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/state/metadata.json" + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/state/metadata.json", + "version_id": "1" }, "edit_mode": "UI_LOCKED", "email_notifications": {}, @@ -445,6 +507,13 @@ Resources: 0 created, 0 changed, 0 deleted, 5 unchanged "webhook_notifications": {} }, "changes": { + "deployment.version_id": { + "action": "skip", + "reason": "auto", + "old": "1", + "new": "3", + "remote": "1" + }, "email_notifications": { "action": "skip", "reason": "empty", @@ -468,8 +537,10 @@ Resources: 0 created, 0 changed, 0 deleted, 5 unchanged "created_time": [UNIX_TIME_MILLIS][2], "creator_user_name": "[USERNAME]", "deployment": { + "deployment_id": "[NUMID]", "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/state/metadata.json" + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/state/metadata.json", + "version_id": "1" }, "edit_mode": "UI_LOCKED", "email_notifications": {}, @@ -485,6 +556,13 @@ Resources: 0 created, 0 changed, 0 deleted, 5 unchanged "webhook_notifications": {} }, "changes": { + "deployment.version_id": { + "action": "skip", + "reason": "auto", + "old": "1", + "new": "3", + "remote": "1" + }, "email_notifications": { "action": "skip", "reason": "empty", @@ -508,8 +586,10 @@ Resources: 0 created, 0 changed, 0 deleted, 5 unchanged "created_time": [UNIX_TIME_MILLIS][3], "creator_user_name": "[USERNAME]", "deployment": { + "deployment_id": "[NUMID]", "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/state/metadata.json" + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/state/metadata.json", + "version_id": "1" }, "edit_mode": "UI_LOCKED", "email_notifications": {}, @@ -525,6 +605,13 @@ Resources: 0 created, 0 changed, 0 deleted, 5 unchanged "webhook_notifications": {} }, "changes": { + "deployment.version_id": { + "action": "skip", + "reason": "auto", + "old": "1", + "new": "3", + "remote": "1" + }, "email_notifications": { "action": "skip", "reason": "empty", @@ -548,8 +635,10 @@ Resources: 0 created, 0 changed, 0 deleted, 5 unchanged "created_time": [UNIX_TIME_MILLIS][4], "creator_user_name": "[USERNAME]", "deployment": { + "deployment_id": "[NUMID]", "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/state/metadata.json" + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/state/metadata.json", + "version_id": "1" }, "edit_mode": "UI_LOCKED", "email_notifications": {}, @@ -565,6 +654,13 @@ Resources: 0 created, 0 changed, 0 deleted, 5 unchanged "webhook_notifications": {} }, "changes": { + "deployment.version_id": { + "action": "skip", + "reason": "auto", + "old": "1", + "new": "3", + "remote": "1" + }, "email_notifications": { "action": "skip", "reason": "empty", diff --git a/acceptance/bundle/dms/multiple-resources/script b/acceptance/bundle/dms/multiple-resources/script index 78ab3028390..6315788d22a 100644 --- a/acceptance/bundle/dms/multiple-resources/script +++ b/acceptance/bundle/dms/multiple-resources/script @@ -5,7 +5,7 @@ trace $CLI bundle deploy trace print_requests.py --dms //versions/1/operations --sort --del-body state --oneline title "The plan records DEPLOYMENT_METADATA_SERVICE as its backend" -trace $CLI bundle plan -o json | nostamp.py +trace $CLI bundle plan -o json title "The service lists all five resources at version 1" deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-multiple-resources-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') @@ -19,7 +19,7 @@ trace $CLI bundle deploy trace print_requests.py --dms //api/2.0/bundle --sort --oneline title "The plan records DEPLOYMENT_METADATA_SERVICE as its backend" -trace $CLI bundle plan -o json | nostamp.py +trace $CLI bundle plan -o json title "The service still lists all five resources, now at version 2" deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-multiple-resources-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') diff --git a/acceptance/bundle/dms/no-drift/output.txt b/acceptance/bundle/dms/no-drift/output.txt index 87d254cf2e2..064ba5acfeb 100644 --- a/acceptance/bundle/dms/no-drift/output.txt +++ b/acceptance/bundle/dms/no-drift/output.txt @@ -34,6 +34,9 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, + "deployment_id": "[NUMID]", + "next_version_id": "2", + "last_version_id": "1", "storage_backend": "DEPLOYMENT_METADATA_SERVICE", "plan": { "resources.jobs.foo": { @@ -42,8 +45,10 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged "created_time": [UNIX_TIME_MILLIS][0], "creator_user_name": "[USERNAME]", "deployment": { + "deployment_id": "[NUMID]", "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-no-drift-[UNIQUE_NAME]/default/state/metadata.json" + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-no-drift-[UNIQUE_NAME]/default/state/metadata.json", + "version_id": "1" }, "edit_mode": "UI_LOCKED", "email_notifications": {}, @@ -59,6 +64,13 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged "webhook_notifications": {} }, "changes": { + "deployment.version_id": { + "action": "skip", + "reason": "auto", + "old": "1", + "new": "2", + "remote": "1" + }, "email_notifications": { "action": "skip", "reason": "empty", @@ -83,8 +95,10 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged "channel": "CURRENT", "creator_user_name": "[USERNAME]", "deployment": { + "deployment_id": "[NUMID]", "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-no-drift-[UNIQUE_NAME]/default/state/metadata.json" + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-no-drift-[UNIQUE_NAME]/default/state/metadata.json", + "version_id": "1" }, "edition": "ADVANCED", "effective_publishing_mode": "DEFAULT_PUBLISHING_MODE", @@ -103,6 +117,15 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged "schema": "default", "serverless": true, "state": "IDLE" + }, + "changes": { + "deployment.version_id": { + "action": "skip", + "reason": "auto", + "old": "1", + "new": "2", + "remote": "1" + } } } } @@ -277,6 +300,9 @@ Resources: 0 created, 1 changed, 0 deleted, 1 unchanged "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 2, + "deployment_id": "[NUMID]", + "next_version_id": "3", + "last_version_id": "2", "storage_backend": "DEPLOYMENT_METADATA_SERVICE", "plan": { "resources.jobs.foo": { @@ -285,8 +311,10 @@ Resources: 0 created, 1 changed, 0 deleted, 1 unchanged "created_time": [UNIX_TIME_MILLIS][0], "creator_user_name": "[USERNAME]", "deployment": { + "deployment_id": "[NUMID]", "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-no-drift-[UNIQUE_NAME]/default/state/metadata.json" + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-no-drift-[UNIQUE_NAME]/default/state/metadata.json", + "version_id": "1" }, "edit_mode": "UI_LOCKED", "email_notifications": {}, @@ -302,6 +330,13 @@ Resources: 0 created, 1 changed, 0 deleted, 1 unchanged "webhook_notifications": {} }, "changes": { + "deployment.version_id": { + "action": "skip", + "reason": "auto", + "old": "1", + "new": "3", + "remote": "1" + }, "email_notifications": { "action": "skip", "reason": "empty", @@ -326,8 +361,10 @@ Resources: 0 created, 1 changed, 0 deleted, 1 unchanged "channel": "CURRENT", "creator_user_name": "[USERNAME]", "deployment": { + "deployment_id": "[NUMID]", "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-no-drift-[UNIQUE_NAME]/default/state/metadata.json" + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-no-drift-[UNIQUE_NAME]/default/state/metadata.json", + "version_id": "2" }, "development": true, "edition": "ADVANCED", @@ -347,6 +384,15 @@ Resources: 0 created, 1 changed, 0 deleted, 1 unchanged "schema": "default", "serverless": true, "state": "IDLE" + }, + "changes": { + "deployment.version_id": { + "action": "skip", + "reason": "auto", + "old": "2", + "new": "3", + "remote": "2" + } } } } diff --git a/acceptance/bundle/dms/no-drift/script b/acceptance/bundle/dms/no-drift/script index 55bff445646..64455e8f1b4 100644 --- a/acceptance/bundle/dms/no-drift/script +++ b/acceptance/bundle/dms/no-drift/script @@ -8,7 +8,7 @@ trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${d trace $CLI bundle summary -o json | jq .bundle.deployment.history title "The plan is a no-op now and records DEPLOYMENT_METADATA_SERVICE as its backend" -trace $CLI bundle plan -o json | nostamp.py +trace $CLI bundle plan -o json title "The raw recorded state the service holds for this deployment" trace print_state.py @@ -27,7 +27,7 @@ trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${d trace $CLI bundle summary -o json | jq .bundle.deployment.history title "The plan shows version 2 for the updated pipeline and DEPLOYMENT_METADATA_SERVICE as backend" -trace $CLI bundle plan -o json | nostamp.py +trace $CLI bundle plan -o json title "The raw recorded state the service holds after the update" trace print_state.py diff --git a/acceptance/bundle/dms/no-resources/output.txt b/acceptance/bundle/dms/no-resources/output.txt index f75ba46e2fa..c5b67b15d30 100644 --- a/acceptance/bundle/dms/no-resources/output.txt +++ b/acceptance/bundle/dms/no-resources/output.txt @@ -49,6 +49,9 @@ Resources: 0 created, 0 changed, 0 deleted, 0 unchanged { "plan_version": 3, "cli_version": "[CLI_VERSION]", + "deployment_id": "[NUMID]", + "next_version_id": "2", + "last_version_id": "1", "storage_backend": "DEPLOYMENT_METADATA_SERVICE", "plan": {} } @@ -112,6 +115,9 @@ Resources: 0 created, 0 changed, 0 deleted, 0 unchanged { "plan_version": 3, "cli_version": "[CLI_VERSION]", + "deployment_id": "[NUMID]", + "next_version_id": "3", + "last_version_id": "2", "storage_backend": "DEPLOYMENT_METADATA_SERVICE", "plan": {} } diff --git a/acceptance/bundle/dms/no-resources/script b/acceptance/bundle/dms/no-resources/script index 001d1e835f5..168b360744f 100644 --- a/acceptance/bundle/dms/no-resources/script +++ b/acceptance/bundle/dms/no-resources/script @@ -6,7 +6,7 @@ trace print_requests.py --dms //api/2.0/bundle --get trace MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-no-resources-${UNIQUE_NAME}/default/state/resources.deployment.json" | jq '{object_type,path}' title "The plan records DEPLOYMENT_METADATA_SERVICE as its backend" -trace $CLI bundle plan -o json | nostamp.py +trace $CLI bundle plan -o json title "The service lists no resources for the deployment" deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-no-resources-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') @@ -20,7 +20,7 @@ trace $CLI bundle deploy trace print_requests.py --dms //api/2.0/bundle --get title "The plan records DEPLOYMENT_METADATA_SERVICE as its backend" -trace $CLI bundle plan -o json | nostamp.py +trace $CLI bundle plan -o json title "The service still lists no resources" deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-no-resources-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') diff --git a/acceptance/bundle/dms/operation-upload-fails-once/output.txt b/acceptance/bundle/dms/operation-upload-fails-once/output.txt index 8e74957520c..481042276d2 100644 --- a/acceptance/bundle/dms/operation-upload-fails-once/output.txt +++ b/acceptance/bundle/dms/operation-upload-fails-once/output.txt @@ -13,6 +13,9 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, + "deployment_id": "[NUMID]", + "next_version_id": "2", + "last_version_id": "1", "storage_backend": "DEPLOYMENT_METADATA_SERVICE", "plan": { "resources.schemas.foo": { diff --git a/acceptance/bundle/dms/operation-upload-fails-once/script b/acceptance/bundle/dms/operation-upload-fails-once/script index d69268d8f87..ce340a1c09f 100644 --- a/acceptance/bundle/dms/operation-upload-fails-once/script +++ b/acceptance/bundle/dms/operation-upload-fails-once/script @@ -5,7 +5,7 @@ trace $CLI bundle deploy rm -f out.requests.txt title "The plan records DEPLOYMENT_METADATA_SERVICE backend and the schema" -trace $CLI bundle plan -o json | nostamp.py +trace $CLI bundle plan -o json title "The service lists the schema at version 1" deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-upload-fails-once-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') diff --git a/acceptance/bundle/dms/provenance/output.txt b/acceptance/bundle/dms/provenance/output.txt index df026816182..a87d280b403 100644 --- a/acceptance/bundle/dms/provenance/output.txt +++ b/acceptance/bundle/dms/provenance/output.txt @@ -58,6 +58,9 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, + "deployment_id": "[NUMID]", + "next_version_id": "2", + "last_version_id": "1", "storage_backend": "DEPLOYMENT_METADATA_SERVICE", "plan": { "resources.jobs.foo": { @@ -66,8 +69,10 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "created_time": [UNIX_TIME_MILLIS], "creator_user_name": "[USERNAME]", "deployment": { + "deployment_id": "[NUMID]", "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-provenance-[UNIQUE_NAME]/dev/state/metadata.json" + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-provenance-[UNIQUE_NAME]/dev/state/metadata.json", + "version_id": "1" }, "edit_mode": "UI_LOCKED", "email_notifications": {}, @@ -86,6 +91,13 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "webhook_notifications": {} }, "changes": { + "deployment.version_id": { + "action": "skip", + "reason": "auto", + "old": "1", + "new": "2", + "remote": "1" + }, "email_notifications": { "action": "skip", "reason": "empty", diff --git a/acceptance/bundle/dms/provenance/script b/acceptance/bundle/dms/provenance/script index 1c6661c30d7..f1b8163d219 100644 --- a/acceptance/bundle/dms/provenance/script +++ b/acceptance/bundle/dms/provenance/script @@ -9,7 +9,7 @@ add_repl.py "$(git rev-parse HEAD)" COMMIT trace print_requests.py --dms //versions title "The plan is a no-op and records DEPLOYMENT_METADATA_SERVICE as its backend" -trace $CLI bundle plan -o json | nostamp.py +trace $CLI bundle plan -o json title "The service denormalizes them onto the deployment, so a reader gets the provenance of the latest version" deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-provenance-${UNIQUE_NAME}/dev/state/resources.deployment.json" -o json | python3 -c 'import sys,json; print(json.load(sys.stdin)["object_id"])') diff --git a/acceptance/bundle/dms/record-failure/output.txt b/acceptance/bundle/dms/record-failure/output.txt index a08dd0df054..9dfbd0af041 100644 --- a/acceptance/bundle/dms/record-failure/output.txt +++ b/acceptance/bundle/dms/record-failure/output.txt @@ -15,6 +15,9 @@ Files: 6 uploaded, 0 deleted { "plan_version": 3, "cli_version": "[CLI_VERSION]", + "deployment_id": "[NUMID]", + "next_version_id": "2", + "last_version_id": "1", "storage_backend": "DEPLOYMENT_METADATA_SERVICE", "plan": { "resources.jobs.doomed": { @@ -22,8 +25,10 @@ Files: 6 uploaded, 0 deleted "new_state": { "value": { "deployment": { + "deployment_id": "[NUMID]", "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-record-failure-[UNIQUE_NAME]/default/state/metadata.json" + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-record-failure-[UNIQUE_NAME]/default/state/metadata.json", + "version_id": "2" }, "edit_mode": "UI_LOCKED", "format": "MULTI_TASK", diff --git a/acceptance/bundle/dms/record-failure/script b/acceptance/bundle/dms/record-failure/script index 8fc543ff20d..0df6cfac842 100644 --- a/acceptance/bundle/dms/record-failure/script +++ b/acceptance/bundle/dms/record-failure/script @@ -2,7 +2,7 @@ envsubst < databricks.yml.tmpl > databricks.yml title "A resource that fails to apply is recorded as a failed operation carrying the error, so the deployment history says why rather than omitting the resource" trace musterr $CLI bundle deploy -trace $CLI bundle plan -o json | nostamp.py +trace $CLI bundle plan -o json trace print_requests.py --dms //api/2.0/bundle title "The failed resource is not listed at all: state is what projects a resource, and a failed create records none, so a later deploy plans to create it rather than treating it as already deployed" diff --git a/acceptance/bundle/dms/record/output.txt b/acceptance/bundle/dms/record/output.txt index 7f62c55528d..9a1459183f2 100644 --- a/acceptance/bundle/dms/record/output.txt +++ b/acceptance/bundle/dms/record/output.txt @@ -90,6 +90,9 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 3, + "deployment_id": "[NUMID]", + "next_version_id": "2", + "last_version_id": "1", "storage_backend": "DEPLOYMENT_METADATA_SERVICE", "plan": { "resources.jobs.foo": { @@ -98,8 +101,10 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "created_time": [UNIX_TIME_MILLIS], "creator_user_name": "[USERNAME]", "deployment": { + "deployment_id": "[NUMID]", "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-record-[UNIQUE_NAME]/default/state/metadata.json" + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-record-[UNIQUE_NAME]/default/state/metadata.json", + "version_id": "1" }, "edit_mode": "UI_LOCKED", "email_notifications": {}, @@ -115,6 +120,13 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "webhook_notifications": {} }, "changes": { + "deployment.version_id": { + "action": "skip", + "reason": "auto", + "old": "1", + "new": "2", + "remote": "1" + }, "email_notifications": { "action": "skip", "reason": "empty", diff --git a/acceptance/bundle/dms/record/script b/acceptance/bundle/dms/record/script index 5c565b792d1..aeb95228b7e 100644 --- a/acceptance/bundle/dms/record/script +++ b/acceptance/bundle/dms/record/script @@ -16,7 +16,7 @@ trace MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_U trace print_requests.py --get --dms //api/2.0/bundle title "The plan is a no-op now and records DEPLOYMENT_METADATA_SERVICE as its backend" -trace $CLI bundle plan -o json | nostamp.py +trace $CLI bundle plan -o json title "The service lists the job, recorded at version 1" deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-record-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') diff --git a/acceptance/bundle/dms/stale-plan/output.txt b/acceptance/bundle/dms/stale-plan/output.txt index 3b89e18875a..33385bec6af 100644 --- a/acceptance/bundle/dms/stale-plan/output.txt +++ b/acceptance/bundle/dms/stale-plan/output.txt @@ -12,7 +12,8 @@ "value": { "deployment": { "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-stale-plan-[UNIQUE_NAME]/default/state/metadata.json" + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-stale-plan-[UNIQUE_NAME]/default/state/metadata.json", + "version_id": "1" }, "edit_mode": "UI_LOCKED", "format": "MULTI_TASK", diff --git a/acceptance/bundle/dms/stale-plan/script b/acceptance/bundle/dms/stale-plan/script index f50dae91193..d5d68ca0c66 100644 --- a/acceptance/bundle/dms/stale-plan/script +++ b/acceptance/bundle/dms/stale-plan/script @@ -2,7 +2,7 @@ envsubst < databricks.yml.tmpl > databricks.yml title "Save a plan, then deploy it: records version 1" $CLI bundle plan -o json > tmp.plan.json -trace $CLI bundle plan -o json | nostamp.py +trace $CLI bundle plan -o json # READPLAN deploys from the saved raw plan; without it the plan is recomputed. Untraced so the # variant-dependent command line stays out of the golden (the deploy output is identical either way). $CLI bundle deploy $(readplanarg tmp.plan.json) diff --git a/acceptance/bundle/dms/successful-recreate/output.txt b/acceptance/bundle/dms/successful-recreate/output.txt index 1fd34e383e4..1412ee7e0ad 100644 --- a/acceptance/bundle/dms/successful-recreate/output.txt +++ b/acceptance/bundle/dms/successful-recreate/output.txt @@ -66,6 +66,9 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 1, + "deployment_id": "[NUMID]", + "next_version_id": "2", + "last_version_id": "1", "storage_backend": "DEPLOYMENT_METADATA_SERVICE", "plan": { "resources.schemas.foo": { @@ -199,6 +202,9 @@ Resources: 1 created, 0 changed, 1 deleted, 0 unchanged "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", "serial": 2, + "deployment_id": "[NUMID]", + "next_version_id": "3", + "last_version_id": "2", "storage_backend": "DEPLOYMENT_METADATA_SERVICE", "plan": { "resources.schemas.foo": { diff --git a/acceptance/bundle/dms/successful-recreate/script b/acceptance/bundle/dms/successful-recreate/script index 6a8e0fe8914..965dcafdce9 100644 --- a/acceptance/bundle/dms/successful-recreate/script +++ b/acceptance/bundle/dms/successful-recreate/script @@ -5,7 +5,7 @@ trace $CLI bundle deploy trace print_requests.py --dms //api/2.0/bundle title "The plan is a no-op now and records DEPLOYMENT_METADATA_SERVICE as its backend" -trace $CLI bundle plan -o json | nostamp.py +trace $CLI bundle plan -o json title "The service lists the schema, recorded at version 1" deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-successful-recreate-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') @@ -26,7 +26,7 @@ trace $CLI bundle deploy --auto-approve trace print_requests.py --dms //api/2.0/bundle title "The plan now shows the recreate action" -trace $CLI bundle plan -o json | nostamp.py +trace $CLI bundle plan -o json title "The service lists the schema at version 2 with recreate action" trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' diff --git a/acceptance/bundle/dms/version-never-created/output.txt b/acceptance/bundle/dms/version-never-created/output.txt index db21f2a000a..11df2223dba 100644 --- a/acceptance/bundle/dms/version-never-created/output.txt +++ b/acceptance/bundle/dms/version-never-created/output.txt @@ -15,6 +15,8 @@ Files: 5 uploaded, 0 deleted { "plan_version": 3, "cli_version": "[CLI_VERSION]", + "deployment_id": "[NUMID]", + "next_version_id": "1", "storage_backend": "DEPLOYMENT_METADATA_SERVICE", "plan": { "resources.jobs.foo": { @@ -22,8 +24,10 @@ Files: 5 uploaded, 0 deleted "new_state": { "value": { "deployment": { + "deployment_id": "[NUMID]", "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-version-never-created/default/state/metadata.json" + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-version-never-created/default/state/metadata.json", + "version_id": "1" }, "edit_mode": "UI_LOCKED", "format": "MULTI_TASK", diff --git a/acceptance/bundle/dms/version-never-created/script b/acceptance/bundle/dms/version-never-created/script index aca7b030713..d801153b1c7 100644 --- a/acceptance/bundle/dms/version-never-created/script +++ b/acceptance/bundle/dms/version-never-created/script @@ -1,6 +1,6 @@ title "The first version fails, so no deployment record exists - only the node naming its ID" trace musterr $CLI bundle deploy -trace $CLI bundle plan -o json | nostamp.py +trace $CLI bundle plan -o json trace MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-version-never-created/default/state/resources.deployment.json" | jq '{object_type}' trace print_requests.py --dms //api/2.0/bundle --get --oneline From aadf9726048bdbae71db147bea6050528570b5ef Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 7 Sep 2026 01:11:12 +0200 Subject: [PATCH 26/46] acceptance/dms: assert tombstone state via print_state.py --no-dms MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add --no-dms flag to print_state.py: prints resources.json as-is without fetching state from the deployment metadata service. The existing behaviour (enriching the output with DMS-held resources) exists for non-DMS tests so their goldens stay identical whether recording is on or off; DMS tests need the opposite — to assert that the local file IS the tombstone (empty state, just the header + record_deployment_history feature flag). Update all bundle/dms test scripts to use print_state.py --no-dms. Regenerate goldens: state blocks that previously showed full resource state now show "state": {}, confirming the service holds the resources and the local file carries only the tombstone marker. Co-authored-by: Isaac --- acceptance/bin/print_state.py | 8 +- .../bundle/dms/declined-deploy/output.txt | 89 ++++++-- acceptance/bundle/dms/declined-deploy/script | 4 +- acceptance/bundle/dms/depends-on/output.txt | 48 +---- acceptance/bundle/dms/depends-on/script | 2 +- .../bundle/dms/emptied-resource/output.txt | 12 +- acceptance/bundle/dms/emptied-resource/script | 2 +- .../bundle/dms/failed-delete/output.txt | 46 +---- acceptance/bundle/dms/failed-delete/script | 7 +- .../bundle/dms/failed-recreate/output.txt | 15 +- acceptance/bundle/dms/failed-recreate/script | 4 +- .../dms/failed-update-permissions/output.txt | 90 +-------- .../dms/failed-update-permissions/script | 4 +- .../bundle/dms/failed-update/output.txt | 26 +-- acceptance/bundle/dms/failed-update/script | 4 +- .../bundle/dms/multiple-resources/output.txt | 190 +----------------- .../bundle/dms/multiple-resources/script | 4 +- acceptance/bundle/dms/no-drift/output.txt | 95 +-------- acceptance/bundle/dms/no-drift/script | 4 +- acceptance/bundle/dms/no-resources/output.txt | 4 +- acceptance/bundle/dms/no-resources/script | 4 +- .../operation-upload-fails-once/output.txt | 26 +-- .../dms/operation-upload-fails-once/script | 4 +- acceptance/bundle/dms/provenance/output.txt | 26 +-- acceptance/bundle/dms/provenance/script | 2 +- .../bundle/dms/record-failure/output.txt | 2 +- acceptance/bundle/dms/record-failure/script | 2 +- acceptance/bundle/dms/record/output.txt | 23 +-- acceptance/bundle/dms/record/script | 2 +- acceptance/bundle/dms/stale-plan/output.txt | 23 +-- acceptance/bundle/dms/stale-plan/script | 2 +- .../bundle/dms/successful-recreate/output.txt | 26 +-- .../bundle/dms/successful-recreate/script | 4 +- 33 files changed, 146 insertions(+), 658 deletions(-) diff --git a/acceptance/bin/print_state.py b/acceptance/bin/print_state.py index f63189c02b3..351c3083396 100755 --- a/acceptance/bin/print_state.py +++ b/acceptance/bin/print_state.py @@ -76,12 +76,18 @@ def main(): parser = argparse.ArgumentParser() parser.add_argument("-t", "--target") parser.add_argument("--backup", action="store_true") + parser.add_argument( + "--no-dms", + action="store_true", + help="Print resources.json as-is without fetching state from DMS. " + "Use in bundle/dms tests to assert the tombstone content of the file.", + ) args = parser.parse_args() for filename in get_state_files(args.target, args.backup): if not os.path.exists(filename): continue - if filename.endswith("resources.json") and records_deployment_history(): + if filename.endswith("resources.json") and records_deployment_history() and not args.no_dms: print_recorded_state(filename, args.target) else: print_file(filename) diff --git a/acceptance/bundle/dms/declined-deploy/output.txt b/acceptance/bundle/dms/declined-deploy/output.txt index 552567f1cab..a4d25045708 100644 --- a/acceptance/bundle/dms/declined-deploy/output.txt +++ b/acceptance/bundle/dms/declined-deploy/output.txt @@ -97,7 +97,17 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged } } ->>> print_state.py +>>> print_requests.py --dms --get //api/2.0/bundle +{ + "method": "GET", + "path": "/api/2.0/bundle/deployments/[NUMID]" +} +{ + "method": "GET", + "path": "/api/2.0/bundle/deployments/[NUMID]/resources" +} + +>>> print_state.py --no-dms { "state_version": 2, "cli_version": "[CLI_VERSION]", @@ -106,15 +116,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "features": { "record_deployment_history": {} }, - "state": { - "resources.schemas.foo": { - "__id__": "main.dms_declined_deploy_schema_[UNIQUE_NAME]", - "state": { - "catalog_name": "main", - "name": "dms_declined_deploy_schema_[UNIQUE_NAME]" - } - } - } + "state": {} } === A destructive change without --auto-approve is declined: this console cannot prompt @@ -141,18 +143,6 @@ Files: 4 uploaded, 0 deleted "method": "GET", "path": "/api/2.0/bundle/deployments/[NUMID]/resources" } -{ - "method": "GET", - "path": "/api/2.0/bundle/deployments/[NUMID]/resources" -} -{ - "method": "GET", - "path": "/api/2.0/bundle/deployments/[NUMID]" -} -{ - "method": "GET", - "path": "/api/2.0/bundle/deployments/[NUMID]/resources" -} === The recorded resources are unchanged by the decline: the schema is still at version 1 >>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[NUMID] -o json @@ -178,3 +168,58 @@ Destroy: 1 deleted === After the destroy the deployment is gone, so the summary reports no history rather than a dangling id >>> [CLI] bundle summary -o json false + +>>> print_requests.py --dms --get //api/2.0/bundle +{ + "method": "GET", + "path": "/api/2.0/bundle/deployments/[NUMID]/resources" +} +{ + "method": "GET", + "path": "/api/2.0/bundle/deployments/[NUMID]" +} +{ + "method": "GET", + "path": "/api/2.0/bundle/deployments/[NUMID]/resources" +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions", + "q": { + "version_id": "2" + }, + "body": { + "cli_version": "[CLI_VERSION]", + "version_type": "VERSION_TYPE_DESTROY", + "previous_version_id": "1", + "operations": [ + { + "resource_key": "schemas.foo", + "action_type": "OPERATION_ACTION_TYPE_DELETE" + } + ] + } +} +{ + "method": "PATCH", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations/schemas.foo", + "q": { + "update_mask": "state,error_message,status" + }, + "body": { + "error_message": "", + "sequence_id": "0", + "status": "OPERATION_STATUS_SUCCEEDED" + } +} +{ + "method": "POST", + "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/complete", + "body": { + "completion_reason": "VERSION_COMPLETE_SUCCESS" + } +} +{ + "method": "DELETE", + "path": "/api/2.0/bundle/deployments/[NUMID]" +} diff --git a/acceptance/bundle/dms/declined-deploy/script b/acceptance/bundle/dms/declined-deploy/script index d31b65e6129..c51686ba386 100644 --- a/acceptance/bundle/dms/declined-deploy/script +++ b/acceptance/bundle/dms/declined-deploy/script @@ -6,7 +6,8 @@ trace print_requests.py --dms --get //api/2.0/bundle title "The plan is a no-op now and records DEPLOYMENT_METADATA_SERVICE; the service holds the schema at version 1" trace $CLI bundle plan -o json -trace print_state.py +trace print_requests.py --dms --get //api/2.0/bundle +trace print_state.py --no-dms title "A destructive change without --auto-approve is declined: this console cannot prompt" # Changing the catalog recreates the schema, which needs approval. @@ -29,3 +30,4 @@ trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${d trace $CLI bundle destroy --auto-approve title "After the destroy the deployment is gone, so the summary reports no history rather than a dangling id" trace $CLI bundle summary -o json | jq '.bundle.deployment | has("history")' +trace print_requests.py --dms --get //api/2.0/bundle diff --git a/acceptance/bundle/dms/depends-on/output.txt b/acceptance/bundle/dms/depends-on/output.txt index 83d71a1e2ad..063c1da9067 100644 --- a/acceptance/bundle/dms/depends-on/output.txt +++ b/acceptance/bundle/dms/depends-on/output.txt @@ -173,7 +173,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged ] === The raw recorded state the service holds for this deployment ->>> print_state.py +>>> print_state.py --no-dms { "state_version": 2, "cli_version": "[CLI_VERSION]", @@ -182,51 +182,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged "features": { "record_deployment_history": {} }, - "state": { - "resources.jobs.child": { - "__id__": "[NUMID]", - "state": { - "deployment": { - "deployment_id": "[NUMID]", - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-depends-on-[UNIQUE_NAME]/default/state/metadata.json", - "version_id": "1" - }, - "description": "depends on [NUMID]", - "edit_mode": "UI_LOCKED", - "format": "MULTI_TASK", - "max_concurrent_runs": 1, - "name": "child", - "queue": { - "enabled": true - } - }, - "depends_on": [ - { - "node": "resources.jobs.parent", - "label": "${resources.jobs.parent.id}" - } - ] - }, - "resources.jobs.parent": { - "__id__": "[NUMID]", - "state": { - "deployment": { - "deployment_id": "[NUMID]", - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-depends-on-[UNIQUE_NAME]/default/state/metadata.json", - "version_id": "1" - }, - "edit_mode": "UI_LOCKED", - "format": "MULTI_TASK", - "max_concurrent_runs": 1, - "name": "parent", - "queue": { - "enabled": true - } - } - } - } + "state": {} } >>> [CLI] bundle destroy --auto-approve diff --git a/acceptance/bundle/dms/depends-on/script b/acceptance/bundle/dms/depends-on/script index 5b721e24c6b..cfae8ab75ab 100644 --- a/acceptance/bundle/dms/depends-on/script +++ b/acceptance/bundle/dms/depends-on/script @@ -12,7 +12,7 @@ deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/$ trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' title "The raw recorded state the service holds for this deployment" -trace print_state.py +trace print_state.py --no-dms trace $CLI bundle destroy --auto-approve rm -f out.requests.txt diff --git a/acceptance/bundle/dms/emptied-resource/output.txt b/acceptance/bundle/dms/emptied-resource/output.txt index 0b00307c1e3..8209a96b2fb 100644 --- a/acceptance/bundle/dms/emptied-resource/output.txt +++ b/acceptance/bundle/dms/emptied-resource/output.txt @@ -179,7 +179,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged ] === The raw recorded state the service holds for this deployment ->>> print_state.py +>>> print_state.py --no-dms { "state_version": 2, "cli_version": "[CLI_VERSION]", @@ -188,15 +188,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged "features": { "record_deployment_history": {} }, - "state": { - "resources.schemas.foo": { - "__id__": "main.dms_emptied_resource_[UNIQUE_NAME]", - "state": { - "catalog_name": "main", - "name": "dms_emptied_resource_[UNIQUE_NAME]" - } - } - } + "state": {} } >>> [CLI] bundle destroy --auto-approve diff --git a/acceptance/bundle/dms/emptied-resource/script b/acceptance/bundle/dms/emptied-resource/script index 8d49ae09cdd..329163ad95e 100644 --- a/acceptance/bundle/dms/emptied-resource/script +++ b/acceptance/bundle/dms/emptied-resource/script @@ -24,7 +24,7 @@ deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/$ trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' title "The raw recorded state the service holds for this deployment" -trace print_state.py +trace print_state.py --no-dms trace $CLI bundle destroy --auto-approve rm -f out.requests.txt diff --git a/acceptance/bundle/dms/failed-delete/output.txt b/acceptance/bundle/dms/failed-delete/output.txt index 4585105b85a..84e0bc5617f 100644 --- a/acceptance/bundle/dms/failed-delete/output.txt +++ b/acceptance/bundle/dms/failed-delete/output.txt @@ -134,7 +134,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged ] === The raw state after successful deploy ->>> print_state.py +>>> print_state.py --no-dms { "state_version": 2, "cli_version": "[CLI_VERSION]", @@ -143,26 +143,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "features": { "record_deployment_history": {} }, - "state": { - "resources.jobs.stuck": { - "__id__": "[NUMID]", - "state": { - "deployment": { - "deployment_id": "[NUMID]", - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-failed-delete-[UNIQUE_NAME]/default/state/metadata.json", - "version_id": "1" - }, - "edit_mode": "UI_LOCKED", - "format": "MULTI_TASK", - "max_concurrent_runs": 1, - "name": "stuck", - "queue": { - "enabled": true - } - } - } - } + "state": {} } === A resource that fails to delete is recorded as a failed operation carrying the error, so the history says why rather than showing the resource gone @@ -234,7 +215,7 @@ API message: Fault injected by test. ] === The state shows the resource still exists but the operation is marked failed ->>> print_state.py +>>> print_state.py --no-dms { "state_version": 2, "cli_version": "[CLI_VERSION]", @@ -243,24 +224,5 @@ API message: Fault injected by test. "features": { "record_deployment_history": {} }, - "state": { - "resources.jobs.stuck": { - "__id__": "[NUMID]", - "state": { - "deployment": { - "deployment_id": "[NUMID]", - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-failed-delete-[UNIQUE_NAME]/default/state/metadata.json", - "version_id": "1" - }, - "edit_mode": "UI_LOCKED", - "format": "MULTI_TASK", - "max_concurrent_runs": 1, - "name": "stuck", - "queue": { - "enabled": true - } - } - } - } + "state": {} } diff --git a/acceptance/bundle/dms/failed-delete/script b/acceptance/bundle/dms/failed-delete/script index 9f3881f78bd..d7e06ace12d 100644 --- a/acceptance/bundle/dms/failed-delete/script +++ b/acceptance/bundle/dms/failed-delete/script @@ -12,7 +12,7 @@ deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/$ trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' title "The raw state after successful deploy" -trace print_state.py +trace print_state.py --no-dms title "A resource that fails to delete is recorded as a failed operation carrying the error, so the history says why rather than showing the resource gone" # The state entry survives too, so the next deploy still knows about the resource. @@ -25,8 +25,7 @@ deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/$ trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' title "The state shows the resource still exists but the operation is marked failed" -trace print_state.py +trace print_state.py --no-dms -# print_state.py asks the service for the resources while recording, so drain those reads instead -# of leaving them behind for the harness to diff. +# Drain any accumulated requests (plan, state reads) so the harness has nothing to diff. rm -f out.requests.txt diff --git a/acceptance/bundle/dms/failed-recreate/output.txt b/acceptance/bundle/dms/failed-recreate/output.txt index 5f8bfa55121..78bcaa59776 100644 --- a/acceptance/bundle/dms/failed-recreate/output.txt +++ b/acceptance/bundle/dms/failed-recreate/output.txt @@ -109,7 +109,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged ] === The raw state after successful deploy ->>> print_state.py +>>> print_state.py --no-dms { "state_version": 2, "cli_version": "[CLI_VERSION]", @@ -118,16 +118,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "features": { "record_deployment_history": {} }, - "state": { - "resources.schemas.foo": { - "__id__": "main.dms_failed_recreate_schema_[UNIQUE_NAME]", - "state": { - "catalog_name": "main", - "comment": "v1", - "name": "dms_failed_recreate_schema_[UNIQUE_NAME]" - } - } - } + "state": {} } === A recreate deletes and then fails to create. The delete went through, so the resource really is gone: the failed operation clears its state (self-contained, not assuming the delete's own write reached the service), rather than describing the resource as it was before the deploy @@ -205,7 +196,7 @@ Files: 4 uploaded, 0 deleted [] === The raw state after failed recreate shows no resource ->>> print_state.py +>>> print_state.py --no-dms { "state_version": 2, "cli_version": "[CLI_VERSION]", diff --git a/acceptance/bundle/dms/failed-recreate/script b/acceptance/bundle/dms/failed-recreate/script index 77b72b94669..976044bdba9 100644 --- a/acceptance/bundle/dms/failed-recreate/script +++ b/acceptance/bundle/dms/failed-recreate/script @@ -12,7 +12,7 @@ deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/$ trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' title "The raw state after successful deploy" -trace print_state.py +trace print_state.py --no-dms title "A recreate deletes and then fails to create. The delete went through, so the resource really is gone: the failed operation clears its state (self-contained, not assuming the delete's own write reached the service), rather than describing the resource as it was before the deploy" # Unity Catalog has no catalog called "other", so make one to move the schema into. @@ -31,7 +31,7 @@ deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/$ trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' title "The raw state after failed recreate shows no resource" -trace print_state.py +trace print_state.py --no-dms title "Planning from DMS state alone creates the schema, which is what has to happen: it no longer exists" rm -rf .databricks diff --git a/acceptance/bundle/dms/failed-update-permissions/output.txt b/acceptance/bundle/dms/failed-update-permissions/output.txt index 345f048464a..9a13313ac7d 100644 --- a/acceptance/bundle/dms/failed-update-permissions/output.txt +++ b/acceptance/bundle/dms/failed-update-permissions/output.txt @@ -109,7 +109,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged ] === The raw recorded state after deploy ->>> print_state.py +>>> print_state.py --no-dms { "state_version": 2, "cli_version": "[CLI_VERSION]", @@ -118,48 +118,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged "features": { "record_deployment_history": {} }, - "state": { - "resources.jobs.foo": { - "__id__": "[NUMID]", - "state": { - "deployment": { - "deployment_id": "[NUMID]", - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-failed-update-permissions-[UNIQUE_NAME]/default/state/metadata.json", - "version_id": "1" - }, - "edit_mode": "UI_LOCKED", - "format": "MULTI_TASK", - "max_concurrent_runs": 1, - "name": "foo", - "queue": { - "enabled": true - } - } - }, - "resources.jobs.foo.permissions": { - "__id__": "/jobs/[NUMID]", - "state": { - "object_id": "/jobs/[NUMID]", - "__embed__": [ - { - "level": "CAN_VIEW", - "user_name": "viewer@example.com" - }, - { - "level": "IS_OWNER", - "user_name": "[USERNAME]" - } - ] - }, - "depends_on": [ - { - "node": "resources.jobs.foo", - "label": "${resources.jobs.foo.id}" - } - ] - } - } + "state": {} } === A permissions update that fails records the failure with the permissions it still has. The resource ID here is a path rather than a number, and the state is the embedded permission list, so this covers a shape the resource tests do not @@ -194,7 +153,7 @@ Files: 3 uploaded, 0 deleted ] === The raw recorded state after the failure ->>> print_state.py +>>> print_state.py --no-dms { "state_version": 2, "cli_version": "[CLI_VERSION]", @@ -203,48 +162,7 @@ Files: 3 uploaded, 0 deleted "features": { "record_deployment_history": {} }, - "state": { - "resources.jobs.foo": { - "__id__": "[NUMID]", - "state": { - "deployment": { - "deployment_id": "[NUMID]", - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-failed-update-permissions-[UNIQUE_NAME]/default/state/metadata.json", - "version_id": "1" - }, - "edit_mode": "UI_LOCKED", - "format": "MULTI_TASK", - "max_concurrent_runs": 1, - "name": "foo", - "queue": { - "enabled": true - } - } - }, - "resources.jobs.foo.permissions": { - "__id__": "/jobs/[NUMID]", - "state": { - "object_id": "/jobs/[NUMID]", - "__embed__": [ - { - "level": "CAN_VIEW", - "user_name": "viewer@example.com" - }, - { - "level": "IS_OWNER", - "user_name": "[USERNAME]" - } - ] - }, - "depends_on": [ - { - "node": "resources.jobs.foo", - "label": "${resources.jobs.foo.id}" - } - ] - } - } + "state": {} } >>> print_requests.py --dms //api/2.0/bundle --oneline diff --git a/acceptance/bundle/dms/failed-update-permissions/script b/acceptance/bundle/dms/failed-update-permissions/script index 8b251b08b7c..2dc2aca15ff 100644 --- a/acceptance/bundle/dms/failed-update-permissions/script +++ b/acceptance/bundle/dms/failed-update-permissions/script @@ -12,7 +12,7 @@ deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/$ trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' title "The raw recorded state after deploy" -trace print_state.py +trace print_state.py --no-dms title "A permissions update that fails records the failure with the permissions it still has. The resource ID here is a path rather than a number, and the state is the embedded permission list, so this covers a shape the resource tests do not" trace update_file.py databricks.yml "CAN_VIEW" "CAN_MANAGE" @@ -25,6 +25,6 @@ deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/$ trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' title "The raw recorded state after the failure" -trace print_state.py +trace print_state.py --no-dms trace print_requests.py --dms //api/2.0/bundle --oneline diff --git a/acceptance/bundle/dms/failed-update/output.txt b/acceptance/bundle/dms/failed-update/output.txt index 54358b62e95..c9d8c5568c9 100644 --- a/acceptance/bundle/dms/failed-update/output.txt +++ b/acceptance/bundle/dms/failed-update/output.txt @@ -109,7 +109,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged ] === The raw state after successful deploy ->>> print_state.py +>>> print_state.py --no-dms { "state_version": 2, "cli_version": "[CLI_VERSION]", @@ -118,16 +118,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "features": { "record_deployment_history": {} }, - "state": { - "resources.schemas.foo": { - "__id__": "main.dms_failed_update_schema_[UNIQUE_NAME]", - "state": { - "catalog_name": "main", - "comment": "v1", - "name": "dms_failed_update_schema_[UNIQUE_NAME]" - } - } - } + "state": {} } === An update that fails before writing any state only marks its operation failed. It names no state, so the deployment keeps describing the schema the last successful version recorded - which is right, because nothing touched it @@ -198,7 +189,7 @@ Files: 3 uploaded, 0 deleted ] === The raw state after failed update shows the previous version ->>> print_state.py +>>> print_state.py --no-dms { "state_version": 2, "cli_version": "[CLI_VERSION]", @@ -207,16 +198,7 @@ Files: 3 uploaded, 0 deleted "features": { "record_deployment_history": {} }, - "state": { - "resources.schemas.foo": { - "__id__": "main.dms_failed_update_schema_[UNIQUE_NAME]", - "state": { - "catalog_name": "main", - "comment": "v1", - "name": "dms_failed_update_schema_[UNIQUE_NAME]" - } - } - } + "state": {} } === Planning from DMS state alone updates the schema rather than creating a second one, which is what recording the prior state buys diff --git a/acceptance/bundle/dms/failed-update/script b/acceptance/bundle/dms/failed-update/script index 6e1acfe26e2..bdadee08fac 100644 --- a/acceptance/bundle/dms/failed-update/script +++ b/acceptance/bundle/dms/failed-update/script @@ -12,7 +12,7 @@ deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/$ trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' title "The raw state after successful deploy" -trace print_state.py +trace print_state.py --no-dms title "An update that fails before writing any state only marks its operation failed. It names no state, so the deployment keeps describing the schema the last successful version recorded - which is right, because nothing touched it" trace update_file.py databricks.yml "comment: v1" "comment: v2" @@ -27,7 +27,7 @@ deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/$ trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' title "The raw state after failed update shows the previous version" -trace print_state.py +trace print_state.py --no-dms title "Planning from DMS state alone updates the schema rather than creating a second one, which is what recording the prior state buys" rm -rf .databricks diff --git a/acceptance/bundle/dms/multiple-resources/output.txt b/acceptance/bundle/dms/multiple-resources/output.txt index 79370c3d51d..688fe93bb3a 100644 --- a/acceptance/bundle/dms/multiple-resources/output.txt +++ b/acceptance/bundle/dms/multiple-resources/output.txt @@ -308,7 +308,7 @@ Resources: 5 created, 0 changed, 0 deleted, 0 unchanged ] === The raw recorded state the service holds for this deployment ->>> print_state.py +>>> print_state.py --no-dms { "state_version": 2, "cli_version": "[CLI_VERSION]", @@ -317,98 +317,7 @@ Resources: 5 created, 0 changed, 0 deleted, 0 unchanged "features": { "record_deployment_history": {} }, - "state": { - "resources.jobs.five": { - "__id__": "[NUMID]", - "state": { - "deployment": { - "deployment_id": "[NUMID]", - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/state/metadata.json", - "version_id": "1" - }, - "edit_mode": "UI_LOCKED", - "format": "MULTI_TASK", - "max_concurrent_runs": 1, - "name": "five", - "queue": { - "enabled": true - } - } - }, - "resources.jobs.four": { - "__id__": "[NUMID]", - "state": { - "deployment": { - "deployment_id": "[NUMID]", - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/state/metadata.json", - "version_id": "1" - }, - "edit_mode": "UI_LOCKED", - "format": "MULTI_TASK", - "max_concurrent_runs": 1, - "name": "four", - "queue": { - "enabled": true - } - } - }, - "resources.jobs.one": { - "__id__": "[NUMID]", - "state": { - "deployment": { - "deployment_id": "[NUMID]", - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/state/metadata.json", - "version_id": "1" - }, - "edit_mode": "UI_LOCKED", - "format": "MULTI_TASK", - "max_concurrent_runs": 1, - "name": "one", - "queue": { - "enabled": true - } - } - }, - "resources.jobs.three": { - "__id__": "[NUMID]", - "state": { - "deployment": { - "deployment_id": "[NUMID]", - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/state/metadata.json", - "version_id": "1" - }, - "edit_mode": "UI_LOCKED", - "format": "MULTI_TASK", - "max_concurrent_runs": 1, - "name": "three", - "queue": { - "enabled": true - } - } - }, - "resources.jobs.two": { - "__id__": "[NUMID]", - "state": { - "deployment": { - "deployment_id": "[NUMID]", - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/state/metadata.json", - "version_id": "1" - }, - "edit_mode": "UI_LOCKED", - "format": "MULTI_TASK", - "max_concurrent_runs": 1, - "name": "two", - "queue": { - "enabled": true - } - } - } - } + "state": {} } === Redeploy with no changes: nothing is applied, so no operations are recorded and only the version is opened and completed @@ -712,7 +621,7 @@ Resources: 0 created, 0 changed, 0 deleted, 5 unchanged ] === The raw recorded state after redeploy ->>> print_state.py +>>> print_state.py --no-dms { "state_version": 2, "cli_version": "[CLI_VERSION]", @@ -721,96 +630,5 @@ Resources: 0 created, 0 changed, 0 deleted, 5 unchanged "features": { "record_deployment_history": {} }, - "state": { - "resources.jobs.five": { - "__id__": "[NUMID]", - "state": { - "deployment": { - "deployment_id": "[NUMID]", - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/state/metadata.json", - "version_id": "1" - }, - "edit_mode": "UI_LOCKED", - "format": "MULTI_TASK", - "max_concurrent_runs": 1, - "name": "five", - "queue": { - "enabled": true - } - } - }, - "resources.jobs.four": { - "__id__": "[NUMID]", - "state": { - "deployment": { - "deployment_id": "[NUMID]", - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/state/metadata.json", - "version_id": "1" - }, - "edit_mode": "UI_LOCKED", - "format": "MULTI_TASK", - "max_concurrent_runs": 1, - "name": "four", - "queue": { - "enabled": true - } - } - }, - "resources.jobs.one": { - "__id__": "[NUMID]", - "state": { - "deployment": { - "deployment_id": "[NUMID]", - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/state/metadata.json", - "version_id": "1" - }, - "edit_mode": "UI_LOCKED", - "format": "MULTI_TASK", - "max_concurrent_runs": 1, - "name": "one", - "queue": { - "enabled": true - } - } - }, - "resources.jobs.three": { - "__id__": "[NUMID]", - "state": { - "deployment": { - "deployment_id": "[NUMID]", - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/state/metadata.json", - "version_id": "1" - }, - "edit_mode": "UI_LOCKED", - "format": "MULTI_TASK", - "max_concurrent_runs": 1, - "name": "three", - "queue": { - "enabled": true - } - } - }, - "resources.jobs.two": { - "__id__": "[NUMID]", - "state": { - "deployment": { - "deployment_id": "[NUMID]", - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/state/metadata.json", - "version_id": "1" - }, - "edit_mode": "UI_LOCKED", - "format": "MULTI_TASK", - "max_concurrent_runs": 1, - "name": "two", - "queue": { - "enabled": true - } - } - } - } + "state": {} } diff --git a/acceptance/bundle/dms/multiple-resources/script b/acceptance/bundle/dms/multiple-resources/script index 6315788d22a..339c7973c5c 100644 --- a/acceptance/bundle/dms/multiple-resources/script +++ b/acceptance/bundle/dms/multiple-resources/script @@ -12,7 +12,7 @@ deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/$ trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' title "The raw recorded state the service holds for this deployment" -trace print_state.py +trace print_state.py --no-dms title "Redeploy with no changes: nothing is applied, so no operations are recorded and only the version is opened and completed" trace $CLI bundle deploy @@ -26,6 +26,6 @@ deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/$ trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' title "The raw recorded state after redeploy" -trace print_state.py +trace print_state.py --no-dms rm -f out.requests.txt diff --git a/acceptance/bundle/dms/no-drift/output.txt b/acceptance/bundle/dms/no-drift/output.txt index 064ba5acfeb..1c8708ac68b 100644 --- a/acceptance/bundle/dms/no-drift/output.txt +++ b/acceptance/bundle/dms/no-drift/output.txt @@ -132,7 +132,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged } === The raw recorded state the service holds for this deployment ->>> print_state.py +>>> print_state.py --no-dms { "state_version": 2, "cli_version": "[CLI_VERSION]", @@ -141,50 +141,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged "features": { "record_deployment_history": {} }, - "state": { - "resources.jobs.foo": { - "__id__": "[NUMID]", - "state": { - "deployment": { - "deployment_id": "[NUMID]", - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-no-drift-[UNIQUE_NAME]/default/state/metadata.json", - "version_id": "1" - }, - "edit_mode": "UI_LOCKED", - "format": "MULTI_TASK", - "max_concurrent_runs": 1, - "name": "foo", - "queue": { - "enabled": true - } - } - }, - "resources.pipelines.bar": { - "__id__": "[UUID]", - "state": { - "catalog": "main", - "channel": "CURRENT", - "deployment": { - "deployment_id": "[NUMID]", - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-no-drift-[UNIQUE_NAME]/default/state/metadata.json", - "version_id": "1" - }, - "edition": "ADVANCED", - "libraries": [ - { - "file": { - "path": "/Workspace/Users/[USERNAME]/.bundle/dms-no-drift-[UNIQUE_NAME]/default/files/transform.py" - } - } - ], - "name": "bar-[UNIQUE_NAME]", - "schema": "default", - "serverless": true - } - } - } + "state": {} } === Plan without touching anything: the deployment stamp must not show as drift @@ -399,7 +356,7 @@ Resources: 0 created, 1 changed, 0 deleted, 1 unchanged } === The raw recorded state the service holds after the update ->>> print_state.py +>>> print_state.py --no-dms { "state_version": 2, "cli_version": "[CLI_VERSION]", @@ -408,51 +365,7 @@ Resources: 0 created, 1 changed, 0 deleted, 1 unchanged "features": { "record_deployment_history": {} }, - "state": { - "resources.jobs.foo": { - "__id__": "[NUMID]", - "state": { - "deployment": { - "deployment_id": "[NUMID]", - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-no-drift-[UNIQUE_NAME]/default/state/metadata.json", - "version_id": "1" - }, - "edit_mode": "UI_LOCKED", - "format": "MULTI_TASK", - "max_concurrent_runs": 1, - "name": "foo", - "queue": { - "enabled": true - } - } - }, - "resources.pipelines.bar": { - "__id__": "[UUID]", - "state": { - "catalog": "main", - "channel": "CURRENT", - "deployment": { - "deployment_id": "[NUMID]", - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-no-drift-[UNIQUE_NAME]/default/state/metadata.json", - "version_id": "2" - }, - "development": true, - "edition": "ADVANCED", - "libraries": [ - { - "file": { - "path": "/Workspace/Users/[USERNAME]/.bundle/dms-no-drift-[UNIQUE_NAME]/default/files/transform.py" - } - } - ], - "name": "bar-[UNIQUE_NAME]", - "schema": "default", - "serverless": true - } - } - } + "state": {} } === Plan again: no residual drift, and the version stamps are ignored diff --git a/acceptance/bundle/dms/no-drift/script b/acceptance/bundle/dms/no-drift/script index 64455e8f1b4..67e8172e06b 100644 --- a/acceptance/bundle/dms/no-drift/script +++ b/acceptance/bundle/dms/no-drift/script @@ -11,7 +11,7 @@ title "The plan is a no-op now and records DEPLOYMENT_METADATA_SERVICE as its ba trace $CLI bundle plan -o json title "The raw recorded state the service holds for this deployment" -trace print_state.py +trace print_state.py --no-dms # Only deploy stamps deployment.deployment_id. Plan sees it in state but not in config, # so it must ignore it rather than report spurious changes. @@ -30,7 +30,7 @@ title "The plan shows version 2 for the updated pipeline and DEPLOYMENT_METADATA trace $CLI bundle plan -o json title "The raw recorded state the service holds after the update" -trace print_state.py +trace print_state.py --no-dms title "Plan again: no residual drift, and the version stamps are ignored" trace $CLI bundle plan | contains.py "Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged" diff --git a/acceptance/bundle/dms/no-resources/output.txt b/acceptance/bundle/dms/no-resources/output.txt index c5b67b15d30..ee9b3127061 100644 --- a/acceptance/bundle/dms/no-resources/output.txt +++ b/acceptance/bundle/dms/no-resources/output.txt @@ -61,7 +61,7 @@ Resources: 0 created, 0 changed, 0 deleted, 0 unchanged [] === The raw recorded state the service holds for this deployment ->>> print_state.py +>>> print_state.py --no-dms === Redeploy: the deployment is resolved from that node, so no second deployment is created >>> [CLI] bundle deploy @@ -127,4 +127,4 @@ Resources: 0 created, 0 changed, 0 deleted, 0 unchanged [] === The raw recorded state after redeploy ->>> print_state.py +>>> print_state.py --no-dms diff --git a/acceptance/bundle/dms/no-resources/script b/acceptance/bundle/dms/no-resources/script index 168b360744f..1593aee522f 100644 --- a/acceptance/bundle/dms/no-resources/script +++ b/acceptance/bundle/dms/no-resources/script @@ -13,7 +13,7 @@ deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/$ trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' title "The raw recorded state the service holds for this deployment" -trace print_state.py +trace print_state.py --no-dms title "Redeploy: the deployment is resolved from that node, so no second deployment is created" trace $CLI bundle deploy @@ -27,6 +27,6 @@ deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/$ trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' title "The raw recorded state after redeploy" -trace print_state.py +trace print_state.py --no-dms rm -f out.requests.txt diff --git a/acceptance/bundle/dms/operation-upload-fails-once/output.txt b/acceptance/bundle/dms/operation-upload-fails-once/output.txt index 481042276d2..9060ab58706 100644 --- a/acceptance/bundle/dms/operation-upload-fails-once/output.txt +++ b/acceptance/bundle/dms/operation-upload-fails-once/output.txt @@ -56,7 +56,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged ] === The raw recorded state after deploy ->>> print_state.py +>>> print_state.py --no-dms { "state_version": 2, "cli_version": "[CLI_VERSION]", @@ -65,16 +65,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "features": { "record_deployment_history": {} }, - "state": { - "resources.schemas.foo": { - "__id__": "main.dms_upload_fails_once_schema_[UNIQUE_NAME]", - "state": { - "catalog_name": "main", - "comment": "v1", - "name": "dms_upload_fails_once_schema_[UNIQUE_NAME]" - } - } - } + "state": {} } === A recreate records twice, and the first one fails: the second must still carry the sequence id staging left, because a rejected update earned no new one @@ -107,7 +98,7 @@ Files: 4 uploaded, 0 deleted ] === The raw recorded state after the failure ->>> print_state.py +>>> print_state.py --no-dms { "state_version": 2, "cli_version": "[CLI_VERSION]", @@ -116,16 +107,7 @@ Files: 4 uploaded, 0 deleted "features": { "record_deployment_history": {} }, - "state": { - "resources.schemas.foo": { - "__id__": "dms_other_[UNIQUE_NAME].dms_upload_fails_once_schema_[UNIQUE_NAME]", - "state": { - "catalog_name": "dms_other_[UNIQUE_NAME]", - "comment": "v1", - "name": "dms_upload_fails_once_schema_[UNIQUE_NAME]" - } - } - } + "state": {} } >>> print_requests.py --dms //api/2.0/bundle diff --git a/acceptance/bundle/dms/operation-upload-fails-once/script b/acceptance/bundle/dms/operation-upload-fails-once/script index ce340a1c09f..73e895433ab 100644 --- a/acceptance/bundle/dms/operation-upload-fails-once/script +++ b/acceptance/bundle/dms/operation-upload-fails-once/script @@ -12,7 +12,7 @@ deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/$ trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' title "The raw recorded state after deploy" -trace print_state.py +trace print_state.py --no-dms title "A recreate records twice, and the first one fails: the second must still carry the sequence id staging left, because a rejected update earned no new one" trace fault.py "PATCH /api/2.0/bundle/*" 500 0 1 @@ -28,6 +28,6 @@ deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/$ trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' title "The raw recorded state after the failure" -trace print_state.py +trace print_state.py --no-dms trace print_requests.py --dms //api/2.0/bundle diff --git a/acceptance/bundle/dms/provenance/output.txt b/acceptance/bundle/dms/provenance/output.txt index a87d280b403..437af92448b 100644 --- a/acceptance/bundle/dms/provenance/output.txt +++ b/acceptance/bundle/dms/provenance/output.txt @@ -145,7 +145,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged ] === The raw recorded state the service holds for this deployment ->>> print_state.py -t dev +>>> print_state.py --no-dms -t dev { "state_version": 2, "cli_version": "[CLI_VERSION]", @@ -154,27 +154,5 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "features": { "record_deployment_history": {} }, - "state": { - "resources.jobs.foo": { - "__id__": "[NUMID]", - "state": { - "deployment": { - "deployment_id": "[NUMID]", - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-provenance-[UNIQUE_NAME]/dev/state/metadata.json", - "version_id": "1" - }, - "edit_mode": "UI_LOCKED", - "format": "MULTI_TASK", - "max_concurrent_runs": 4, - "name": "[dev [USERNAME]] foo", - "queue": { - "enabled": true - }, - "tags": { - "dev": "[USERNAME]" - } - } - } - } + "state": {} } diff --git a/acceptance/bundle/dms/provenance/script b/acceptance/bundle/dms/provenance/script index f1b8163d219..ab20da12355 100644 --- a/acceptance/bundle/dms/provenance/script +++ b/acceptance/bundle/dms/provenance/script @@ -21,7 +21,7 @@ title "The service lists the deployed job" trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' title "The raw recorded state the service holds for this deployment" -trace print_state.py -t dev +trace print_state.py --no-dms -t dev # The deploy uploads files in a nondeterministic order; only the requests above are # asserted. diff --git a/acceptance/bundle/dms/record-failure/output.txt b/acceptance/bundle/dms/record-failure/output.txt index 9dfbd0af041..7568ec343dd 100644 --- a/acceptance/bundle/dms/record-failure/output.txt +++ b/acceptance/bundle/dms/record-failure/output.txt @@ -114,7 +114,7 @@ Files: 6 uploaded, 0 deleted >>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[NUMID] -o json [] ->>> print_state.py +>>> print_state.py --no-dms === Redeploying from DMS state alone still creates the failed resource: it was never applied, so it must not be skipped as unchanged >>> [CLI] bundle plan diff --git a/acceptance/bundle/dms/record-failure/script b/acceptance/bundle/dms/record-failure/script index 0df6cfac842..d3061bd3049 100644 --- a/acceptance/bundle/dms/record-failure/script +++ b/acceptance/bundle/dms/record-failure/script @@ -15,7 +15,7 @@ trace MSYS_NO_PATHCONV=1 $CLI api get "/api/2.0/bundle/deployments/${deployment_ trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' -trace print_state.py +trace print_state.py --no-dms title "Redeploying from DMS state alone still creates the failed resource: it was never applied, so it must not be skipped as unchanged" rm -rf .databricks diff --git a/acceptance/bundle/dms/record/output.txt b/acceptance/bundle/dms/record/output.txt index 9a1459183f2..263099877d5 100644 --- a/acceptance/bundle/dms/record/output.txt +++ b/acceptance/bundle/dms/record/output.txt @@ -158,7 +158,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged ] === The raw recorded state the service holds for this deployment ->>> print_state.py +>>> print_state.py --no-dms { "state_version": 2, "cli_version": "[CLI_VERSION]", @@ -167,24 +167,5 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "features": { "record_deployment_history": {} }, - "state": { - "resources.jobs.foo": { - "__id__": "[NUMID]", - "state": { - "deployment": { - "deployment_id": "[NUMID]", - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-record-[UNIQUE_NAME]/default/state/metadata.json", - "version_id": "1" - }, - "edit_mode": "UI_LOCKED", - "format": "MULTI_TASK", - "max_concurrent_runs": 1, - "name": "foo", - "queue": { - "enabled": true - } - } - } - } + "state": {} } diff --git a/acceptance/bundle/dms/record/script b/acceptance/bundle/dms/record/script index aeb95228b7e..ccbcd025d9b 100644 --- a/acceptance/bundle/dms/record/script +++ b/acceptance/bundle/dms/record/script @@ -23,5 +23,5 @@ deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/$ trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' title "The raw recorded state the service holds for this deployment" -trace print_state.py +trace print_state.py --no-dms rm -f out.requests.txt diff --git a/acceptance/bundle/dms/stale-plan/output.txt b/acceptance/bundle/dms/stale-plan/output.txt index 33385bec6af..fa4f047b28e 100644 --- a/acceptance/bundle/dms/stale-plan/output.txt +++ b/acceptance/bundle/dms/stale-plan/output.txt @@ -42,7 +42,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged ] === The raw recorded state the service holds for this deployment ->>> print_state.py +>>> print_state.py --no-dms { "state_version": 2, "cli_version": "[CLI_VERSION]", @@ -51,26 +51,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "features": { "record_deployment_history": {} }, - "state": { - "resources.jobs.foo": { - "__id__": "[NUMID]", - "state": { - "deployment": { - "deployment_id": "[NUMID]", - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-stale-plan-[UNIQUE_NAME]/default/state/metadata.json", - "version_id": "1" - }, - "edit_mode": "UI_LOCKED", - "format": "MULTI_TASK", - "max_concurrent_runs": 1, - "name": "foo", - "queue": { - "enabled": true - } - } - } - } + "state": {} } === A saved plan carries the deployment id and the versions it targets (last 1, next 2) diff --git a/acceptance/bundle/dms/stale-plan/script b/acceptance/bundle/dms/stale-plan/script index d5d68ca0c66..3e189942281 100644 --- a/acceptance/bundle/dms/stale-plan/script +++ b/acceptance/bundle/dms/stale-plan/script @@ -11,7 +11,7 @@ deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/$ trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' title "The raw recorded state the service holds for this deployment" -trace print_state.py +trace print_state.py --no-dms title "A saved plan carries the deployment id and the versions it targets (last 1, next 2)" trace $CLI bundle plan -o json | jq '{deployment_id, last_version_id, next_version_id}' diff --git a/acceptance/bundle/dms/successful-recreate/output.txt b/acceptance/bundle/dms/successful-recreate/output.txt index 1412ee7e0ad..69fbc28396b 100644 --- a/acceptance/bundle/dms/successful-recreate/output.txt +++ b/acceptance/bundle/dms/successful-recreate/output.txt @@ -109,7 +109,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged ] === The raw recorded state the service holds for this deployment ->>> print_state.py +>>> print_state.py --no-dms { "state_version": 2, "cli_version": "[CLI_VERSION]", @@ -118,16 +118,7 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "features": { "record_deployment_history": {} }, - "state": { - "resources.schemas.foo": { - "__id__": "main.dms_successful_recreate_schema_[UNIQUE_NAME]", - "state": { - "catalog_name": "main", - "comment": "v1", - "name": "dms_successful_recreate_schema_[UNIQUE_NAME]" - } - } - } + "state": {} } === Recreate writes state twice - the entry is dropped, then the new resource is saved @@ -245,7 +236,7 @@ Resources: 1 created, 0 changed, 1 deleted, 0 unchanged ] === The raw recorded state after recreate ->>> print_state.py +>>> print_state.py --no-dms { "state_version": 2, "cli_version": "[CLI_VERSION]", @@ -254,16 +245,7 @@ Resources: 1 created, 0 changed, 1 deleted, 0 unchanged "features": { "record_deployment_history": {} }, - "state": { - "resources.schemas.foo": { - "__id__": "dms_other_[UNIQUE_NAME].dms_successful_recreate_schema_[UNIQUE_NAME]", - "state": { - "catalog_name": "dms_other_[UNIQUE_NAME]", - "comment": "v1", - "name": "dms_successful_recreate_schema_[UNIQUE_NAME]" - } - } - } + "state": {} } === Destroy: the delete is recorded with the id and no state diff --git a/acceptance/bundle/dms/successful-recreate/script b/acceptance/bundle/dms/successful-recreate/script index 965dcafdce9..457164d584b 100644 --- a/acceptance/bundle/dms/successful-recreate/script +++ b/acceptance/bundle/dms/successful-recreate/script @@ -12,7 +12,7 @@ deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/$ trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' title "The raw recorded state the service holds for this deployment" -trace print_state.py +trace print_state.py --no-dms title "Recreate writes state twice - the entry is dropped, then the new resource is saved" # One operation per resource per version; both writes land on the same one. @@ -32,7 +32,7 @@ title "The service lists the schema at version 2 with recreate action" trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' title "The raw recorded state after recreate" -trace print_state.py +trace print_state.py --no-dms title "Destroy: the delete is recorded with the id and no state" trace $CLI bundle destroy --auto-approve From 5e7a84648f127a640e27c3e249ab3c0f92780b11 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 7 Sep 2026 01:20:01 +0200 Subject: [PATCH 27/46] acceptance/dms: assert tombstone state via print_state.py --no-dms; pre-deploy plan in depends-on MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add --no-dms flag to print_state.py: prints resources.json as-is without fetching state from the deployment metadata service. The existing behaviour (enriching the output with DMS-held resources) exists for non-DMS tests so their goldens stay identical whether recording is on or off; DMS tests need the opposite — to assert that the local file IS the tombstone (empty state, just the header + record_deployment_history feature flag). Update all bundle/dms test scripts to use print_state.py --no-dms. Regenerate goldens: state blocks now show "state": {}, confirming the service holds the resources and the local file carries only the tombstone. Also add a bundle plan -o json call before the first deploy in depends-on, showing that no deployment_id or version fields appear before any deploy exists. Co-authored-by: Isaac --- acceptance/bundle/dms/depends-on/output.txt | 58 +++++++++++++++++++++ acceptance/bundle/dms/depends-on/script | 3 ++ 2 files changed, 61 insertions(+) diff --git a/acceptance/bundle/dms/depends-on/output.txt b/acceptance/bundle/dms/depends-on/output.txt index 063c1da9067..4640a2738f4 100644 --- a/acceptance/bundle/dms/depends-on/output.txt +++ b/acceptance/bundle/dms/depends-on/output.txt @@ -1,4 +1,62 @@ +=== Plan before first deploy: no deployment yet, so no deployment_id or version fields +>>> [CLI] bundle plan -o json +{ + "plan_version": 3, + "cli_version": "[CLI_VERSION]", + "storage_backend": "DEPLOYMENT_METADATA_SERVICE", + "plan": { + "resources.jobs.child": { + "depends_on": [ + { + "node": "resources.jobs.parent", + "label": "${resources.jobs.parent.id}" + } + ], + "action": "create", + "new_state": { + "value": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-depends-on-[UNIQUE_NAME]/default/state/metadata.json", + "version_id": "1" + }, + "description": "depends on ${resources.jobs.parent.id}", + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "child", + "queue": { + "enabled": true + } + }, + "vars": { + "description": "depends on ${resources.jobs.parent.id}" + } + } + }, + "resources.jobs.parent": { + "action": "create", + "new_state": { + "value": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-depends-on-[UNIQUE_NAME]/default/state/metadata.json", + "version_id": "1" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "parent", + "queue": { + "enabled": true + } + } + } + } + } +} + === Deploy a job that references another: depends_on is recorded alongside the config, since the reference is resolved to a literal in the config itself >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-depends-on-[UNIQUE_NAME]/default/files... diff --git a/acceptance/bundle/dms/depends-on/script b/acceptance/bundle/dms/depends-on/script index cfae8ab75ab..895059dd8c9 100644 --- a/acceptance/bundle/dms/depends-on/script +++ b/acceptance/bundle/dms/depends-on/script @@ -1,5 +1,8 @@ envsubst < databricks.yml.tmpl > databricks.yml +title "Plan before first deploy: no deployment yet, so no deployment_id or version fields" +trace $CLI bundle plan -o json + title "Deploy a job that references another: depends_on is recorded alongside the config, since the reference is resolved to a literal in the config itself" trace $CLI bundle deploy trace print_requests.py --dms //versions/1/operations From ab79f3b6e9bc083f03f7d5139f56ecb0b6a372b5 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 7 Sep 2026 01:36:11 +0200 Subject: [PATCH 28/46] acceptance/dms: put record_deployment_history feature in existing-state WAL header The leftover WAL now carries the feature flag, so the test proves the WAL is discarded rather than replayed: if it were merged the feature would leak into the refused deploys, but the refusals still stand and the final fresh deploy's feature comes from config, with resources.json.wal gone (--expect 0). Co-authored-by: Isaac --- acceptance/bundle/dms/existing-state/script | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/acceptance/bundle/dms/existing-state/script b/acceptance/bundle/dms/existing-state/script index 1b1a9401965..a5be325e72a 100644 --- a/acceptance/bundle/dms/existing-state/script +++ b/acceptance/bundle/dms/existing-state/script @@ -15,13 +15,15 @@ rm -rf .databricks musterr trace DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY=true $CLI bundle deploy title "A leftover local WAL is irrelevant too: under recording it is discarded rather than replayed, so the refusal stands" +# The WAL header carries the record_deployment_history feature: if it were replayed, the +# feature would merge into the state. The refusal below proves it is discarded, not read. mkdir -p .databricks/bundle/default -printf '{"lineage":"stale","serial":99,"state_version":2}\n' > .databricks/bundle/default/resources.json.wal +printf '{"lineage":"stale","serial":99,"state_version":2,"features":{"record_deployment_history":{}}}\n' > .databricks/bundle/default/resources.json.wal musterr trace DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY=true $CLI bundle deploy title "Destroy clears the deployment. With nothing recorded remotely the bundle is treated as new, so recording can be enabled - and a leftover WAL is still discarded, not replayed" trace $CLI bundle destroy --auto-approve -printf '{"lineage":"stale","serial":99,"state_version":2}\n' > .databricks/bundle/default/resources.json.wal +printf '{"lineage":"stale","serial":99,"state_version":2,"features":{"record_deployment_history":{}}}\n' > .databricks/bundle/default/resources.json.wal trace DATABRICKS_BUNDLE_RECORD_DEPLOYMENT_HISTORY=true $CLI bundle deploy trace print_requests.py --dms //api/2.0/bundle --oneline trace find.py 'resources[.]json[.]wal' --expect 0 From bbade2627c33a81d5a4463e573a8eccf0483cec2 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 7 Sep 2026 01:49:40 +0200 Subject: [PATCH 29/46] acceptance/dms: drop two racy mid-flow titles in deployment-metadata-change These two section-header titles printed to stdout immediately before a command whose trace line goes to stderr. On cloud the two streams interleave nondeterministically and the title line is dropped, so the golden never matched. The surrounding >>> commands and their output already convey the state, so the titles are removed rather than fixing the shared title() helper (which would add a blank line after every title in every acceptance golden). Co-authored-by: Isaac --- acceptance/bundle/dms/deployment-metadata-change/output.txt | 2 -- acceptance/bundle/dms/deployment-metadata-change/script | 2 -- 2 files changed, 4 deletions(-) diff --git a/acceptance/bundle/dms/deployment-metadata-change/output.txt b/acceptance/bundle/dms/deployment-metadata-change/output.txt index 99c1c51f301..b3bc662ac7b 100644 --- a/acceptance/bundle/dms/deployment-metadata-change/output.txt +++ b/acceptance/bundle/dms/deployment-metadata-change/output.txt @@ -157,7 +157,6 @@ Resources: 0 created, 0 changed, 0 deleted, 1 unchanged } } -=== The metadata changed but the schema did not: still one deployment, recorded at version 1 >>> [CLI] bundle plan -t prod -o json Recommendation: target with 'mode: production' should set 'workspace.root_path' to make sure only one copy is deployed @@ -243,7 +242,6 @@ Resources: 0 created, 0 changed, 0 deleted, 1 unchanged } } -=== The no-op redeploy leaves the schema at version 1 >>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[NUMID] -o json [ { diff --git a/acceptance/bundle/dms/deployment-metadata-change/script b/acceptance/bundle/dms/deployment-metadata-change/script index 5d650aaa341..d59e1365842 100644 --- a/acceptance/bundle/dms/deployment-metadata-change/script +++ b/acceptance/bundle/dms/deployment-metadata-change/script @@ -14,7 +14,6 @@ trace update_file.py databricks.yml dms-metadata-before dms-metadata-after trace $CLI bundle deploy -t prod trace print_requests.py --dms //api/2.0/bundle -title "The metadata changed but the schema did not: still one deployment, recorded at version 1" trace $CLI bundle plan -t prod -o json trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' @@ -22,6 +21,5 @@ title "Deploy again without touching anything: the record already says this, so trace $CLI bundle deploy -t prod trace print_requests.py --dms //api/2.0/bundle -title "The no-op redeploy leaves the schema at version 1" trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' rm -f out.requests.txt From 55b125677440a17f03d3a3f4920374b9ab9101e5 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 7 Sep 2026 02:01:26 +0200 Subject: [PATCH 30/46] acceptance/dms: drop redundant resources.json guard in print_state.py In DMS/direct mode only resources.json exists among the state files, so the filename check added nothing; the records_deployment_history() + --no-dms gate alone selects the recorded-state path. Co-authored-by: Isaac --- acceptance/bin/print_state.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/acceptance/bin/print_state.py b/acceptance/bin/print_state.py index 351c3083396..6055a3c5ecf 100755 --- a/acceptance/bin/print_state.py +++ b/acceptance/bin/print_state.py @@ -87,7 +87,7 @@ def main(): for filename in get_state_files(args.target, args.backup): if not os.path.exists(filename): continue - if filename.endswith("resources.json") and records_deployment_history() and not args.no_dms: + if records_deployment_history() and not args.no_dms: print_recorded_state(filename, args.target) else: print_file(filename) From 41b1adfc98b6e8ce904752f496ae500f1e0922b0 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 7 Sep 2026 02:04:52 +0200 Subject: [PATCH 31/46] acceptance/dms: assert failed delete records its error via list-operations Add a ListOperations handler to the testserver (GET /api/2.0/bundle/deployments/{id}/versions/{version}/operations) mirroring the service, and call bundle-deployments list-operations in failed-delete to assert the failed delete operation carries its error message and FAILED status. Co-authored-by: Isaac --- .../bundle/dms/failed-delete/output.txt | 11 ++++++ acceptance/bundle/dms/failed-delete/script | 3 ++ libs/testserver/bundledeployments.go | 37 +++++++++++++++++++ libs/testserver/handlers.go | 3 ++ 4 files changed, 54 insertions(+) diff --git a/acceptance/bundle/dms/failed-delete/output.txt b/acceptance/bundle/dms/failed-delete/output.txt index 84e0bc5617f..2d06337ff31 100644 --- a/acceptance/bundle/dms/failed-delete/output.txt +++ b/acceptance/bundle/dms/failed-delete/output.txt @@ -214,6 +214,17 @@ API message: Fault injected by test. } ] +=== The failed operation records the error message +>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-operations deployments/[NUMID]/versions/2 -o json +[ + { + "action_type": "OPERATION_ACTION_TYPE_DELETE", + "error_message": "deleting id=[NUMID]: Fault injected by test. (400 INVALID_PARAMETER_VALUE)", + "resource_key": "jobs.stuck", + "status": "OPERATION_STATUS_FAILED" + } +] + === The state shows the resource still exists but the operation is marked failed >>> print_state.py --no-dms { diff --git a/acceptance/bundle/dms/failed-delete/script b/acceptance/bundle/dms/failed-delete/script index d7e06ace12d..08d949fd198 100644 --- a/acceptance/bundle/dms/failed-delete/script +++ b/acceptance/bundle/dms/failed-delete/script @@ -24,6 +24,9 @@ title "After failure, the service shows the operation failed" deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-failed-delete-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' +title "The failed operation records the error message" +trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-operations "deployments/${deployment_id}/versions/2" -o json | jq -S 'map({resource_key, action_type, status, error_message})' + title "The state shows the resource still exists but the operation is marked failed" trace print_state.py --no-dms diff --git a/libs/testserver/bundledeployments.go b/libs/testserver/bundledeployments.go index 3545036297a..4fccd96c700 100644 --- a/libs/testserver/bundledeployments.go +++ b/libs/testserver/bundledeployments.go @@ -569,6 +569,43 @@ func (s *FakeWorkspace) ListResources(deploymentID string) Response { return Response{Body: bundledeployments.ListResourcesResponse{Resources: resources}} } +func (s *FakeWorkspace) ListOperations(deploymentID, versionID string) Response { + defer s.LockUnlock()() + + d, ok := s.DmsDeployments[deploymentID] + if !ok { + return dmsNotFound("deployment " + deploymentID) + } + + _, ok = d.Versions[versionID] + if !ok { + return dmsNotFound("version " + versionID) + } + + // Collect operations for this version by filtering the operation name prefix. + // Operation names are: deployments/{deployment_id}/versions/{version_id}/operations/{resource_key} + versionPrefix := "deployments/" + deploymentID + "/versions/" + versionID + "/operations/" + var ops []*bundledeployments.Operation + for name, op := range d.Operations { + if strings.HasPrefix(name, versionPrefix) { + ops = append(ops, op) + } + } + + // Sort by resource key so the response order is deterministic. + slices.SortFunc(ops, func(a, b *bundledeployments.Operation) int { + return strings.Compare(a.ResourceKey, b.ResourceKey) + }) + + response := bundledeployments.ListOperationsResponse{ + Operations: make([]bundledeployments.Operation, len(ops)), + } + for i, op := range ops { + response.Operations[i] = *op + } + return Response{Body: response} +} + // leavesLiveResource reports whether a failed operation of this type leaves a resource behind // that it still has to describe. A create leaves nothing, and a recreate already deleted. func leavesLiveResource(action bundledeployments.OperationActionType) bool { diff --git a/libs/testserver/handlers.go b/libs/testserver/handlers.go index 33a742cc2ce..9fe87c64188 100644 --- a/libs/testserver/handlers.go +++ b/libs/testserver/handlers.go @@ -307,6 +307,9 @@ func AddDefaultHandlers(server *Server) { server.Handle("GET", "/api/2.0/bundle/deployments/{deployment_id}/resources", func(req Request) any { return req.Workspace.ListResources(req.Vars["deployment_id"]) }) + server.Handle("GET", "/api/2.0/bundle/deployments/{deployment_id}/versions/{version_id}/operations", func(req Request) any { + return req.Workspace.ListOperations(req.Vars["deployment_id"], req.Vars["version_id"]) + }) server.Handle("POST", "/api/2.2/jobs/delete", func(req Request) any { var request jobs.DeleteJob From 4563cbc0f81405c9aa34ec65fddd73fbad3d66bb Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 7 Sep 2026 02:25:48 +0200 Subject: [PATCH 32/46] bundle/direct/dstate: restore original migrateState feature-version comment Reverts a comment rewording; the longer original phrasing is clearer about why a featureStateVersion state with no features is accepted without running the migrations and its on-disk version left as-is. Co-authored-by: Isaac --- bundle/direct/dstate/migrate.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/bundle/direct/dstate/migrate.go b/bundle/direct/dstate/migrate.go index 6a891900c36..e4d21a7054a 100644 --- a/bundle/direct/dstate/migrate.go +++ b/bundle/direct/dstate/migrate.go @@ -15,10 +15,12 @@ import ( // migrateState runs all necessary migrations on the database. // It is called after loading state from disk. func migrateState(db *Database) error { - // A featureStateVersion state carries a feature list this CLI does not write (see the - // featureStateVersion doc comment). With no features it is equivalent to currentStateVersion, so - // accept it and leave the on-disk version as-is; with any feature it depends on capabilities this - // CLI lacks, so refuse it and tell the user to upgrade. + // featureStateVersion states carry a feature list this CLI does not yet write or + // understand (see the featureStateVersion doc comment). A featureStateVersion + // state with no features is equivalent to currentStateVersion, so accept it and + // return without running the migrations below, leaving the on-disk version at + // featureStateVersion rather than flipping it down. One that records any feature + // depends on capabilities this CLI lacks, so refuse it and tell the user to upgrade. if db.StateVersion == featureStateVersion { if len(db.Features) == 0 { return nil From 5435321e46d18f5ee52b9da69a869abfe8b3f1b8 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 7 Sep 2026 02:26:01 +0200 Subject: [PATCH 33/46] acceptance/dms: failed-recreate fails naturally via nonexistent catalog (cloud-enabled) Drop the fault.py injection; point the recreated schema at a catalog that does not exist so the create half of the recreate fails on its own. The testserver now models catalog existence: SchemasCreate returns 404 CATALOG_DOES_NOT_EXIST when the parent catalog is absent, and the fake seeds the "main" catalog so the common case (schemas in main) keeps working. This makes the failure identical locally and on a real workspace, so the test runs with Cloud = true. Co-authored-by: Isaac --- .../bundle/dms/failed-recreate/out.requests.txt | 7 ------- .../bundle/dms/failed-recreate/out.test.toml | 2 +- acceptance/bundle/dms/failed-recreate/output.txt | 16 +++++++--------- acceptance/bundle/dms/failed-recreate/script | 10 +++------- acceptance/bundle/dms/failed-recreate/test.toml | 9 ++++----- libs/testserver/catalogs_test.go | 7 ++++--- libs/testserver/fake_workspace.go | 7 ++++++- libs/testserver/schemas.go | 12 ++++++++++++ 8 files changed, 37 insertions(+), 33 deletions(-) delete mode 100644 acceptance/bundle/dms/failed-recreate/out.requests.txt diff --git a/acceptance/bundle/dms/failed-recreate/out.requests.txt b/acceptance/bundle/dms/failed-recreate/out.requests.txt deleted file mode 100644 index eb169f26a0c..00000000000 --- a/acceptance/bundle/dms/failed-recreate/out.requests.txt +++ /dev/null @@ -1,7 +0,0 @@ -{ - "method": "DELETE", - "path": "/api/2.1/unity-catalog/catalogs/dms_other_[UNIQUE_NAME]", - "q": { - "force": "true" - } -} diff --git a/acceptance/bundle/dms/failed-recreate/out.test.toml b/acceptance/bundle/dms/failed-recreate/out.test.toml index 9921e91a794..23c07f70dca 100644 --- a/acceptance/bundle/dms/failed-recreate/out.test.toml +++ b/acceptance/bundle/dms/failed-recreate/out.test.toml @@ -1,3 +1,3 @@ -Cloud = false +Cloud = true EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] EnvMatrix.DMS = ["true"] diff --git a/acceptance/bundle/dms/failed-recreate/output.txt b/acceptance/bundle/dms/failed-recreate/output.txt index 78bcaa59776..a4a818bf7bc 100644 --- a/acceptance/bundle/dms/failed-recreate/output.txt +++ b/acceptance/bundle/dms/failed-recreate/output.txt @@ -122,23 +122,21 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged } === A recreate deletes and then fails to create. The delete went through, so the resource really is gone: the failed operation clears its state (self-contained, not assuming the delete's own write reached the service), rather than describing the resource as it was before the deploy ->>> update_file.py databricks.yml catalog_name: main catalog_name: dms_other_[UNIQUE_NAME] - ->>> fault.py POST /api/2.1/unity-catalog/schemas 400 0 1 INVALID_PARAMETER_VALUE +>>> update_file.py databricks.yml catalog_name: main catalog_name: dms_nonexistent_[UNIQUE_NAME] >>> musterr [CLI] bundle deploy --auto-approve Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-failed-recreate-[UNIQUE_NAME]/default/files... This action will result in the deletion or recreation of the following UC schemas. Any underlying data may be lost: recreate resources.schemas.foo -Error: cannot recreate resources.schemas.foo: Fault injected by test. (400 INVALID_PARAMETER_VALUE) +Error: cannot recreate resources.schemas.foo: Catalog 'dms_nonexistent_[UNIQUE_NAME]' does not exist. (404 CATALOG_DOES_NOT_EXIST) Endpoint: POST [DATABRICKS_URL]/api/2.1/unity-catalog/schemas -HTTP Status: 400 Bad Request -API error_code: INVALID_PARAMETER_VALUE -API message: Fault injected by test. +HTTP Status: 404 Not Found +API error_code: CATALOG_DOES_NOT_EXIST +API message: Catalog 'dms_nonexistent_[UNIQUE_NAME]' does not exist. -Files: 4 uploaded, 0 deleted +Files: 3 uploaded, 0 deleted >>> print_requests.py --dms //api/2.0/bundle { @@ -178,7 +176,7 @@ Files: 4 uploaded, 0 deleted "update_mask": "state,error_message,status" }, "body": { - "error_message": "Fault injected by test. (400 INVALID_PARAMETER_VALUE)", + "error_message": "Catalog 'dms_nonexistent_[UNIQUE_NAME]' does not exist. (404 CATALOG_DOES_NOT_EXIST)", "sequence_id": "1", "status": "OPERATION_STATUS_FAILED" } diff --git a/acceptance/bundle/dms/failed-recreate/script b/acceptance/bundle/dms/failed-recreate/script index 976044bdba9..8d0b7ddeef1 100644 --- a/acceptance/bundle/dms/failed-recreate/script +++ b/acceptance/bundle/dms/failed-recreate/script @@ -15,13 +15,9 @@ title "The raw state after successful deploy" trace print_state.py --no-dms title "A recreate deletes and then fails to create. The delete went through, so the resource really is gone: the failed operation clears its state (self-contained, not assuming the delete's own write reached the service), rather than describing the resource as it was before the deploy" -# Unity Catalog has no catalog called "other", so make one to move the schema into. -other_catalog="dms_other_${UNIQUE_NAME}" -$CLI catalogs create "$other_catalog" &> LOG.catalog -trap '$CLI catalogs delete "$other_catalog" --force >> LOG.catalog 2>&1' EXIT -trace update_file.py databricks.yml "catalog_name: main" "catalog_name: ${other_catalog}" -# Fail only the create that follows the delete; the delete itself still goes through. -trace fault.py "POST /api/2.1/unity-catalog/schemas" 400 0 1 INVALID_PARAMETER_VALUE +# Point the schema at a catalog that does not exist: the delete goes through, then the +# create fails naturally against both the fake and a real workspace (no fault injection). +trace update_file.py databricks.yml "catalog_name: main" "catalog_name: dms_nonexistent_${UNIQUE_NAME}" trace musterr $CLI bundle deploy --auto-approve # Not sorted: the point of this test is the order the operations are recorded in. trace print_requests.py --dms //api/2.0/bundle diff --git a/acceptance/bundle/dms/failed-recreate/test.toml b/acceptance/bundle/dms/failed-recreate/test.toml index 8a850ba07c5..25c802e47a0 100644 --- a/acceptance/bundle/dms/failed-recreate/test.toml +++ b/acceptance/bundle/dms/failed-recreate/test.toml @@ -1,5 +1,4 @@ -# Local only: the failure is injected, and fault.py registers its rule on the fake, which -# a cloud run does not have. Simulating it for real instead - a catalog or principal that -# does not exist - does not work either: the fake accepts both, so the case would stop -# failing locally. Making it portable needs the fake to model that existence. -Cloud = false +# The recreate fails naturally by pointing the schema at a catalog that does not +# exist - the fake models catalog existence, so this fails the same way locally and +# on cloud without fault injection. +Cloud = true diff --git a/libs/testserver/catalogs_test.go b/libs/testserver/catalogs_test.go index eb05ab8ec56..b699361577c 100644 --- a/libs/testserver/catalogs_test.go +++ b/libs/testserver/catalogs_test.go @@ -17,9 +17,10 @@ func TestCatalogsCreate_RejectsEmptyName(t *testing.T) { response := workspace.CatalogsCreate(Request{Body: []byte(`{"name": ""}`)}) assert.Equal(t, 400, response.StatusCode) - // A stored-but-unreadable catalog is the original bug. Asserted before the - // require below so it is still reported when the rejection is missing. - assert.Empty(t, workspace.Catalogs) + // A stored-but-unreadable catalog under the empty key is the original bug. Asserted + // before the require below so it is still reported when the rejection is missing. + // (The workspace seeds "main", so the map is not empty; the empty key must be absent.) + assert.NotContains(t, workspace.Catalogs, "") body, ok := response.Body.(map[string]string) require.True(t, ok) diff --git a/libs/testserver/fake_workspace.go b/libs/testserver/fake_workspace.go index b544c17538e..c9026e20fc2 100644 --- a/libs/testserver/fake_workspace.go +++ b/libs/testserver/fake_workspace.go @@ -467,7 +467,12 @@ func NewFakeWorkspace(url, token string) *FakeWorkspace { PipelineUpdates: map[string]bool{}, Monitors: map[string]catalog.MonitorInfo{}, Apps: map[string]apps.App{}, - Catalogs: map[string]catalog.CatalogInfo{}, + // Seed the "main" catalog: it exists on every real workspace, and schema + // creation now checks the catalog exists, so tests that deploy schemas into + // main (the common case) keep working without creating it first. + Catalogs: map[string]catalog.CatalogInfo{ + "main": {Name: "main", FullName: "main", CatalogType: catalog.CatalogTypeManagedCatalog}, + }, ExternalLocations: map[string]catalog.ExternalLocationInfo{}, Schemas: map[string]catalog.SchemaInfo{}, RegisteredModels: map[string]catalog.RegisteredModelInfo{}, diff --git a/libs/testserver/schemas.go b/libs/testserver/schemas.go index 6880a55cb8f..92bc0374734 100644 --- a/libs/testserver/schemas.go +++ b/libs/testserver/schemas.go @@ -29,6 +29,18 @@ func (s *FakeWorkspace) SchemasCreate(req Request) Response { } } + // UC rejects a schema whose parent catalog does not exist; model it so a deploy into a + // missing catalog fails the same way locally as on cloud. + if _, ok := s.Catalogs[schema.CatalogName]; !ok { + return Response{ + StatusCode: http.StatusNotFound, + Body: map[string]string{ + "error_code": "CATALOG_DOES_NOT_EXIST", + "message": fmt.Sprintf("Catalog '%s' does not exist.", schema.CatalogName), + }, + } + } + // UC normalizes schema names to lowercase. schema.Name = strings.ToLower(schema.Name) schema.FullName = schema.CatalogName + "." + schema.Name From ae90e569c1e9daad543fdb8bd1930f144131c0c9 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 7 Sep 2026 02:32:37 +0200 Subject: [PATCH 34/46] bundle/direct/dstate: restore TestEmptyFeatureStateAcceptedWithoutFlippingVersion Re-add the test pinning the featureStateVersion special case: a v3 state with no features is accepted and left at v3 (not flipped to currentStateVersion), while a v3 state recording any feature is refused with the upgrade hint. It is the forcing function for the deferred version bump. Co-authored-by: Isaac --- bundle/direct/dstate/state_test.go | 32 ++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/bundle/direct/dstate/state_test.go b/bundle/direct/dstate/state_test.go index f7d605e39f8..57ccd4d3e51 100644 --- a/bundle/direct/dstate/state_test.go +++ b/bundle/direct/dstate/state_test.go @@ -182,6 +182,38 @@ func TestHeaderOnlyWALRecoveryDoesNotAdvanceSerial(t *testing.T) { mustFinalize(t, &recovered) } +// TestEmptyFeatureStateAcceptedWithoutFlippingVersion pins the special case that a +// featureStateVersion state with no features is accepted as-is — the on-disk version +// is left at featureStateVersion, not flipped down to currentStateVersion — and that +// a featureStateVersion state recording any feature is refused. This is scaffolding +// for the deferred version bump, special-cased to featureStateVersion only (see the +// featureStateVersion doc comment). +// +// When the baseline is actually bumped to featureStateVersion, this special case must +// go away. This test is the forcing function: it fails once featureStateVersion is +// removed, making the author decide what the post-bump behavior should be. +func TestEmptyFeatureStateAcceptedWithoutFlippingVersion(t *testing.T) { + // The special case applies to featureStateVersion (3) only. + require.Equal(t, 2, currentStateVersion, "when currentStateVersion is bumped, remove featureStateVersion and this special case") + require.Equal(t, 3, featureStateVersion) + + empty := &Database{Header: Header{StateVersion: featureStateVersion}} + require.NoError(t, migrateState(empty)) + assert.Equal(t, featureStateVersion, empty.StateVersion, "v3 + no features keeps its on-disk version, not flipped to v2") + + // v3 that records a feature is refused: this CLI does not understand features. + withFeature := &Database{Header: Header{ + StateVersion: featureStateVersion, + Features: map[string]struct{}{"future_feature": {}}, + }} + err := migrateState(withFeature) + require.Error(t, err) + assert.Contains(t, err.Error(), "requires features this CLI does not support") + assert.Contains(t, err.Error(), "future_feature") + assert.Contains(t, err.Error(), "upgrade to the latest CLI version") + assert.Contains(t, err.Error(), featuresDocURL) +} + func TestDeleteState(t *testing.T) { path := filepath.Join(t.TempDir(), "state.json") From 6a21a0eda4ae1b8723a5ef495a03137d3900d582 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 7 Sep 2026 02:35:36 +0200 Subject: [PATCH 35/46] bundle/direct/dstate: use lock/defer-unlock in RecordingError RecordingError only reads operationBuffer and calls buf.Err() (a quick read on the buffer's own mutex), so holding db.mu across it via defer is safe and reads cleaner. CompleteVersion keeps its manual unlock: the network work after the snapshot must run outside the lock. Co-authored-by: Isaac --- bundle/direct/dstate/state.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/bundle/direct/dstate/state.go b/bundle/direct/dstate/state.go index f43503c7a05..6a41661c516 100644 --- a/bundle/direct/dstate/state.go +++ b/bundle/direct/dstate/state.go @@ -208,12 +208,11 @@ func (db *DeploymentState) RecordingError() error { return nil } db.mu.Lock() - buf := db.operationBuffer - db.mu.Unlock() - if buf == nil { + defer db.mu.Unlock() + if db.operationBuffer == nil { return nil } - return buf.Err() + return db.operationBuffer.Err() } // CompleteVersion marks the recorded version done, reporting whether it completed here. A no-op From 980659959d078b455ea922fcf4bbeaa59ef1f96d Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 7 Sep 2026 02:42:07 +0200 Subject: [PATCH 36/46] acceptance/dms: cover bind/unbind refusal while recording deployment history New test showing bind and unbind against a bundle that records deployment history. Before any deploy there is no recorded state, so bind just fails to find the workspace resource and unbind is a no-op. Once a deployment is recorded, both are refused ("bind/unbind is not supported for a bundle that records deployment history"): the service owns the resource set, so adopting or releasing a resource by editing local state is not allowed. Local-only, since the refusal is a client-side gate. Co-authored-by: Isaac --- .../databricks.yml.tmpl | 9 ++++++++ .../dms/bind-unbind-unsupported/out.test.toml | 3 +++ .../dms/bind-unbind-unsupported/output.txt | 22 +++++++++++++++++++ .../bundle/dms/bind-unbind-unsupported/script | 18 +++++++++++++++ .../dms/bind-unbind-unsupported/test.toml | 3 +++ 5 files changed, 55 insertions(+) create mode 100644 acceptance/bundle/dms/bind-unbind-unsupported/databricks.yml.tmpl create mode 100644 acceptance/bundle/dms/bind-unbind-unsupported/out.test.toml create mode 100644 acceptance/bundle/dms/bind-unbind-unsupported/output.txt create mode 100644 acceptance/bundle/dms/bind-unbind-unsupported/script create mode 100644 acceptance/bundle/dms/bind-unbind-unsupported/test.toml diff --git a/acceptance/bundle/dms/bind-unbind-unsupported/databricks.yml.tmpl b/acceptance/bundle/dms/bind-unbind-unsupported/databricks.yml.tmpl new file mode 100644 index 00000000000..7c9830e927c --- /dev/null +++ b/acceptance/bundle/dms/bind-unbind-unsupported/databricks.yml.tmpl @@ -0,0 +1,9 @@ +bundle: + name: dms-bind-unbind-$UNIQUE_NAME +experimental: + record_deployment_history: true + +resources: + jobs: + foo: + name: foo diff --git a/acceptance/bundle/dms/bind-unbind-unsupported/out.test.toml b/acceptance/bundle/dms/bind-unbind-unsupported/out.test.toml new file mode 100644 index 00000000000..9921e91a794 --- /dev/null +++ b/acceptance/bundle/dms/bind-unbind-unsupported/out.test.toml @@ -0,0 +1,3 @@ +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.DMS = ["true"] diff --git a/acceptance/bundle/dms/bind-unbind-unsupported/output.txt b/acceptance/bundle/dms/bind-unbind-unsupported/output.txt new file mode 100644 index 00000000000..484998ca590 --- /dev/null +++ b/acceptance/bundle/dms/bind-unbind-unsupported/output.txt @@ -0,0 +1,22 @@ + +=== Before any deploy there is no recorded state: bind just fails to find the workspace resource, and unbind is a no-op +>>> musterr [CLI] bundle deployment bind foo 123 +Error: failed to fetch the resource, err: Job 123 does not exist. + +>>> [CLI] bundle deployment unbind foo + +=== Deploy so the bundle records the job with the deployment metadata service +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-bind-unbind-[UNIQUE_NAME]/default/files... +Created jobs.foo +Files: 6 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +=== With deployment history recorded, both bind and unbind are refused +>>> musterr [CLI] bundle deployment bind foo [FOO_ID] +Error: bind is not supported for a bundle that records deployment history + + +>>> musterr [CLI] bundle deployment unbind foo +Error: unbind is not supported for a bundle that records deployment history + diff --git a/acceptance/bundle/dms/bind-unbind-unsupported/script b/acceptance/bundle/dms/bind-unbind-unsupported/script new file mode 100644 index 00000000000..0bed434163a --- /dev/null +++ b/acceptance/bundle/dms/bind-unbind-unsupported/script @@ -0,0 +1,18 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +title "Before any deploy there is no recorded state: bind just fails to find the workspace resource, and unbind is a no-op" +trace musterr $CLI bundle deployment bind foo 123 +trace $CLI bundle deployment unbind foo + +title "Deploy so the bundle records the job with the deployment metadata service" +trace $CLI bundle deploy + +# bind and unbind adopt or release a resource by editing local state directly. A recorded +# deployment keeps its resource set in the service, so both are refused: bind hits the gate +# after fetching the resource, unbind when it opens the recording state without recording. +title "With deployment history recorded, both bind and unbind are refused" +job_id=$(read_id.py foo) +trace musterr $CLI bundle deployment bind foo "$job_id" +trace musterr $CLI bundle deployment unbind foo + +rm -f out.requests.txt diff --git a/acceptance/bundle/dms/bind-unbind-unsupported/test.toml b/acceptance/bundle/dms/bind-unbind-unsupported/test.toml new file mode 100644 index 00000000000..8bc2d5e8cf8 --- /dev/null +++ b/acceptance/bundle/dms/bind-unbind-unsupported/test.toml @@ -0,0 +1,3 @@ +# bind/unbind refusal is a client-side gate on the recording config; it needs no real +# workspace, so this stays local-only rather than inheriting the suite's cloud run. +Cloud = false From 8d65ab2523d3bd75e7c34ed46445502f72fbe9f4 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 7 Sep 2026 02:53:27 +0200 Subject: [PATCH 37/46] libs/dms: fold latestState update into record under one lock RecordOperation and RecordFailure each took db.mu separately to touch latestState and then again inside record to touch pending, so the two maps could be observed mid-update. Move the latestState read/write into record, guarded by the same lock as the pending update: a state write (isStateWrite) records the latest state, a failure merges its status/message onto that recorded state (or falls back to the passed update). RecordOperation and RecordFailure no longer lock directly. Co-authored-by: Isaac --- libs/dms/operation_buffer.go | 36 +++++++++++++++---------------- libs/dms/operation_buffer_test.go | 6 +++--- 2 files changed, 20 insertions(+), 22 deletions(-) diff --git a/libs/dms/operation_buffer.go b/libs/dms/operation_buffer.go index ac5cab63dd8..11fe07fc471 100644 --- a/libs/dms/operation_buffer.go +++ b/libs/dms/operation_buffer.go @@ -76,13 +76,7 @@ func (b *OperationBuffer) RecordOperation(ctx context.Context, resourceKey strin return } - // Remember the state this write recorded, kept past the writer taking it, so a later failure - // for this resource keeps this state rather than the pre-deploy one (see RecordFailure). - b.mu.Lock() - b.latestState[resourceKey] = update - b.mu.Unlock() - - b.record(resourceKey, update) + b.record(resourceKey, update, true, nil) } // RecordFailure records that a resource did not apply, so the history says why rather than @@ -92,22 +86,26 @@ func (b *OperationBuffer) RecordOperation(ctx context.Context, resourceKey strin // pair the new id with the old state. state is the fallback, used only when no write recorded // anything: the operation failed before it saved state. func (b *OperationBuffer) RecordFailure(resourceKey, resourceID string, state json.RawMessage, cause error) { - b.mu.Lock() - latest, recorded := b.latestState[resourceKey] - b.mu.Unlock() - - if recorded { - b.record(resourceKey, latest.Merge(failureUpdate(cause))) - return - } - - b.record(resourceKey, NewFailureUpdate(resourceID, state, cause)) + // The passed update is the fallback used only when no write recorded state yet; record + // otherwise merges the failure onto the state the last write recorded (see below). + b.record(resourceKey, NewFailureUpdate(resourceID, state, cause), false, cause) } // record makes update the one waiting for resourceKey, waiting itself while the queue is -// full. Recording after Drain panics, so every caller must return before Drain. -func (b *OperationBuffer) record(resourceKey string, update OperationUpdate) { +// full. isStateWrite records update as the latest state for this resource, kept past the +// writer taking it from pending so a later failure keeps it (see the latestState field); +// a failure (isStateWrite false) instead merges its status and message onto that recorded +// state when there is one, so it keeps the state - which the service may already hold - +// rather than the pre-deploy one, and after a rename pairs the new id with the old state. +// The latestState read/write shares this lock with the pending update so the two never +// disagree. Recording after Drain panics, so every caller must return before Drain. +func (b *OperationBuffer) record(resourceKey string, update OperationUpdate, isStateWrite bool, cause error) { b.mu.Lock() + if isStateWrite { + b.latestState[resourceKey] = update + } else if latest, recorded := b.latestState[resourceKey]; recorded { + update = latest.Merge(failureUpdate(cause)) + } waiting, queued := b.pending[resourceKey] if queued { update = waiting.Merge(update) diff --git a/libs/dms/operation_buffer_test.go b/libs/dms/operation_buffer_test.go index b4fba8ce124..cb01c8b57d5 100644 --- a/libs/dms/operation_buffer_test.go +++ b/libs/dms/operation_buffer_test.go @@ -38,8 +38,8 @@ func TestBufferCoalescesWhileAKeyIsPending(t *testing.T) { // Two writes for one resource with nothing draining. They carry the resource's full // state, so only the newest needs to go: one slot in the queue, not two. - s.record("resources.jobs.foo", stateUpdate(t, "v1")) - s.record("resources.jobs.foo", stateUpdate(t, "v2")) + s.record("resources.jobs.foo", stateUpdate(t, "v1"), true, nil) + s.record("resources.jobs.foo", stateUpdate(t, "v2"), true, nil) assert.Len(t, s.queue, 1) update, ok := s.take("resources.jobs.foo") @@ -48,7 +48,7 @@ func TestBufferCoalescesWhileAKeyIsPending(t *testing.T) { // Taken means a request has it, and an in-flight request cannot be recalled, so the next // write gets its own slot rather than joining it. - s.record("resources.jobs.foo", stateUpdate(t, "v3")) + s.record("resources.jobs.foo", stateUpdate(t, "v3"), true, nil) assert.Len(t, s.queue, 2) update, ok = s.take("resources.jobs.foo") require.True(t, ok) From c0a2e503db09a03f10986a537e970b8656bb33c5 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 7 Sep 2026 02:54:05 +0200 Subject: [PATCH 38/46] acceptance/dms: print plan header via jq + readable plan instead of full -o json MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a dms_plan helper (bundle/dms/script.prepare) that prints the plan JSON with the per-resource plan body dropped (jq del(.plan)) — leaving the header fields cli_version, lineage, serial, deployment_id, next/last_version_id, storage_backend — then runs bundle plan again for the readable summary. Replace every full `bundle plan -o json` dump in the bundle/dms tests with it. The per-resource plan carried remote_state that diverges between the fake and a real workspace (e.g. a serverless pipeline's effective_serverless_compute_id), which broke no-drift on cloud; dropping it keeps local and cloud identical while still asserting the DMS header fields and the human-readable plan. Co-authored-by: Isaac --- .../bundle/dms/declined-deploy/output.txt | 38 +- acceptance/bundle/dms/declined-deploy/script | 2 +- acceptance/bundle/dms/depends-on/output.txt | 170 +----- acceptance/bundle/dms/depends-on/script | 4 +- .../dms/deployment-metadata-change/output.txt | 66 +-- .../dms/deployment-metadata-change/script | 4 +- .../bundle/dms/emptied-resource/output.txt | 30 +- acceptance/bundle/dms/emptied-resource/script | 2 +- .../bundle/dms/existing-state/output.txt | 47 +- acceptance/bundle/dms/existing-state/script | 2 +- .../bundle/dms/failed-delete/output.txt | 56 +- acceptance/bundle/dms/failed-delete/script | 2 +- .../bundle/dms/failed-recreate/output.txt | 31 +- acceptance/bundle/dms/failed-recreate/script | 2 +- .../dms/failed-update-permissions/output.txt | 78 +-- .../dms/failed-update-permissions/script | 2 +- .../bundle/dms/failed-update/output.txt | 31 +- acceptance/bundle/dms/failed-update/script | 2 +- .../bundle/dms/multiple-resources/output.txt | 504 +----------------- .../bundle/dms/multiple-resources/script | 4 +- acceptance/bundle/dms/no-drift/output.txt | 193 +------ acceptance/bundle/dms/no-drift/script | 4 +- acceptance/bundle/dms/no-resources/output.txt | 20 +- acceptance/bundle/dms/no-resources/script | 4 +- .../operation-upload-fails-once/output.txt | 31 +- .../dms/operation-upload-fails-once/script | 2 +- acceptance/bundle/dms/provenance/output.txt | 59 +- acceptance/bundle/dms/provenance/script | 2 +- .../bundle/dms/record-failure/output.txt | 42 +- acceptance/bundle/dms/record-failure/script | 2 +- acceptance/bundle/dms/record/output.txt | 56 +- acceptance/bundle/dms/record/script | 2 +- acceptance/bundle/dms/script.prepare | 9 + acceptance/bundle/dms/stale-plan/output.txt | 28 +- acceptance/bundle/dms/stale-plan/script | 2 +- .../bundle/dms/successful-recreate/output.txt | 62 +-- .../bundle/dms/successful-recreate/script | 4 +- .../dms/version-never-created/output.txt | 31 +- .../bundle/dms/version-never-created/script | 2 +- 39 files changed, 167 insertions(+), 1465 deletions(-) create mode 100644 acceptance/bundle/dms/script.prepare diff --git a/acceptance/bundle/dms/declined-deploy/output.txt b/acceptance/bundle/dms/declined-deploy/output.txt index a4d25045708..2aa56f9ab78 100644 --- a/acceptance/bundle/dms/declined-deploy/output.txt +++ b/acceptance/bundle/dms/declined-deploy/output.txt @@ -69,34 +69,12 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "deployment_id": "[NUMID]", "next_version_id": "2", "last_version_id": "1", - "storage_backend": "DEPLOYMENT_METADATA_SERVICE", - "plan": { - "resources.schemas.foo": { - "action": "skip", - "remote_state": { - "browse_only": false, - "catalog_name": "main", - "catalog_type": "MANAGED_CATALOG", - "created_at": [UNIX_TIME_MILLIS][0], - "created_by": "[USERNAME]", - "effective_predictive_optimization_flag": { - "inherited_from_name": "[METASTORE_NAME]", - "inherited_from_type": "METASTORE", - "value": "ENABLE" - }, - "enable_predictive_optimization": "INHERIT", - "full_name": "main.dms_declined_deploy_schema_[UNIQUE_NAME]", - "metastore_id": "[UUID]", - "name": "dms_declined_deploy_schema_[UNIQUE_NAME]", - "owner": "[USERNAME]", - "schema_id": "[UUID]", - "updated_at": [UNIX_TIME_MILLIS][0], - "updated_by": "[USERNAME]" - } - } - } + "storage_backend": "DEPLOYMENT_METADATA_SERVICE" } +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged + >>> print_requests.py --dms --get //api/2.0/bundle { "method": "GET", @@ -106,6 +84,14 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "method": "GET", "path": "/api/2.0/bundle/deployments/[NUMID]/resources" } +{ + "method": "GET", + "path": "/api/2.0/bundle/deployments/[NUMID]" +} +{ + "method": "GET", + "path": "/api/2.0/bundle/deployments/[NUMID]/resources" +} >>> print_state.py --no-dms { diff --git a/acceptance/bundle/dms/declined-deploy/script b/acceptance/bundle/dms/declined-deploy/script index c51686ba386..56a33fea036 100644 --- a/acceptance/bundle/dms/declined-deploy/script +++ b/acceptance/bundle/dms/declined-deploy/script @@ -5,7 +5,7 @@ trace $CLI bundle deploy trace print_requests.py --dms --get //api/2.0/bundle title "The plan is a no-op now and records DEPLOYMENT_METADATA_SERVICE; the service holds the schema at version 1" -trace $CLI bundle plan -o json +dms_plan trace print_requests.py --dms --get //api/2.0/bundle trace print_state.py --no-dms diff --git a/acceptance/bundle/dms/depends-on/output.txt b/acceptance/bundle/dms/depends-on/output.txt index 4640a2738f4..ae9e3cda49e 100644 --- a/acceptance/bundle/dms/depends-on/output.txt +++ b/acceptance/bundle/dms/depends-on/output.txt @@ -4,59 +4,15 @@ { "plan_version": 3, "cli_version": "[CLI_VERSION]", - "storage_backend": "DEPLOYMENT_METADATA_SERVICE", - "plan": { - "resources.jobs.child": { - "depends_on": [ - { - "node": "resources.jobs.parent", - "label": "${resources.jobs.parent.id}" - } - ], - "action": "create", - "new_state": { - "value": { - "deployment": { - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-depends-on-[UNIQUE_NAME]/default/state/metadata.json", - "version_id": "1" - }, - "description": "depends on ${resources.jobs.parent.id}", - "edit_mode": "UI_LOCKED", - "format": "MULTI_TASK", - "max_concurrent_runs": 1, - "name": "child", - "queue": { - "enabled": true - } - }, - "vars": { - "description": "depends on ${resources.jobs.parent.id}" - } - } - }, - "resources.jobs.parent": { - "action": "create", - "new_state": { - "value": { - "deployment": { - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-depends-on-[UNIQUE_NAME]/default/state/metadata.json", - "version_id": "1" - }, - "edit_mode": "UI_LOCKED", - "format": "MULTI_TASK", - "max_concurrent_runs": 1, - "name": "parent", - "queue": { - "enabled": true - } - } - } - } - } + "storage_backend": "DEPLOYMENT_METADATA_SERVICE" } +>>> [CLI] bundle plan +create jobs.child +create jobs.parent + +Plan: 2 to add, 0 to change, 0 to delete, 0 unchanged + === Deploy a job that references another: depends_on is recorded alongside the config, since the reference is resolved to a literal in the config itself >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-depends-on-[UNIQUE_NAME]/default/files... @@ -105,116 +61,12 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged "deployment_id": "[NUMID]", "next_version_id": "2", "last_version_id": "1", - "storage_backend": "DEPLOYMENT_METADATA_SERVICE", - "plan": { - "resources.jobs.child": { - "depends_on": [ - { - "node": "resources.jobs.parent", - "label": "${resources.jobs.parent.id}" - } - ], - "action": "skip", - "remote_state": { - "created_time": [UNIX_TIME_MILLIS][0], - "creator_user_name": "[USERNAME]", - "deployment": { - "deployment_id": "[NUMID]", - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-depends-on-[UNIQUE_NAME]/default/state/metadata.json", - "version_id": "1" - }, - "description": "depends on [NUMID]", - "edit_mode": "UI_LOCKED", - "email_notifications": {}, - "format": "MULTI_TASK", - "job_id": [NUMID], - "max_concurrent_runs": 1, - "name": "child", - "queue": { - "enabled": true - }, - "run_as_user_name": "[USERNAME]", - "timeout_seconds": 0, - "webhook_notifications": {} - }, - "changes": { - "deployment.version_id": { - "action": "skip", - "reason": "auto", - "old": "1", - "new": "2", - "remote": "1" - }, - "email_notifications": { - "action": "skip", - "reason": "empty", - "remote": {} - }, - "timeout_seconds": { - "action": "skip", - "reason": "empty", - "remote": 0 - }, - "webhook_notifications": { - "action": "skip", - "reason": "empty", - "remote": {} - } - } - }, - "resources.jobs.parent": { - "action": "skip", - "remote_state": { - "created_time": [UNIX_TIME_MILLIS][1], - "creator_user_name": "[USERNAME]", - "deployment": { - "deployment_id": "[NUMID]", - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-depends-on-[UNIQUE_NAME]/default/state/metadata.json", - "version_id": "1" - }, - "edit_mode": "UI_LOCKED", - "email_notifications": {}, - "format": "MULTI_TASK", - "job_id": [NUMID], - "max_concurrent_runs": 1, - "name": "parent", - "queue": { - "enabled": true - }, - "run_as_user_name": "[USERNAME]", - "timeout_seconds": 0, - "webhook_notifications": {} - }, - "changes": { - "deployment.version_id": { - "action": "skip", - "reason": "auto", - "old": "1", - "new": "2", - "remote": "1" - }, - "email_notifications": { - "action": "skip", - "reason": "empty", - "remote": {} - }, - "timeout_seconds": { - "action": "skip", - "reason": "empty", - "remote": 0 - }, - "webhook_notifications": { - "action": "skip", - "reason": "empty", - "remote": {} - } - } - } - } + "storage_backend": "DEPLOYMENT_METADATA_SERVICE" } +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged + === The service lists both jobs, each recorded at version 1 >>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[NUMID] -o json [ diff --git a/acceptance/bundle/dms/depends-on/script b/acceptance/bundle/dms/depends-on/script index 895059dd8c9..8c0f865305b 100644 --- a/acceptance/bundle/dms/depends-on/script +++ b/acceptance/bundle/dms/depends-on/script @@ -1,14 +1,14 @@ envsubst < databricks.yml.tmpl > databricks.yml title "Plan before first deploy: no deployment yet, so no deployment_id or version fields" -trace $CLI bundle plan -o json +dms_plan title "Deploy a job that references another: depends_on is recorded alongside the config, since the reference is resolved to a literal in the config itself" trace $CLI bundle deploy trace print_requests.py --dms //versions/1/operations title "The plan is a no-op now and records DEPLOYMENT_METADATA_SERVICE as its backend" -trace $CLI bundle plan -o json +dms_plan title "The service lists both jobs, each recorded at version 1" deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-depends-on-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') diff --git a/acceptance/bundle/dms/deployment-metadata-change/output.txt b/acceptance/bundle/dms/deployment-metadata-change/output.txt index b3bc662ac7b..8d1199d14c1 100644 --- a/acceptance/bundle/dms/deployment-metadata-change/output.txt +++ b/acceptance/bundle/dms/deployment-metadata-change/output.txt @@ -69,34 +69,12 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "deployment_id": "[NUMID]", "next_version_id": "2", "last_version_id": "1", - "storage_backend": "DEPLOYMENT_METADATA_SERVICE", - "plan": { - "resources.schemas.foo": { - "action": "skip", - "remote_state": { - "browse_only": false, - "catalog_name": "main", - "catalog_type": "MANAGED_CATALOG", - "created_at": [UNIX_TIME_MILLIS][0], - "created_by": "[USERNAME]", - "effective_predictive_optimization_flag": { - "inherited_from_name": "[METASTORE_NAME]", - "inherited_from_type": "METASTORE", - "value": "ENABLE" - }, - "enable_predictive_optimization": "INHERIT", - "full_name": "main.dms_metadata_schema_[UNIQUE_NAME]", - "metastore_id": "[UUID]", - "name": "dms_metadata_schema_[UNIQUE_NAME]", - "owner": "[USERNAME]", - "schema_id": "[UUID]", - "updated_at": [UNIX_TIME_MILLIS][0], - "updated_by": "[USERNAME]" - } - } - } + "storage_backend": "DEPLOYMENT_METADATA_SERVICE" } +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged + >>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[NUMID] -o json [ { @@ -172,34 +150,18 @@ A common practice is to use a username or principal name in this path, i.e. use "deployment_id": "[NUMID]", "next_version_id": "3", "last_version_id": "2", - "storage_backend": "DEPLOYMENT_METADATA_SERVICE", - "plan": { - "resources.schemas.foo": { - "action": "skip", - "remote_state": { - "browse_only": false, - "catalog_name": "main", - "catalog_type": "MANAGED_CATALOG", - "created_at": [UNIX_TIME_MILLIS][0], - "created_by": "[USERNAME]", - "effective_predictive_optimization_flag": { - "inherited_from_name": "[METASTORE_NAME]", - "inherited_from_type": "METASTORE", - "value": "ENABLE" - }, - "enable_predictive_optimization": "INHERIT", - "full_name": "main.dms_metadata_schema_[UNIQUE_NAME]", - "metastore_id": "[UUID]", - "name": "dms_metadata_schema_[UNIQUE_NAME]", - "owner": "[USERNAME]", - "schema_id": "[UUID]", - "updated_at": [UNIX_TIME_MILLIS][0], - "updated_by": "[USERNAME]" - } - } - } + "storage_backend": "DEPLOYMENT_METADATA_SERVICE" } +>>> [CLI] bundle plan -t prod +Recommendation: target with 'mode: production' should set 'workspace.root_path' to make sure only one copy is deployed + +A common practice is to use a username or principal name in this path, i.e. use + + root_path: /Workspace/Users/[USERNAME]/.bundle/${bundle.name}/${bundle.target} + +Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged + >>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[NUMID] -o json [ { diff --git a/acceptance/bundle/dms/deployment-metadata-change/script b/acceptance/bundle/dms/deployment-metadata-change/script index d59e1365842..e3bc9895307 100644 --- a/acceptance/bundle/dms/deployment-metadata-change/script +++ b/acceptance/bundle/dms/deployment-metadata-change/script @@ -5,7 +5,7 @@ trace $CLI bundle deploy trace print_requests.py --dms //api/2.0/bundle title "The plan records DEPLOYMENT_METADATA_SERVICE; the service holds the schema at version 1" -trace $CLI bundle plan -o json +dms_plan deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-metadata-pinned-${UNIQUE_NAME}/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' @@ -14,7 +14,7 @@ trace update_file.py databricks.yml dms-metadata-before dms-metadata-after trace $CLI bundle deploy -t prod trace print_requests.py --dms //api/2.0/bundle -trace $CLI bundle plan -t prod -o json +dms_plan -t prod trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' title "Deploy again without touching anything: the record already says this, so no deployment update" diff --git a/acceptance/bundle/dms/emptied-resource/output.txt b/acceptance/bundle/dms/emptied-resource/output.txt index 8209a96b2fb..5b70ee58e6d 100644 --- a/acceptance/bundle/dms/emptied-resource/output.txt +++ b/acceptance/bundle/dms/emptied-resource/output.txt @@ -140,34 +140,12 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged "deployment_id": "[NUMID]", "next_version_id": "3", "last_version_id": "2", - "storage_backend": "DEPLOYMENT_METADATA_SERVICE", - "plan": { - "resources.schemas.foo": { - "action": "skip", - "remote_state": { - "browse_only": false, - "catalog_name": "main", - "catalog_type": "MANAGED_CATALOG", - "created_at": [UNIX_TIME_MILLIS][0], - "created_by": "[USERNAME]", - "effective_predictive_optimization_flag": { - "inherited_from_name": "[METASTORE_NAME]", - "inherited_from_type": "METASTORE", - "value": "ENABLE" - }, - "enable_predictive_optimization": "INHERIT", - "full_name": "main.dms_emptied_resource_[UNIQUE_NAME]", - "metastore_id": "[UUID]", - "name": "dms_emptied_resource_[UNIQUE_NAME]", - "owner": "[USERNAME]", - "schema_id": "[UUID]", - "updated_at": [UNIX_TIME_MILLIS][1], - "updated_by": "[USERNAME]" - } - } - } + "storage_backend": "DEPLOYMENT_METADATA_SERVICE" } +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged + === The grants node emptied out, so the service no longer lists it - only the schema remains >>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[NUMID] -o json [ diff --git a/acceptance/bundle/dms/emptied-resource/script b/acceptance/bundle/dms/emptied-resource/script index 329163ad95e..49ff14b8a41 100644 --- a/acceptance/bundle/dms/emptied-resource/script +++ b/acceptance/bundle/dms/emptied-resource/script @@ -17,7 +17,7 @@ title "Plan again: reading state back from the service works and reports no work trace $CLI bundle plan | contains.py "Plan: 0 to add, 0 to change, 0 to delete" "!unexpected end of JSON input" title "The plan is a no-op and records DEPLOYMENT_METADATA_SERVICE as its backend" -trace $CLI bundle plan -o json +dms_plan title "The grants node emptied out, so the service no longer lists it - only the schema remains" deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-emptied-resource-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') diff --git a/acceptance/bundle/dms/existing-state/output.txt b/acceptance/bundle/dms/existing-state/output.txt index 0d35577a43c..b4c1b705c28 100644 --- a/acceptance/bundle/dms/existing-state/output.txt +++ b/acceptance/bundle/dms/existing-state/output.txt @@ -11,51 +11,12 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "plan_version": 3, "cli_version": "[CLI_VERSION]", "lineage": "[UUID]", - "serial": 1, - "plan": { - "resources.jobs.one": { - "action": "skip", - "remote_state": { - "created_time": [UNIX_TIME_MILLIS], - "creator_user_name": "[USERNAME]", - "deployment": { - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-existing-state-[UNIQUE_NAME]/default/state/metadata.json" - }, - "edit_mode": "UI_LOCKED", - "email_notifications": {}, - "format": "MULTI_TASK", - "job_id": [NUMID], - "max_concurrent_runs": 1, - "name": "one", - "queue": { - "enabled": true - }, - "run_as_user_name": "[USERNAME]", - "timeout_seconds": 0, - "webhook_notifications": {} - }, - "changes": { - "email_notifications": { - "action": "skip", - "reason": "empty", - "remote": {} - }, - "timeout_seconds": { - "action": "skip", - "reason": "empty", - "remote": 0 - }, - "webhook_notifications": { - "action": "skip", - "reason": "empty", - "remote": {} - } - } - } - } + "serial": 1 } +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged + >>> print_requests.py --dms //api/2.0/bundle --oneline >>> jq {state_version, features} .databricks/bundle/default/resources.json diff --git a/acceptance/bundle/dms/existing-state/script b/acceptance/bundle/dms/existing-state/script index a5be325e72a..aa3bee78294 100644 --- a/acceptance/bundle/dms/existing-state/script +++ b/acceptance/bundle/dms/existing-state/script @@ -2,7 +2,7 @@ envsubst < databricks.yml.tmpl > databricks.yml title "Deploy without recording: ordinary direct-engine state tracking one job, unknown to DMS" trace $CLI bundle deploy -trace $CLI bundle plan -o json +dms_plan trace print_requests.py --dms //api/2.0/bundle --oneline trace jq '{state_version, features}' .databricks/bundle/default/resources.json diff --git a/acceptance/bundle/dms/failed-delete/output.txt b/acceptance/bundle/dms/failed-delete/output.txt index 2d06337ff31..c196fc9e64e 100644 --- a/acceptance/bundle/dms/failed-delete/output.txt +++ b/acceptance/bundle/dms/failed-delete/output.txt @@ -69,60 +69,12 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "deployment_id": "[NUMID]", "next_version_id": "2", "last_version_id": "1", - "storage_backend": "DEPLOYMENT_METADATA_SERVICE", - "plan": { - "resources.jobs.stuck": { - "action": "skip", - "remote_state": { - "created_time": [UNIX_TIME_MILLIS], - "creator_user_name": "[USERNAME]", - "deployment": { - "deployment_id": "[NUMID]", - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-failed-delete-[UNIQUE_NAME]/default/state/metadata.json", - "version_id": "1" - }, - "edit_mode": "UI_LOCKED", - "email_notifications": {}, - "format": "MULTI_TASK", - "job_id": [NUMID], - "max_concurrent_runs": 1, - "name": "stuck", - "queue": { - "enabled": true - }, - "run_as_user_name": "[USERNAME]", - "timeout_seconds": 0, - "webhook_notifications": {} - }, - "changes": { - "deployment.version_id": { - "action": "skip", - "reason": "auto", - "old": "1", - "new": "2", - "remote": "1" - }, - "email_notifications": { - "action": "skip", - "reason": "empty", - "remote": {} - }, - "timeout_seconds": { - "action": "skip", - "reason": "empty", - "remote": 0 - }, - "webhook_notifications": { - "action": "skip", - "reason": "empty", - "remote": {} - } - } - } - } + "storage_backend": "DEPLOYMENT_METADATA_SERVICE" } +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged + === After deploy, the service lists the resource at version 1 >>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[NUMID] -o json [ diff --git a/acceptance/bundle/dms/failed-delete/script b/acceptance/bundle/dms/failed-delete/script index 08d949fd198..c343679ef14 100644 --- a/acceptance/bundle/dms/failed-delete/script +++ b/acceptance/bundle/dms/failed-delete/script @@ -5,7 +5,7 @@ trace $CLI bundle deploy trace print_requests.py --dms //api/2.0/bundle title "The plan is a no-op now and records DEPLOYMENT_METADATA_SERVICE as its backend" -trace $CLI bundle plan -o json +dms_plan title "After deploy, the service lists the resource at version 1" deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-failed-delete-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') diff --git a/acceptance/bundle/dms/failed-recreate/output.txt b/acceptance/bundle/dms/failed-recreate/output.txt index a4a818bf7bc..894fabe56a5 100644 --- a/acceptance/bundle/dms/failed-recreate/output.txt +++ b/acceptance/bundle/dms/failed-recreate/output.txt @@ -69,35 +69,12 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "deployment_id": "[NUMID]", "next_version_id": "2", "last_version_id": "1", - "storage_backend": "DEPLOYMENT_METADATA_SERVICE", - "plan": { - "resources.schemas.foo": { - "action": "skip", - "remote_state": { - "browse_only": false, - "catalog_name": "main", - "catalog_type": "MANAGED_CATALOG", - "comment": "v1", - "created_at": [UNIX_TIME_MILLIS][0], - "created_by": "[USERNAME]", - "effective_predictive_optimization_flag": { - "inherited_from_name": "[METASTORE_NAME]", - "inherited_from_type": "METASTORE", - "value": "ENABLE" - }, - "enable_predictive_optimization": "INHERIT", - "full_name": "main.dms_failed_recreate_schema_[UNIQUE_NAME]", - "metastore_id": "[UUID]", - "name": "dms_failed_recreate_schema_[UNIQUE_NAME]", - "owner": "[USERNAME]", - "schema_id": "[UUID]", - "updated_at": [UNIX_TIME_MILLIS][0], - "updated_by": "[USERNAME]" - } - } - } + "storage_backend": "DEPLOYMENT_METADATA_SERVICE" } +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged + === After deploy, the service lists the resource at version 1 >>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[NUMID] -o json [ diff --git a/acceptance/bundle/dms/failed-recreate/script b/acceptance/bundle/dms/failed-recreate/script index 8d0b7ddeef1..e2dbc58793e 100644 --- a/acceptance/bundle/dms/failed-recreate/script +++ b/acceptance/bundle/dms/failed-recreate/script @@ -5,7 +5,7 @@ trace $CLI bundle deploy trace print_requests.py --dms //api/2.0/bundle title "The plan is a no-op and records DEPLOYMENT_METADATA_SERVICE as its backend" -trace $CLI bundle plan -o json +dms_plan title "After deploy, the service lists the resource at version 1" deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-failed-recreate-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') diff --git a/acceptance/bundle/dms/failed-update-permissions/output.txt b/acceptance/bundle/dms/failed-update-permissions/output.txt index 9a13313ac7d..8edf660d3f9 100644 --- a/acceptance/bundle/dms/failed-update-permissions/output.txt +++ b/acceptance/bundle/dms/failed-update-permissions/output.txt @@ -17,82 +17,12 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged "deployment_id": "[NUMID]", "next_version_id": "2", "last_version_id": "1", - "storage_backend": "DEPLOYMENT_METADATA_SERVICE", - "plan": { - "resources.jobs.foo": { - "action": "skip", - "remote_state": { - "created_time": [UNIX_TIME_MILLIS], - "creator_user_name": "[USERNAME]", - "deployment": { - "deployment_id": "[NUMID]", - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-failed-update-permissions-[UNIQUE_NAME]/default/state/metadata.json", - "version_id": "1" - }, - "edit_mode": "UI_LOCKED", - "email_notifications": {}, - "format": "MULTI_TASK", - "job_id": [NUMID], - "max_concurrent_runs": 1, - "name": "foo", - "queue": { - "enabled": true - }, - "run_as_user_name": "[USERNAME]", - "timeout_seconds": 0, - "webhook_notifications": {} - }, - "changes": { - "deployment.version_id": { - "action": "skip", - "reason": "auto", - "old": "1", - "new": "2", - "remote": "1" - }, - "email_notifications": { - "action": "skip", - "reason": "empty", - "remote": {} - }, - "timeout_seconds": { - "action": "skip", - "reason": "empty", - "remote": 0 - }, - "webhook_notifications": { - "action": "skip", - "reason": "empty", - "remote": {} - } - } - }, - "resources.jobs.foo.permissions": { - "depends_on": [ - { - "node": "resources.jobs.foo", - "label": "${resources.jobs.foo.id}" - } - ], - "action": "skip", - "remote_state": { - "object_id": "/jobs/[NUMID]", - "__embed__": [ - { - "level": "CAN_VIEW", - "user_name": "viewer@example.com" - }, - { - "level": "IS_OWNER", - "user_name": "[USERNAME]" - } - ] - } - } - } + "storage_backend": "DEPLOYMENT_METADATA_SERVICE" } +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged + === The service lists the job at version 1 >>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[NUMID] -o json [ diff --git a/acceptance/bundle/dms/failed-update-permissions/script b/acceptance/bundle/dms/failed-update-permissions/script index 2dc2aca15ff..24a9f9ee45e 100644 --- a/acceptance/bundle/dms/failed-update-permissions/script +++ b/acceptance/bundle/dms/failed-update-permissions/script @@ -5,7 +5,7 @@ trace $CLI bundle deploy rm -f out.requests.txt title "The plan records DEPLOYMENT_METADATA_SERVICE backend and the job" -trace $CLI bundle plan -o json +dms_plan title "The service lists the job at version 1" deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-failed-update-permissions-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') diff --git a/acceptance/bundle/dms/failed-update/output.txt b/acceptance/bundle/dms/failed-update/output.txt index c9d8c5568c9..bd837caeda2 100644 --- a/acceptance/bundle/dms/failed-update/output.txt +++ b/acceptance/bundle/dms/failed-update/output.txt @@ -69,35 +69,12 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "deployment_id": "[NUMID]", "next_version_id": "2", "last_version_id": "1", - "storage_backend": "DEPLOYMENT_METADATA_SERVICE", - "plan": { - "resources.schemas.foo": { - "action": "skip", - "remote_state": { - "browse_only": false, - "catalog_name": "main", - "catalog_type": "MANAGED_CATALOG", - "comment": "v1", - "created_at": [UNIX_TIME_MILLIS][0], - "created_by": "[USERNAME]", - "effective_predictive_optimization_flag": { - "inherited_from_name": "[METASTORE_NAME]", - "inherited_from_type": "METASTORE", - "value": "ENABLE" - }, - "enable_predictive_optimization": "INHERIT", - "full_name": "main.dms_failed_update_schema_[UNIQUE_NAME]", - "metastore_id": "[UUID]", - "name": "dms_failed_update_schema_[UNIQUE_NAME]", - "owner": "[USERNAME]", - "schema_id": "[UUID]", - "updated_at": [UNIX_TIME_MILLIS][0], - "updated_by": "[USERNAME]" - } - } - } + "storage_backend": "DEPLOYMENT_METADATA_SERVICE" } +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged + === After deploy, the service lists the resource at version 1 >>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[NUMID] -o json [ diff --git a/acceptance/bundle/dms/failed-update/script b/acceptance/bundle/dms/failed-update/script index bdadee08fac..a956ed2fa5b 100644 --- a/acceptance/bundle/dms/failed-update/script +++ b/acceptance/bundle/dms/failed-update/script @@ -5,7 +5,7 @@ trace $CLI bundle deploy trace print_requests.py --dms //api/2.0/bundle title "The plan is a no-op and records DEPLOYMENT_METADATA_SERVICE as its backend" -trace $CLI bundle plan -o json +dms_plan title "After deploy, the service lists the resource at version 1" deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-failed-update-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') diff --git a/acceptance/bundle/dms/multiple-resources/output.txt b/acceptance/bundle/dms/multiple-resources/output.txt index 688fe93bb3a..aee36fb25dc 100644 --- a/acceptance/bundle/dms/multiple-resources/output.txt +++ b/acceptance/bundle/dms/multiple-resources/output.txt @@ -27,256 +27,12 @@ Resources: 5 created, 0 changed, 0 deleted, 0 unchanged "deployment_id": "[NUMID]", "next_version_id": "2", "last_version_id": "1", - "storage_backend": "DEPLOYMENT_METADATA_SERVICE", - "plan": { - "resources.jobs.five": { - "action": "skip", - "remote_state": { - "created_time": [UNIX_TIME_MILLIS][0], - "creator_user_name": "[USERNAME]", - "deployment": { - "deployment_id": "[NUMID]", - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/state/metadata.json", - "version_id": "1" - }, - "edit_mode": "UI_LOCKED", - "email_notifications": {}, - "format": "MULTI_TASK", - "job_id": [NUMID], - "max_concurrent_runs": 1, - "name": "five", - "queue": { - "enabled": true - }, - "run_as_user_name": "[USERNAME]", - "timeout_seconds": 0, - "webhook_notifications": {} - }, - "changes": { - "deployment.version_id": { - "action": "skip", - "reason": "auto", - "old": "1", - "new": "2", - "remote": "1" - }, - "email_notifications": { - "action": "skip", - "reason": "empty", - "remote": {} - }, - "timeout_seconds": { - "action": "skip", - "reason": "empty", - "remote": 0 - }, - "webhook_notifications": { - "action": "skip", - "reason": "empty", - "remote": {} - } - } - }, - "resources.jobs.four": { - "action": "skip", - "remote_state": { - "created_time": [UNIX_TIME_MILLIS][1], - "creator_user_name": "[USERNAME]", - "deployment": { - "deployment_id": "[NUMID]", - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/state/metadata.json", - "version_id": "1" - }, - "edit_mode": "UI_LOCKED", - "email_notifications": {}, - "format": "MULTI_TASK", - "job_id": [NUMID], - "max_concurrent_runs": 1, - "name": "four", - "queue": { - "enabled": true - }, - "run_as_user_name": "[USERNAME]", - "timeout_seconds": 0, - "webhook_notifications": {} - }, - "changes": { - "deployment.version_id": { - "action": "skip", - "reason": "auto", - "old": "1", - "new": "2", - "remote": "1" - }, - "email_notifications": { - "action": "skip", - "reason": "empty", - "remote": {} - }, - "timeout_seconds": { - "action": "skip", - "reason": "empty", - "remote": 0 - }, - "webhook_notifications": { - "action": "skip", - "reason": "empty", - "remote": {} - } - } - }, - "resources.jobs.one": { - "action": "skip", - "remote_state": { - "created_time": [UNIX_TIME_MILLIS][2], - "creator_user_name": "[USERNAME]", - "deployment": { - "deployment_id": "[NUMID]", - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/state/metadata.json", - "version_id": "1" - }, - "edit_mode": "UI_LOCKED", - "email_notifications": {}, - "format": "MULTI_TASK", - "job_id": [NUMID], - "max_concurrent_runs": 1, - "name": "one", - "queue": { - "enabled": true - }, - "run_as_user_name": "[USERNAME]", - "timeout_seconds": 0, - "webhook_notifications": {} - }, - "changes": { - "deployment.version_id": { - "action": "skip", - "reason": "auto", - "old": "1", - "new": "2", - "remote": "1" - }, - "email_notifications": { - "action": "skip", - "reason": "empty", - "remote": {} - }, - "timeout_seconds": { - "action": "skip", - "reason": "empty", - "remote": 0 - }, - "webhook_notifications": { - "action": "skip", - "reason": "empty", - "remote": {} - } - } - }, - "resources.jobs.three": { - "action": "skip", - "remote_state": { - "created_time": [UNIX_TIME_MILLIS][3], - "creator_user_name": "[USERNAME]", - "deployment": { - "deployment_id": "[NUMID]", - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/state/metadata.json", - "version_id": "1" - }, - "edit_mode": "UI_LOCKED", - "email_notifications": {}, - "format": "MULTI_TASK", - "job_id": [NUMID], - "max_concurrent_runs": 1, - "name": "three", - "queue": { - "enabled": true - }, - "run_as_user_name": "[USERNAME]", - "timeout_seconds": 0, - "webhook_notifications": {} - }, - "changes": { - "deployment.version_id": { - "action": "skip", - "reason": "auto", - "old": "1", - "new": "2", - "remote": "1" - }, - "email_notifications": { - "action": "skip", - "reason": "empty", - "remote": {} - }, - "timeout_seconds": { - "action": "skip", - "reason": "empty", - "remote": 0 - }, - "webhook_notifications": { - "action": "skip", - "reason": "empty", - "remote": {} - } - } - }, - "resources.jobs.two": { - "action": "skip", - "remote_state": { - "created_time": [UNIX_TIME_MILLIS][4], - "creator_user_name": "[USERNAME]", - "deployment": { - "deployment_id": "[NUMID]", - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/state/metadata.json", - "version_id": "1" - }, - "edit_mode": "UI_LOCKED", - "email_notifications": {}, - "format": "MULTI_TASK", - "job_id": [NUMID], - "max_concurrent_runs": 1, - "name": "two", - "queue": { - "enabled": true - }, - "run_as_user_name": "[USERNAME]", - "timeout_seconds": 0, - "webhook_notifications": {} - }, - "changes": { - "deployment.version_id": { - "action": "skip", - "reason": "auto", - "old": "1", - "new": "2", - "remote": "1" - }, - "email_notifications": { - "action": "skip", - "reason": "empty", - "remote": {} - }, - "timeout_seconds": { - "action": "skip", - "reason": "empty", - "remote": 0 - }, - "webhook_notifications": { - "action": "skip", - "reason": "empty", - "remote": {} - } - } - } - } + "storage_backend": "DEPLOYMENT_METADATA_SERVICE" } +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 5 unchanged + === The service lists all five resources at version 1 >>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[NUMID] -o json [ @@ -340,256 +96,12 @@ Resources: 0 created, 0 changed, 0 deleted, 5 unchanged "deployment_id": "[NUMID]", "next_version_id": "3", "last_version_id": "2", - "storage_backend": "DEPLOYMENT_METADATA_SERVICE", - "plan": { - "resources.jobs.five": { - "action": "skip", - "remote_state": { - "created_time": [UNIX_TIME_MILLIS][0], - "creator_user_name": "[USERNAME]", - "deployment": { - "deployment_id": "[NUMID]", - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/state/metadata.json", - "version_id": "1" - }, - "edit_mode": "UI_LOCKED", - "email_notifications": {}, - "format": "MULTI_TASK", - "job_id": [NUMID], - "max_concurrent_runs": 1, - "name": "five", - "queue": { - "enabled": true - }, - "run_as_user_name": "[USERNAME]", - "timeout_seconds": 0, - "webhook_notifications": {} - }, - "changes": { - "deployment.version_id": { - "action": "skip", - "reason": "auto", - "old": "1", - "new": "3", - "remote": "1" - }, - "email_notifications": { - "action": "skip", - "reason": "empty", - "remote": {} - }, - "timeout_seconds": { - "action": "skip", - "reason": "empty", - "remote": 0 - }, - "webhook_notifications": { - "action": "skip", - "reason": "empty", - "remote": {} - } - } - }, - "resources.jobs.four": { - "action": "skip", - "remote_state": { - "created_time": [UNIX_TIME_MILLIS][1], - "creator_user_name": "[USERNAME]", - "deployment": { - "deployment_id": "[NUMID]", - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/state/metadata.json", - "version_id": "1" - }, - "edit_mode": "UI_LOCKED", - "email_notifications": {}, - "format": "MULTI_TASK", - "job_id": [NUMID], - "max_concurrent_runs": 1, - "name": "four", - "queue": { - "enabled": true - }, - "run_as_user_name": "[USERNAME]", - "timeout_seconds": 0, - "webhook_notifications": {} - }, - "changes": { - "deployment.version_id": { - "action": "skip", - "reason": "auto", - "old": "1", - "new": "3", - "remote": "1" - }, - "email_notifications": { - "action": "skip", - "reason": "empty", - "remote": {} - }, - "timeout_seconds": { - "action": "skip", - "reason": "empty", - "remote": 0 - }, - "webhook_notifications": { - "action": "skip", - "reason": "empty", - "remote": {} - } - } - }, - "resources.jobs.one": { - "action": "skip", - "remote_state": { - "created_time": [UNIX_TIME_MILLIS][2], - "creator_user_name": "[USERNAME]", - "deployment": { - "deployment_id": "[NUMID]", - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/state/metadata.json", - "version_id": "1" - }, - "edit_mode": "UI_LOCKED", - "email_notifications": {}, - "format": "MULTI_TASK", - "job_id": [NUMID], - "max_concurrent_runs": 1, - "name": "one", - "queue": { - "enabled": true - }, - "run_as_user_name": "[USERNAME]", - "timeout_seconds": 0, - "webhook_notifications": {} - }, - "changes": { - "deployment.version_id": { - "action": "skip", - "reason": "auto", - "old": "1", - "new": "3", - "remote": "1" - }, - "email_notifications": { - "action": "skip", - "reason": "empty", - "remote": {} - }, - "timeout_seconds": { - "action": "skip", - "reason": "empty", - "remote": 0 - }, - "webhook_notifications": { - "action": "skip", - "reason": "empty", - "remote": {} - } - } - }, - "resources.jobs.three": { - "action": "skip", - "remote_state": { - "created_time": [UNIX_TIME_MILLIS][3], - "creator_user_name": "[USERNAME]", - "deployment": { - "deployment_id": "[NUMID]", - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/state/metadata.json", - "version_id": "1" - }, - "edit_mode": "UI_LOCKED", - "email_notifications": {}, - "format": "MULTI_TASK", - "job_id": [NUMID], - "max_concurrent_runs": 1, - "name": "three", - "queue": { - "enabled": true - }, - "run_as_user_name": "[USERNAME]", - "timeout_seconds": 0, - "webhook_notifications": {} - }, - "changes": { - "deployment.version_id": { - "action": "skip", - "reason": "auto", - "old": "1", - "new": "3", - "remote": "1" - }, - "email_notifications": { - "action": "skip", - "reason": "empty", - "remote": {} - }, - "timeout_seconds": { - "action": "skip", - "reason": "empty", - "remote": 0 - }, - "webhook_notifications": { - "action": "skip", - "reason": "empty", - "remote": {} - } - } - }, - "resources.jobs.two": { - "action": "skip", - "remote_state": { - "created_time": [UNIX_TIME_MILLIS][4], - "creator_user_name": "[USERNAME]", - "deployment": { - "deployment_id": "[NUMID]", - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/state/metadata.json", - "version_id": "1" - }, - "edit_mode": "UI_LOCKED", - "email_notifications": {}, - "format": "MULTI_TASK", - "job_id": [NUMID], - "max_concurrent_runs": 1, - "name": "two", - "queue": { - "enabled": true - }, - "run_as_user_name": "[USERNAME]", - "timeout_seconds": 0, - "webhook_notifications": {} - }, - "changes": { - "deployment.version_id": { - "action": "skip", - "reason": "auto", - "old": "1", - "new": "3", - "remote": "1" - }, - "email_notifications": { - "action": "skip", - "reason": "empty", - "remote": {} - }, - "timeout_seconds": { - "action": "skip", - "reason": "empty", - "remote": 0 - }, - "webhook_notifications": { - "action": "skip", - "reason": "empty", - "remote": {} - } - } - } - } + "storage_backend": "DEPLOYMENT_METADATA_SERVICE" } +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 5 unchanged + === The service still lists all five resources, now at version 2 >>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[NUMID] -o json [ diff --git a/acceptance/bundle/dms/multiple-resources/script b/acceptance/bundle/dms/multiple-resources/script index 339c7973c5c..5ce8d484f83 100644 --- a/acceptance/bundle/dms/multiple-resources/script +++ b/acceptance/bundle/dms/multiple-resources/script @@ -5,7 +5,7 @@ trace $CLI bundle deploy trace print_requests.py --dms //versions/1/operations --sort --del-body state --oneline title "The plan records DEPLOYMENT_METADATA_SERVICE as its backend" -trace $CLI bundle plan -o json +dms_plan title "The service lists all five resources at version 1" deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-multiple-resources-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') @@ -19,7 +19,7 @@ trace $CLI bundle deploy trace print_requests.py --dms //api/2.0/bundle --sort --oneline title "The plan records DEPLOYMENT_METADATA_SERVICE as its backend" -trace $CLI bundle plan -o json +dms_plan title "The service still lists all five resources, now at version 2" deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-multiple-resources-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') diff --git a/acceptance/bundle/dms/no-drift/output.txt b/acceptance/bundle/dms/no-drift/output.txt index 1c8708ac68b..f37562b84f0 100644 --- a/acceptance/bundle/dms/no-drift/output.txt +++ b/acceptance/bundle/dms/no-drift/output.txt @@ -37,100 +37,12 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged "deployment_id": "[NUMID]", "next_version_id": "2", "last_version_id": "1", - "storage_backend": "DEPLOYMENT_METADATA_SERVICE", - "plan": { - "resources.jobs.foo": { - "action": "skip", - "remote_state": { - "created_time": [UNIX_TIME_MILLIS][0], - "creator_user_name": "[USERNAME]", - "deployment": { - "deployment_id": "[NUMID]", - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-no-drift-[UNIQUE_NAME]/default/state/metadata.json", - "version_id": "1" - }, - "edit_mode": "UI_LOCKED", - "email_notifications": {}, - "format": "MULTI_TASK", - "job_id": [NUMID], - "max_concurrent_runs": 1, - "name": "foo", - "queue": { - "enabled": true - }, - "run_as_user_name": "[USERNAME]", - "timeout_seconds": 0, - "webhook_notifications": {} - }, - "changes": { - "deployment.version_id": { - "action": "skip", - "reason": "auto", - "old": "1", - "new": "2", - "remote": "1" - }, - "email_notifications": { - "action": "skip", - "reason": "empty", - "remote": {} - }, - "timeout_seconds": { - "action": "skip", - "reason": "empty", - "remote": 0 - }, - "webhook_notifications": { - "action": "skip", - "reason": "empty", - "remote": {} - } - } - }, - "resources.pipelines.bar": { - "action": "skip", - "remote_state": { - "catalog": "main", - "channel": "CURRENT", - "creator_user_name": "[USERNAME]", - "deployment": { - "deployment_id": "[NUMID]", - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-no-drift-[UNIQUE_NAME]/default/state/metadata.json", - "version_id": "1" - }, - "edition": "ADVANCED", - "effective_publishing_mode": "DEFAULT_PUBLISHING_MODE", - "id": "[UUID]", - "last_modified": [UNIX_TIME_MILLIS][1], - "libraries": [ - { - "file": { - "path": "/Workspace/Users/[USERNAME]/.bundle/dms-no-drift-[UNIQUE_NAME]/default/files/transform.py" - } - } - ], - "name": "bar-[UNIQUE_NAME]", - "pipeline_id": "[UUID]", - "run_as_user_name": "[USERNAME]", - "schema": "default", - "serverless": true, - "state": "IDLE" - }, - "changes": { - "deployment.version_id": { - "action": "skip", - "reason": "auto", - "old": "1", - "new": "2", - "remote": "1" - } - } - } - } + "storage_backend": "DEPLOYMENT_METADATA_SERVICE" } +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged + === The raw recorded state the service holds for this deployment >>> print_state.py --no-dms { @@ -260,101 +172,12 @@ Resources: 0 created, 1 changed, 0 deleted, 1 unchanged "deployment_id": "[NUMID]", "next_version_id": "3", "last_version_id": "2", - "storage_backend": "DEPLOYMENT_METADATA_SERVICE", - "plan": { - "resources.jobs.foo": { - "action": "skip", - "remote_state": { - "created_time": [UNIX_TIME_MILLIS][0], - "creator_user_name": "[USERNAME]", - "deployment": { - "deployment_id": "[NUMID]", - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-no-drift-[UNIQUE_NAME]/default/state/metadata.json", - "version_id": "1" - }, - "edit_mode": "UI_LOCKED", - "email_notifications": {}, - "format": "MULTI_TASK", - "job_id": [NUMID], - "max_concurrent_runs": 1, - "name": "foo", - "queue": { - "enabled": true - }, - "run_as_user_name": "[USERNAME]", - "timeout_seconds": 0, - "webhook_notifications": {} - }, - "changes": { - "deployment.version_id": { - "action": "skip", - "reason": "auto", - "old": "1", - "new": "3", - "remote": "1" - }, - "email_notifications": { - "action": "skip", - "reason": "empty", - "remote": {} - }, - "timeout_seconds": { - "action": "skip", - "reason": "empty", - "remote": 0 - }, - "webhook_notifications": { - "action": "skip", - "reason": "empty", - "remote": {} - } - } - }, - "resources.pipelines.bar": { - "action": "skip", - "remote_state": { - "catalog": "main", - "channel": "CURRENT", - "creator_user_name": "[USERNAME]", - "deployment": { - "deployment_id": "[NUMID]", - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-no-drift-[UNIQUE_NAME]/default/state/metadata.json", - "version_id": "2" - }, - "development": true, - "edition": "ADVANCED", - "effective_publishing_mode": "DEFAULT_PUBLISHING_MODE", - "id": "[UUID]", - "last_modified": [UNIX_TIME_MILLIS][1], - "libraries": [ - { - "file": { - "path": "/Workspace/Users/[USERNAME]/.bundle/dms-no-drift-[UNIQUE_NAME]/default/files/transform.py" - } - } - ], - "name": "bar-[UNIQUE_NAME]", - "pipeline_id": "[UUID]", - "run_as_user_name": "[USERNAME]", - "schema": "default", - "serverless": true, - "state": "IDLE" - }, - "changes": { - "deployment.version_id": { - "action": "skip", - "reason": "auto", - "old": "2", - "new": "3", - "remote": "2" - } - } - } - } + "storage_backend": "DEPLOYMENT_METADATA_SERVICE" } +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 2 unchanged + === The raw recorded state the service holds after the update >>> print_state.py --no-dms { diff --git a/acceptance/bundle/dms/no-drift/script b/acceptance/bundle/dms/no-drift/script index 67e8172e06b..203a2bebc05 100644 --- a/acceptance/bundle/dms/no-drift/script +++ b/acceptance/bundle/dms/no-drift/script @@ -8,7 +8,7 @@ trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${d trace $CLI bundle summary -o json | jq .bundle.deployment.history title "The plan is a no-op now and records DEPLOYMENT_METADATA_SERVICE as its backend" -trace $CLI bundle plan -o json +dms_plan title "The raw recorded state the service holds for this deployment" trace print_state.py --no-dms @@ -27,7 +27,7 @@ trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${d trace $CLI bundle summary -o json | jq .bundle.deployment.history title "The plan shows version 2 for the updated pipeline and DEPLOYMENT_METADATA_SERVICE as backend" -trace $CLI bundle plan -o json +dms_plan title "The raw recorded state the service holds after the update" trace print_state.py --no-dms diff --git a/acceptance/bundle/dms/no-resources/output.txt b/acceptance/bundle/dms/no-resources/output.txt index ee9b3127061..22b9703b44a 100644 --- a/acceptance/bundle/dms/no-resources/output.txt +++ b/acceptance/bundle/dms/no-resources/output.txt @@ -52,10 +52,12 @@ Resources: 0 created, 0 changed, 0 deleted, 0 unchanged "deployment_id": "[NUMID]", "next_version_id": "2", "last_version_id": "1", - "storage_backend": "DEPLOYMENT_METADATA_SERVICE", - "plan": {} + "storage_backend": "DEPLOYMENT_METADATA_SERVICE" } +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 0 unchanged + === The service lists no resources for the deployment >>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[NUMID] -o json [] @@ -78,6 +80,14 @@ Resources: 0 created, 0 changed, 0 deleted, 0 unchanged "method": "GET", "path": "/api/2.0/bundle/deployments/[NUMID]/resources" } +{ + "method": "GET", + "path": "/api/2.0/bundle/deployments/[NUMID]" +} +{ + "method": "GET", + "path": "/api/2.0/bundle/deployments/[NUMID]/resources" +} { "method": "GET", "path": "/api/2.0/bundle/deployments/[NUMID]/resources" @@ -118,10 +128,12 @@ Resources: 0 created, 0 changed, 0 deleted, 0 unchanged "deployment_id": "[NUMID]", "next_version_id": "3", "last_version_id": "2", - "storage_backend": "DEPLOYMENT_METADATA_SERVICE", - "plan": {} + "storage_backend": "DEPLOYMENT_METADATA_SERVICE" } +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 0 unchanged + === The service still lists no resources >>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[NUMID] -o json [] diff --git a/acceptance/bundle/dms/no-resources/script b/acceptance/bundle/dms/no-resources/script index 1593aee522f..d09f7a7faed 100644 --- a/acceptance/bundle/dms/no-resources/script +++ b/acceptance/bundle/dms/no-resources/script @@ -6,7 +6,7 @@ trace print_requests.py --dms //api/2.0/bundle --get trace MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-no-resources-${UNIQUE_NAME}/default/state/resources.deployment.json" | jq '{object_type,path}' title "The plan records DEPLOYMENT_METADATA_SERVICE as its backend" -trace $CLI bundle plan -o json +dms_plan title "The service lists no resources for the deployment" deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-no-resources-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') @@ -20,7 +20,7 @@ trace $CLI bundle deploy trace print_requests.py --dms //api/2.0/bundle --get title "The plan records DEPLOYMENT_METADATA_SERVICE as its backend" -trace $CLI bundle plan -o json +dms_plan title "The service still lists no resources" deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-no-resources-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') diff --git a/acceptance/bundle/dms/operation-upload-fails-once/output.txt b/acceptance/bundle/dms/operation-upload-fails-once/output.txt index 9060ab58706..b1a6a64ea06 100644 --- a/acceptance/bundle/dms/operation-upload-fails-once/output.txt +++ b/acceptance/bundle/dms/operation-upload-fails-once/output.txt @@ -16,35 +16,12 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "deployment_id": "[NUMID]", "next_version_id": "2", "last_version_id": "1", - "storage_backend": "DEPLOYMENT_METADATA_SERVICE", - "plan": { - "resources.schemas.foo": { - "action": "skip", - "remote_state": { - "browse_only": false, - "catalog_name": "main", - "catalog_type": "MANAGED_CATALOG", - "comment": "v1", - "created_at": [UNIX_TIME_MILLIS][0], - "created_by": "[USERNAME]", - "effective_predictive_optimization_flag": { - "inherited_from_name": "[METASTORE_NAME]", - "inherited_from_type": "METASTORE", - "value": "ENABLE" - }, - "enable_predictive_optimization": "INHERIT", - "full_name": "main.dms_upload_fails_once_schema_[UNIQUE_NAME]", - "metastore_id": "[UUID]", - "name": "dms_upload_fails_once_schema_[UNIQUE_NAME]", - "owner": "[USERNAME]", - "schema_id": "[UUID]", - "updated_at": [UNIX_TIME_MILLIS][0], - "updated_by": "[USERNAME]" - } - } - } + "storage_backend": "DEPLOYMENT_METADATA_SERVICE" } +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged + === The service lists the schema at version 1 >>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[NUMID] -o json [ diff --git a/acceptance/bundle/dms/operation-upload-fails-once/script b/acceptance/bundle/dms/operation-upload-fails-once/script index 73e895433ab..dc242c6262a 100644 --- a/acceptance/bundle/dms/operation-upload-fails-once/script +++ b/acceptance/bundle/dms/operation-upload-fails-once/script @@ -5,7 +5,7 @@ trace $CLI bundle deploy rm -f out.requests.txt title "The plan records DEPLOYMENT_METADATA_SERVICE backend and the schema" -trace $CLI bundle plan -o json +dms_plan title "The service lists the schema at version 1" deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-upload-fails-once-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') diff --git a/acceptance/bundle/dms/provenance/output.txt b/acceptance/bundle/dms/provenance/output.txt index 437af92448b..57fee97f7fc 100644 --- a/acceptance/bundle/dms/provenance/output.txt +++ b/acceptance/bundle/dms/provenance/output.txt @@ -61,63 +61,12 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "deployment_id": "[NUMID]", "next_version_id": "2", "last_version_id": "1", - "storage_backend": "DEPLOYMENT_METADATA_SERVICE", - "plan": { - "resources.jobs.foo": { - "action": "skip", - "remote_state": { - "created_time": [UNIX_TIME_MILLIS], - "creator_user_name": "[USERNAME]", - "deployment": { - "deployment_id": "[NUMID]", - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-provenance-[UNIQUE_NAME]/dev/state/metadata.json", - "version_id": "1" - }, - "edit_mode": "UI_LOCKED", - "email_notifications": {}, - "format": "MULTI_TASK", - "job_id": [NUMID], - "max_concurrent_runs": 4, - "name": "[dev [USERNAME]] foo", - "queue": { - "enabled": true - }, - "run_as_user_name": "[USERNAME]", - "tags": { - "dev": "[USERNAME]" - }, - "timeout_seconds": 0, - "webhook_notifications": {} - }, - "changes": { - "deployment.version_id": { - "action": "skip", - "reason": "auto", - "old": "1", - "new": "2", - "remote": "1" - }, - "email_notifications": { - "action": "skip", - "reason": "empty", - "remote": {} - }, - "timeout_seconds": { - "action": "skip", - "reason": "empty", - "remote": 0 - }, - "webhook_notifications": { - "action": "skip", - "reason": "empty", - "remote": {} - } - } - } - } + "storage_backend": "DEPLOYMENT_METADATA_SERVICE" } +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged + === The service denormalizes them onto the deployment, so a reader gets the provenance of the latest version >>> MSYS_NO_PATHCONV=1 [CLI] api get /api/2.0/bundle/deployments/[NUMID] { diff --git a/acceptance/bundle/dms/provenance/script b/acceptance/bundle/dms/provenance/script index ab20da12355..7ffc5636eab 100644 --- a/acceptance/bundle/dms/provenance/script +++ b/acceptance/bundle/dms/provenance/script @@ -9,7 +9,7 @@ add_repl.py "$(git rev-parse HEAD)" COMMIT trace print_requests.py --dms //versions title "The plan is a no-op and records DEPLOYMENT_METADATA_SERVICE as its backend" -trace $CLI bundle plan -o json +dms_plan title "The service denormalizes them onto the deployment, so a reader gets the provenance of the latest version" deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-provenance-${UNIQUE_NAME}/dev/state/resources.deployment.json" -o json | python3 -c 'import sys,json; print(json.load(sys.stdin)["object_id"])') diff --git a/acceptance/bundle/dms/record-failure/output.txt b/acceptance/bundle/dms/record-failure/output.txt index 7568ec343dd..c03c9cfda2a 100644 --- a/acceptance/bundle/dms/record-failure/output.txt +++ b/acceptance/bundle/dms/record-failure/output.txt @@ -18,44 +18,14 @@ Files: 6 uploaded, 0 deleted "deployment_id": "[NUMID]", "next_version_id": "2", "last_version_id": "1", - "storage_backend": "DEPLOYMENT_METADATA_SERVICE", - "plan": { - "resources.jobs.doomed": { - "action": "create", - "new_state": { - "value": { - "deployment": { - "deployment_id": "[NUMID]", - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-record-failure-[UNIQUE_NAME]/default/state/metadata.json", - "version_id": "2" - }, - "edit_mode": "UI_LOCKED", - "format": "MULTI_TASK", - "max_concurrent_runs": 1, - "name": "doomed", - "queue": { - "enabled": true - }, - "tasks": [ - { - "new_cluster": { - "node_type_id": "nonexistent-node-type", - "num_workers": 1, - "spark_version": "15.4.x-scala2.12" - }, - "notebook_task": { - "notebook_path": "/nonexistent" - }, - "task_key": "doomed" - } - ] - } - } - } - } + "storage_backend": "DEPLOYMENT_METADATA_SERVICE" } +>>> [CLI] bundle plan +create jobs.doomed + +Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged + >>> print_requests.py --dms //api/2.0/bundle { "method": "POST", diff --git a/acceptance/bundle/dms/record-failure/script b/acceptance/bundle/dms/record-failure/script index d3061bd3049..8ce1bb2696b 100644 --- a/acceptance/bundle/dms/record-failure/script +++ b/acceptance/bundle/dms/record-failure/script @@ -2,7 +2,7 @@ envsubst < databricks.yml.tmpl > databricks.yml title "A resource that fails to apply is recorded as a failed operation carrying the error, so the deployment history says why rather than omitting the resource" trace musterr $CLI bundle deploy -trace $CLI bundle plan -o json +dms_plan trace print_requests.py --dms //api/2.0/bundle title "The failed resource is not listed at all: state is what projects a resource, and a failed create records none, so a later deploy plans to create it rather than treating it as already deployed" diff --git a/acceptance/bundle/dms/record/output.txt b/acceptance/bundle/dms/record/output.txt index 263099877d5..f5ac8362644 100644 --- a/acceptance/bundle/dms/record/output.txt +++ b/acceptance/bundle/dms/record/output.txt @@ -93,60 +93,12 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "deployment_id": "[NUMID]", "next_version_id": "2", "last_version_id": "1", - "storage_backend": "DEPLOYMENT_METADATA_SERVICE", - "plan": { - "resources.jobs.foo": { - "action": "skip", - "remote_state": { - "created_time": [UNIX_TIME_MILLIS], - "creator_user_name": "[USERNAME]", - "deployment": { - "deployment_id": "[NUMID]", - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-record-[UNIQUE_NAME]/default/state/metadata.json", - "version_id": "1" - }, - "edit_mode": "UI_LOCKED", - "email_notifications": {}, - "format": "MULTI_TASK", - "job_id": [NUMID], - "max_concurrent_runs": 1, - "name": "foo", - "queue": { - "enabled": true - }, - "run_as_user_name": "[USERNAME]", - "timeout_seconds": 0, - "webhook_notifications": {} - }, - "changes": { - "deployment.version_id": { - "action": "skip", - "reason": "auto", - "old": "1", - "new": "2", - "remote": "1" - }, - "email_notifications": { - "action": "skip", - "reason": "empty", - "remote": {} - }, - "timeout_seconds": { - "action": "skip", - "reason": "empty", - "remote": 0 - }, - "webhook_notifications": { - "action": "skip", - "reason": "empty", - "remote": {} - } - } - } - } + "storage_backend": "DEPLOYMENT_METADATA_SERVICE" } +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged + === The service lists the job, recorded at version 1 >>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[NUMID] -o json [ diff --git a/acceptance/bundle/dms/record/script b/acceptance/bundle/dms/record/script index ccbcd025d9b..08b67befc6d 100644 --- a/acceptance/bundle/dms/record/script +++ b/acceptance/bundle/dms/record/script @@ -16,7 +16,7 @@ trace MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_U trace print_requests.py --get --dms //api/2.0/bundle title "The plan is a no-op now and records DEPLOYMENT_METADATA_SERVICE as its backend" -trace $CLI bundle plan -o json +dms_plan title "The service lists the job, recorded at version 1" deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-record-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') diff --git a/acceptance/bundle/dms/script.prepare b/acceptance/bundle/dms/script.prepare new file mode 100644 index 00000000000..f170821b4e8 --- /dev/null +++ b/acceptance/bundle/dms/script.prepare @@ -0,0 +1,9 @@ +# dms_plan prints the plan's header only (via jq del(.plan)), then the readable plan. +# The per-resource plan body carries remote_state that diverges between the fake and a +# real workspace, so the recording tests assert the header fields (cli_version, lineage, +# serial, deployment_id, next/last_version_id, storage_backend) and the human-readable +# summary instead of the full JSON. Pass through any flags, e.g. `dms_plan -t prod`. +dms_plan() { + trace $CLI bundle plan "$@" -o json | jq 'del(.plan)' + trace $CLI bundle plan "$@" +} diff --git a/acceptance/bundle/dms/stale-plan/output.txt b/acceptance/bundle/dms/stale-plan/output.txt index fa4f047b28e..f15171aa94e 100644 --- a/acceptance/bundle/dms/stale-plan/output.txt +++ b/acceptance/bundle/dms/stale-plan/output.txt @@ -4,29 +4,13 @@ { "plan_version": 3, "cli_version": "[CLI_VERSION]", - "storage_backend": "DEPLOYMENT_METADATA_SERVICE", - "plan": { - "resources.jobs.foo": { - "action": "create", - "new_state": { - "value": { - "deployment": { - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-stale-plan-[UNIQUE_NAME]/default/state/metadata.json", - "version_id": "1" - }, - "edit_mode": "UI_LOCKED", - "format": "MULTI_TASK", - "max_concurrent_runs": 1, - "name": "foo", - "queue": { - "enabled": true - } - } - } - } - } + "storage_backend": "DEPLOYMENT_METADATA_SERVICE" } + +>>> [CLI] bundle plan +create jobs.foo + +Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-stale-plan-[UNIQUE_NAME]/default/files... Created jobs.foo Files: 7 uploaded, 0 deleted diff --git a/acceptance/bundle/dms/stale-plan/script b/acceptance/bundle/dms/stale-plan/script index 3e189942281..bf7ae58d3f6 100644 --- a/acceptance/bundle/dms/stale-plan/script +++ b/acceptance/bundle/dms/stale-plan/script @@ -2,7 +2,7 @@ envsubst < databricks.yml.tmpl > databricks.yml title "Save a plan, then deploy it: records version 1" $CLI bundle plan -o json > tmp.plan.json -trace $CLI bundle plan -o json +dms_plan # READPLAN deploys from the saved raw plan; without it the plan is recomputed. Untraced so the # variant-dependent command line stays out of the golden (the deploy output is identical either way). $CLI bundle deploy $(readplanarg tmp.plan.json) diff --git a/acceptance/bundle/dms/successful-recreate/output.txt b/acceptance/bundle/dms/successful-recreate/output.txt index 69fbc28396b..db1033110d2 100644 --- a/acceptance/bundle/dms/successful-recreate/output.txt +++ b/acceptance/bundle/dms/successful-recreate/output.txt @@ -69,35 +69,12 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged "deployment_id": "[NUMID]", "next_version_id": "2", "last_version_id": "1", - "storage_backend": "DEPLOYMENT_METADATA_SERVICE", - "plan": { - "resources.schemas.foo": { - "action": "skip", - "remote_state": { - "browse_only": false, - "catalog_name": "main", - "catalog_type": "MANAGED_CATALOG", - "comment": "v1", - "created_at": [UNIX_TIME_MILLIS][0], - "created_by": "[USERNAME]", - "effective_predictive_optimization_flag": { - "inherited_from_name": "[METASTORE_NAME]", - "inherited_from_type": "METASTORE", - "value": "ENABLE" - }, - "enable_predictive_optimization": "INHERIT", - "full_name": "main.dms_successful_recreate_schema_[UNIQUE_NAME]", - "metastore_id": "[UUID]", - "name": "dms_successful_recreate_schema_[UNIQUE_NAME]", - "owner": "[USERNAME]", - "schema_id": "[UUID]", - "updated_at": [UNIX_TIME_MILLIS][0], - "updated_by": "[USERNAME]" - } - } - } + "storage_backend": "DEPLOYMENT_METADATA_SERVICE" } +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged + === The service lists the schema, recorded at version 1 >>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[NUMID] -o json [ @@ -196,35 +173,12 @@ Resources: 1 created, 0 changed, 1 deleted, 0 unchanged "deployment_id": "[NUMID]", "next_version_id": "3", "last_version_id": "2", - "storage_backend": "DEPLOYMENT_METADATA_SERVICE", - "plan": { - "resources.schemas.foo": { - "action": "skip", - "remote_state": { - "browse_only": false, - "catalog_name": "dms_other_[UNIQUE_NAME]", - "catalog_type": "MANAGED_CATALOG", - "comment": "v1", - "created_at": [UNIX_TIME_MILLIS][1], - "created_by": "[USERNAME]", - "effective_predictive_optimization_flag": { - "inherited_from_name": "[METASTORE_NAME]", - "inherited_from_type": "METASTORE", - "value": "ENABLE" - }, - "enable_predictive_optimization": "INHERIT", - "full_name": "dms_other_[UNIQUE_NAME].dms_successful_recreate_schema_[UNIQUE_NAME]", - "metastore_id": "[UUID]", - "name": "dms_successful_recreate_schema_[UNIQUE_NAME]", - "owner": "[USERNAME]", - "schema_id": "[UUID]", - "updated_at": [UNIX_TIME_MILLIS][1], - "updated_by": "[USERNAME]" - } - } - } + "storage_backend": "DEPLOYMENT_METADATA_SERVICE" } +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged + === The service lists the schema at version 2 with recreate action >>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[NUMID] -o json [ diff --git a/acceptance/bundle/dms/successful-recreate/script b/acceptance/bundle/dms/successful-recreate/script index 457164d584b..100441f9125 100644 --- a/acceptance/bundle/dms/successful-recreate/script +++ b/acceptance/bundle/dms/successful-recreate/script @@ -5,7 +5,7 @@ trace $CLI bundle deploy trace print_requests.py --dms //api/2.0/bundle title "The plan is a no-op now and records DEPLOYMENT_METADATA_SERVICE as its backend" -trace $CLI bundle plan -o json +dms_plan title "The service lists the schema, recorded at version 1" deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-successful-recreate-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') @@ -26,7 +26,7 @@ trace $CLI bundle deploy --auto-approve trace print_requests.py --dms //api/2.0/bundle title "The plan now shows the recreate action" -trace $CLI bundle plan -o json +dms_plan title "The service lists the schema at version 2 with recreate action" trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' diff --git a/acceptance/bundle/dms/version-never-created/output.txt b/acceptance/bundle/dms/version-never-created/output.txt index 11df2223dba..6a963c0ff82 100644 --- a/acceptance/bundle/dms/version-never-created/output.txt +++ b/acceptance/bundle/dms/version-never-created/output.txt @@ -17,31 +17,14 @@ Files: 5 uploaded, 0 deleted "cli_version": "[CLI_VERSION]", "deployment_id": "[NUMID]", "next_version_id": "1", - "storage_backend": "DEPLOYMENT_METADATA_SERVICE", - "plan": { - "resources.jobs.foo": { - "action": "create", - "new_state": { - "value": { - "deployment": { - "deployment_id": "[NUMID]", - "kind": "BUNDLE", - "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-version-never-created/default/state/metadata.json", - "version_id": "1" - }, - "edit_mode": "UI_LOCKED", - "format": "MULTI_TASK", - "max_concurrent_runs": 1, - "name": "foo", - "queue": { - "enabled": true - } - } - } - } - } + "storage_backend": "DEPLOYMENT_METADATA_SERVICE" } +>>> [CLI] bundle plan +create jobs.foo + +Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged + >>> MSYS_NO_PATHCONV=1 [CLI] workspace get-status /Workspace/Users/[USERNAME]/.bundle/dms-version-never-created/default/state/resources.deployment.json { "object_type": "BUNDLE_DEPLOYMENT" @@ -52,6 +35,8 @@ Files: 5 uploaded, 0 deleted {"method": "POST", "path": "/api/2.0/bundle/deployments/[NUMID]/versions", "q": {"version_id": "1"}, "body": {"cli_version": "[CLI_VERSION]", "version_type": "VERSION_TYPE_DEPLOY", "operations": [{"resource_key": "jobs.foo", "action_type": "OPERATION_ACTION_TYPE_CREATE"}]}} {"method": "GET", "path": "/api/2.0/bundle/deployments/[NUMID]"} {"method": "GET", "path": "/api/2.0/bundle/deployments/[NUMID]/resources"} +{"method": "GET", "path": "/api/2.0/bundle/deployments/[NUMID]"} +{"method": "GET", "path": "/api/2.0/bundle/deployments/[NUMID]/resources"} >>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[NUMID] -o json [] diff --git a/acceptance/bundle/dms/version-never-created/script b/acceptance/bundle/dms/version-never-created/script index d801153b1c7..18d3b71c3b4 100644 --- a/acceptance/bundle/dms/version-never-created/script +++ b/acceptance/bundle/dms/version-never-created/script @@ -1,6 +1,6 @@ title "The first version fails, so no deployment record exists - only the node naming its ID" trace musterr $CLI bundle deploy -trace $CLI bundle plan -o json +dms_plan trace MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-version-never-created/default/state/resources.deployment.json" | jq '{object_type}' trace print_requests.py --dms //api/2.0/bundle --get --oneline From d76883e4edddc616901797213e97d9aed9895277 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 7 Sep 2026 03:01:44 +0200 Subject: [PATCH 39/46] libs/dms: shorten record/RecordFailure comments, rename flag to keepExistingUpdateIfAny Trim the verbose doc comments to a couple of plain lines, and rename record's flag from isStateWrite to keepExistingUpdateIfAny (inverting it): true merges a failure onto the resource's last recorded state if one exists, false saves the update as that latest state. Co-authored-by: Isaac --- libs/dms/operation_buffer.go | 33 ++++++++++++------------------- libs/dms/operation_buffer_test.go | 6 +++--- 2 files changed, 16 insertions(+), 23 deletions(-) diff --git a/libs/dms/operation_buffer.go b/libs/dms/operation_buffer.go index 11fe07fc471..0718aac330c 100644 --- a/libs/dms/operation_buffer.go +++ b/libs/dms/operation_buffer.go @@ -76,35 +76,28 @@ func (b *OperationBuffer) RecordOperation(ctx context.Context, resourceKey strin return } - b.record(resourceKey, update, true, nil) + b.record(resourceKey, update, false, nil) } -// RecordFailure records that a resource did not apply, so the history says why rather than -// leaving the resource out. When a write already recorded this resource's state, the failure -// keeps that state - which the service may already hold - and changes only status and message; -// recording the pre-deploy state instead would overwrite the newer state, and after a rename -// pair the new id with the old state. state is the fallback, used only when no write recorded -// anything: the operation failed before it saved state. +// RecordFailure records that a resource did not apply, so the history says why. state is a +// fallback used only when no write recorded state yet; otherwise record keeps the last state. func (b *OperationBuffer) RecordFailure(resourceKey, resourceID string, state json.RawMessage, cause error) { // The passed update is the fallback used only when no write recorded state yet; record // otherwise merges the failure onto the state the last write recorded (see below). - b.record(resourceKey, NewFailureUpdate(resourceID, state, cause), false, cause) + b.record(resourceKey, NewFailureUpdate(resourceID, state, cause), true, cause) } -// record makes update the one waiting for resourceKey, waiting itself while the queue is -// full. isStateWrite records update as the latest state for this resource, kept past the -// writer taking it from pending so a later failure keeps it (see the latestState field); -// a failure (isStateWrite false) instead merges its status and message onto that recorded -// state when there is one, so it keeps the state - which the service may already hold - -// rather than the pre-deploy one, and after a rename pairs the new id with the old state. -// The latestState read/write shares this lock with the pending update so the two never -// disagree. Recording after Drain panics, so every caller must return before Drain. -func (b *OperationBuffer) record(resourceKey string, update OperationUpdate, isStateWrite bool, cause error) { +// record queues update for resourceKey. keepExistingUpdateIfAny (a failure) merges onto the +// resource's last recorded state if one exists; otherwise (a state write) update is saved as +// that latest state. latestState and pending are updated under the one lock. Panics after Drain. +func (b *OperationBuffer) record(resourceKey string, update OperationUpdate, keepExistingUpdateIfAny bool, cause error) { b.mu.Lock() - if isStateWrite { + if keepExistingUpdateIfAny { + if latest, recorded := b.latestState[resourceKey]; recorded { + update = latest.Merge(failureUpdate(cause)) + } + } else { b.latestState[resourceKey] = update - } else if latest, recorded := b.latestState[resourceKey]; recorded { - update = latest.Merge(failureUpdate(cause)) } waiting, queued := b.pending[resourceKey] if queued { diff --git a/libs/dms/operation_buffer_test.go b/libs/dms/operation_buffer_test.go index cb01c8b57d5..49af18c3255 100644 --- a/libs/dms/operation_buffer_test.go +++ b/libs/dms/operation_buffer_test.go @@ -38,8 +38,8 @@ func TestBufferCoalescesWhileAKeyIsPending(t *testing.T) { // Two writes for one resource with nothing draining. They carry the resource's full // state, so only the newest needs to go: one slot in the queue, not two. - s.record("resources.jobs.foo", stateUpdate(t, "v1"), true, nil) - s.record("resources.jobs.foo", stateUpdate(t, "v2"), true, nil) + s.record("resources.jobs.foo", stateUpdate(t, "v1"), false, nil) + s.record("resources.jobs.foo", stateUpdate(t, "v2"), false, nil) assert.Len(t, s.queue, 1) update, ok := s.take("resources.jobs.foo") @@ -48,7 +48,7 @@ func TestBufferCoalescesWhileAKeyIsPending(t *testing.T) { // Taken means a request has it, and an in-flight request cannot be recalled, so the next // write gets its own slot rather than joining it. - s.record("resources.jobs.foo", stateUpdate(t, "v3"), true, nil) + s.record("resources.jobs.foo", stateUpdate(t, "v3"), false, nil) assert.Len(t, s.queue, 2) update, ok = s.take("resources.jobs.foo") require.True(t, ok) From f8397653ee9b482de7d648995208100962c06b12 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 7 Sep 2026 03:08:37 +0200 Subject: [PATCH 40/46] acceptance/dms: bind test uses a real job so it records the DMS refusal Create a real workspace job and bind to it, so bind gets past fetching the resource and reaches the config gate that refuses bind while recording deployment history - recording that error instead of "Job 123 does not exist". Co-authored-by: Isaac --- .../dms/bind-unbind-unsupported/output.txt | 14 ++++--------- .../bundle/dms/bind-unbind-unsupported/script | 20 +++++++++---------- 2 files changed, 13 insertions(+), 21 deletions(-) diff --git a/acceptance/bundle/dms/bind-unbind-unsupported/output.txt b/acceptance/bundle/dms/bind-unbind-unsupported/output.txt index 484998ca590..891466bd2e5 100644 --- a/acceptance/bundle/dms/bind-unbind-unsupported/output.txt +++ b/acceptance/bundle/dms/bind-unbind-unsupported/output.txt @@ -1,22 +1,16 @@ -=== Before any deploy there is no recorded state: bind just fails to find the workspace resource, and unbind is a no-op ->>> musterr [CLI] bundle deployment bind foo 123 -Error: failed to fetch the resource, err: Job 123 does not exist. +=== bind is refused while recording deployment history +>>> musterr [CLI] bundle deployment bind foo [JOB_ID] +Error: bind is not supported for a bundle that records deployment history ->>> [CLI] bundle deployment unbind foo -=== Deploy so the bundle records the job with the deployment metadata service +=== Deploy so the bundle records state, then unbind is refused too >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-bind-unbind-[UNIQUE_NAME]/default/files... Created jobs.foo Files: 6 uploaded, 0 deleted Resources: 1 created, 0 changed, 0 deleted, 0 unchanged -=== With deployment history recorded, both bind and unbind are refused ->>> musterr [CLI] bundle deployment bind foo [FOO_ID] -Error: bind is not supported for a bundle that records deployment history - - >>> musterr [CLI] bundle deployment unbind foo Error: unbind is not supported for a bundle that records deployment history diff --git a/acceptance/bundle/dms/bind-unbind-unsupported/script b/acceptance/bundle/dms/bind-unbind-unsupported/script index 0bed434163a..9af766ca4d3 100644 --- a/acceptance/bundle/dms/bind-unbind-unsupported/script +++ b/acceptance/bundle/dms/bind-unbind-unsupported/script @@ -1,18 +1,16 @@ envsubst < databricks.yml.tmpl > databricks.yml -title "Before any deploy there is no recorded state: bind just fails to find the workspace resource, and unbind is a no-op" -trace musterr $CLI bundle deployment bind foo 123 -trace $CLI bundle deployment unbind foo +# The gate that refuses bind while recording deployment history is on the config, so it fires +# before any deploy. Create a real workspace job so bind gets past fetching the resource and +# reaches the gate, instead of failing because the resource does not exist. +title "bind is refused while recording deployment history" +job_id=$($CLI jobs create --json '{"name": "bind-target-'$UNIQUE_NAME'"}' | jq -r '.job_id') +add_repl "$job_id" JOB_ID +trace musterr $CLI bundle deployment bind foo "$job_id" -title "Deploy so the bundle records the job with the deployment metadata service" +# unbind refuses when it opens an existing recording state, so it needs a deployment first. +title "Deploy so the bundle records state, then unbind is refused too" trace $CLI bundle deploy - -# bind and unbind adopt or release a resource by editing local state directly. A recorded -# deployment keeps its resource set in the service, so both are refused: bind hits the gate -# after fetching the resource, unbind when it opens the recording state without recording. -title "With deployment history recorded, both bind and unbind are refused" -job_id=$(read_id.py foo) -trace musterr $CLI bundle deployment bind foo "$job_id" trace musterr $CLI bundle deployment unbind foo rm -f out.requests.txt From f8894d58caf111756a74bc13a338f83fd6a3991f Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 7 Sep 2026 03:10:32 +0200 Subject: [PATCH 41/46] acceptance/dms: rename emptied-resource to empty-grants, record the known bug, Cloud=false Emptying a grants node makes apply call DeleteState, recording a succeeded UPDATE that names no state while the schema still exists; a real workspace rejects that (400 INVALID_PARAMETER_VALUE). The fake does not model the check, so the recorded output is the buggy behavior. Disable the cloud run and document the fix needed (emptying grants should record an update carrying the emptied state, not a delete). Co-authored-by: Isaac --- .../databricks.yml.tmpl | 4 ++-- .../out.test.toml | 2 +- .../output.txt | 24 +++++++++---------- .../{emptied-resource => empty-grants}/script | 12 ++++++---- acceptance/bundle/dms/empty-grants/test.toml | 6 +++++ 5 files changed, 29 insertions(+), 19 deletions(-) rename acceptance/bundle/dms/{emptied-resource => empty-grants}/databricks.yml.tmpl (69%) rename acceptance/bundle/dms/{emptied-resource => empty-grants}/out.test.toml (83%) rename acceptance/bundle/dms/{emptied-resource => empty-grants}/output.txt (85%) rename acceptance/bundle/dms/{emptied-resource => empty-grants}/script (62%) create mode 100644 acceptance/bundle/dms/empty-grants/test.toml diff --git a/acceptance/bundle/dms/emptied-resource/databricks.yml.tmpl b/acceptance/bundle/dms/empty-grants/databricks.yml.tmpl similarity index 69% rename from acceptance/bundle/dms/emptied-resource/databricks.yml.tmpl rename to acceptance/bundle/dms/empty-grants/databricks.yml.tmpl index b8c21a3cc96..7be9f60129c 100644 --- a/acceptance/bundle/dms/emptied-resource/databricks.yml.tmpl +++ b/acceptance/bundle/dms/empty-grants/databricks.yml.tmpl @@ -1,11 +1,11 @@ bundle: - name: dms-emptied-resource-$UNIQUE_NAME + name: dms-empty-grants-$UNIQUE_NAME experimental: record_deployment_history: true resources: schemas: foo: - name: dms_emptied_resource_$UNIQUE_NAME + name: dms_empty_grants_$UNIQUE_NAME catalog_name: main grants: [{principal: deco-test-user@databricks.com, privileges: [USE_SCHEMA]}] diff --git a/acceptance/bundle/dms/emptied-resource/out.test.toml b/acceptance/bundle/dms/empty-grants/out.test.toml similarity index 83% rename from acceptance/bundle/dms/emptied-resource/out.test.toml rename to acceptance/bundle/dms/empty-grants/out.test.toml index 23c07f70dca..9921e91a794 100644 --- a/acceptance/bundle/dms/emptied-resource/out.test.toml +++ b/acceptance/bundle/dms/empty-grants/out.test.toml @@ -1,3 +1,3 @@ -Cloud = true +Cloud = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] EnvMatrix.DMS = ["true"] diff --git a/acceptance/bundle/dms/emptied-resource/output.txt b/acceptance/bundle/dms/empty-grants/output.txt similarity index 85% rename from acceptance/bundle/dms/emptied-resource/output.txt rename to acceptance/bundle/dms/empty-grants/output.txt index 5b70ee58e6d..12373101601 100644 --- a/acceptance/bundle/dms/emptied-resource/output.txt +++ b/acceptance/bundle/dms/empty-grants/output.txt @@ -1,10 +1,10 @@ === Deploy a schema with one grant: the grants node is recorded with its state >>> [CLI] bundle deploy -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-emptied-resource-[UNIQUE_NAME]/default/files... +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-empty-grants-[UNIQUE_NAME]/default/files... Created schemas.foo Created schemas.foo.grants -Files: 5 uploaded, 0 deleted +Files: 6 uploaded, 0 deleted Resources: 2 created, 0 changed, 0 deleted, 0 unchanged >>> print_requests.py --dms //api/2.0/bundle @@ -12,12 +12,12 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged "method": "POST", "path": "/api/2.0/bundle/deployments", "body": { - "display_name": "dms-emptied-resource-[UNIQUE_NAME]", - "initial_parent_path": "/Workspace/Users/[USERNAME]/.bundle/dms-emptied-resource-[UNIQUE_NAME]/default/state", + "display_name": "dms-empty-grants-[UNIQUE_NAME]", + "initial_parent_path": "/Workspace/Users/[USERNAME]/.bundle/dms-empty-grants-[UNIQUE_NAME]/default/state", "target_name": "default", "workspace_info": { - "file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-emptied-resource-[UNIQUE_NAME]/default/files", - "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-emptied-resource-[UNIQUE_NAME]/default" + "file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-empty-grants-[UNIQUE_NAME]/default/files", + "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-empty-grants-[UNIQUE_NAME]/default" } } } @@ -50,9 +50,9 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged }, "body": { "error_message": "", - "resource_id": "main.dms_emptied_resource_[UNIQUE_NAME]", + "resource_id": "main.dms_empty_grants_[UNIQUE_NAME]", "sequence_id": "0", - "state": "{\"state\":{\"catalog_name\":\"main\",\"name\":\"dms_emptied_resource_[UNIQUE_NAME]\"}}", + "state": "{\"state\":{\"catalog_name\":\"main\",\"name\":\"dms_empty_grants_[UNIQUE_NAME]\"}}", "status": "OPERATION_STATUS_SUCCEEDED" } } @@ -64,9 +64,9 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged }, "body": { "error_message": "", - "resource_id": "schema/main.dms_emptied_resource_[UNIQUE_NAME]", + "resource_id": "schema/main.dms_empty_grants_[UNIQUE_NAME]", "sequence_id": "0", - "state": "{\"state\":{\"securable_type\":\"schema\",\"full_name\":\"main.dms_emptied_resource_[UNIQUE_NAME]\",\"__embed__\":[{\"principal\":\"deco-test-user@databricks.com\",\"privileges\":[\"USE_SCHEMA\"]}]},\"depends_on\":[{\"node\":\"resources.schemas.foo\",\"label\":\"${resources.schemas.foo.id}\"}]}", + "state": "{\"state\":{\"securable_type\":\"schema\",\"full_name\":\"main.dms_empty_grants_[UNIQUE_NAME]\",\"__embed__\":[{\"principal\":\"deco-test-user@databricks.com\",\"privileges\":[\"USE_SCHEMA\"]}]},\"depends_on\":[{\"node\":\"resources.schemas.foo\",\"label\":\"${resources.schemas.foo.id}\"}]}", "status": "OPERATION_STATUS_SUCCEEDED" } } @@ -82,7 +82,7 @@ Resources: 2 created, 0 changed, 0 deleted, 0 unchanged >>> update_file.py databricks.yml grants: [{principal: deco-test-user@databricks.com, privileges: [USE_SCHEMA]}] grants: [] >>> [CLI] bundle deploy -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-emptied-resource-[UNIQUE_NAME]/default/files... +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-empty-grants-[UNIQUE_NAME]/default/files... Updated schemas.foo.grants Files: 3 uploaded, 0 deleted Resources: 0 created, 1 changed, 0 deleted, 1 unchanged @@ -176,6 +176,6 @@ The following resources will be deleted: This action will result in the deletion of the following UC schemas. Any underlying data may be lost: delete resources.schemas.foo -All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/dms-emptied-resource-[UNIQUE_NAME]/default +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/dms-empty-grants-[UNIQUE_NAME]/default Destroy: 1 deleted diff --git a/acceptance/bundle/dms/emptied-resource/script b/acceptance/bundle/dms/empty-grants/script similarity index 62% rename from acceptance/bundle/dms/emptied-resource/script rename to acceptance/bundle/dms/empty-grants/script index 49ff14b8a41..5b0abfe3c16 100644 --- a/acceptance/bundle/dms/emptied-resource/script +++ b/acceptance/bundle/dms/empty-grants/script @@ -8,9 +8,13 @@ title "Revoke the grant, so the grants node empties out" trace update_file.py databricks.yml 'grants: [{principal: deco-test-user@databricks.com, privileges: [USE_SCHEMA]}]' 'grants: []' trace $CLI bundle deploy -# The action type stays update: it is staged from the plan, before apply knows the state -# comes back empty. What drops the resource from the deployment is the update naming state -# with no value. +# KNOWN BUG (why this test is Cloud=false): emptying the grants node makes apply call +# DeleteState, which records the operation as a delete with no state. The staged action +# is still UPDATE, so the service sees a succeeded UPDATE that names no state while the +# schema (a live resource) still exists, and rejects it (400 INVALID_PARAMETER_VALUE). The +# fake does not model that check, so this passes locally; the recorded requests below are +# the buggy output. Fix: emptying grants should record an update carrying the emptied +# grants state, not a delete. trace print_requests.py --dms //api/2.0/bundle title "Plan again: reading state back from the service works and reports no work" @@ -20,7 +24,7 @@ title "The plan is a no-op and records DEPLOYMENT_METADATA_SERVICE as its backen dms_plan title "The grants node emptied out, so the service no longer lists it - only the schema remains" -deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-emptied-resource-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') +deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-empty-grants-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' title "The raw recorded state the service holds for this deployment" diff --git a/acceptance/bundle/dms/empty-grants/test.toml b/acceptance/bundle/dms/empty-grants/test.toml new file mode 100644 index 00000000000..fc548948325 --- /dev/null +++ b/acceptance/bundle/dms/empty-grants/test.toml @@ -0,0 +1,6 @@ +# KNOWN BUG (Cloud=false): emptying a grants node makes apply call DeleteState, which records +# the operation as a delete with no state. The staged action is still UPDATE, so the service +# rejects a succeeded UPDATE that names no state while the schema still exists. The fake does +# not model that check, so this passes locally but fails on a real workspace. Re-enable once +# emptying grants records an update carrying the emptied state instead of a delete. +Cloud = false From 4b01fa8088819208e8e1d6bbec7ac3d46088c849 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 7 Sep 2026 03:12:42 +0200 Subject: [PATCH 42/46] acceptance/dms: assert failed permissions update records its error via list-operations Add a bundle-deployments list-operations call after the failed permissions update so the golden asserts the failed operation (jobs.foo.permissions, UPDATE, FAILED) carries the 403 error message the service recorded. Co-authored-by: Isaac --- .../bundle/dms/failed-update-permissions/output.txt | 11 +++++++++++ .../bundle/dms/failed-update-permissions/script | 3 +++ 2 files changed, 14 insertions(+) diff --git a/acceptance/bundle/dms/failed-update-permissions/output.txt b/acceptance/bundle/dms/failed-update-permissions/output.txt index 8edf660d3f9..e7bf63ed164 100644 --- a/acceptance/bundle/dms/failed-update-permissions/output.txt +++ b/acceptance/bundle/dms/failed-update-permissions/output.txt @@ -82,6 +82,17 @@ Files: 3 uploaded, 0 deleted } ] +=== The failed operation (version 2) records the error message +>>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-operations deployments/[NUMID]/versions/2 -o json +[ + { + "action_type": "OPERATION_ACTION_TYPE_UPDATE", + "error_message": "updating id=/jobs/[NUMID]: Fault injected by test. (403 PERMISSION_DENIED)", + "resource_key": "jobs.foo.permissions", + "status": "OPERATION_STATUS_FAILED" + } +] + === The raw recorded state after the failure >>> print_state.py --no-dms { diff --git a/acceptance/bundle/dms/failed-update-permissions/script b/acceptance/bundle/dms/failed-update-permissions/script index 24a9f9ee45e..f9da3a3220f 100644 --- a/acceptance/bundle/dms/failed-update-permissions/script +++ b/acceptance/bundle/dms/failed-update-permissions/script @@ -24,6 +24,9 @@ title "The service lists the job with the failure" deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-failed-update-permissions-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' +title "The failed operation (version 2) records the error message" +trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-operations "deployments/${deployment_id}/versions/2" -o json | jq -S 'map({resource_key, action_type, status, error_message})' + title "The raw recorded state after the failure" trace print_state.py --no-dms From 7bf906ac6ee5c199d0066cbd6ab18527daa7efeb Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 7 Sep 2026 03:14:48 +0200 Subject: [PATCH 43/46] acceptance/dms: remove operation-upload-fails test Dropped as unnecessary; operation-upload-fails-once still covers the upload-failure path. Co-authored-by: Isaac --- .../databricks.yml.tmpl | 23 ------------------- .../dms/operation-upload-fails/out.test.toml | 3 --- .../dms/operation-upload-fails/output.txt | 4 ---- .../bundle/dms/operation-upload-fails/script | 9 -------- .../dms/operation-upload-fails/test.toml | 10 -------- 5 files changed, 49 deletions(-) delete mode 100644 acceptance/bundle/dms/operation-upload-fails/databricks.yml.tmpl delete mode 100644 acceptance/bundle/dms/operation-upload-fails/out.test.toml delete mode 100644 acceptance/bundle/dms/operation-upload-fails/output.txt delete mode 100644 acceptance/bundle/dms/operation-upload-fails/script delete mode 100644 acceptance/bundle/dms/operation-upload-fails/test.toml diff --git a/acceptance/bundle/dms/operation-upload-fails/databricks.yml.tmpl b/acceptance/bundle/dms/operation-upload-fails/databricks.yml.tmpl deleted file mode 100644 index b6aa93265df..00000000000 --- a/acceptance/bundle/dms/operation-upload-fails/databricks.yml.tmpl +++ /dev/null @@ -1,23 +0,0 @@ -bundle: - name: dms-operation-upload-fails-$UNIQUE_NAME -experimental: - record_deployment_history: true - -resources: - jobs: - one: - name: one - two: - name: two - three: - name: three - four: - name: four - five: - name: five - six: - name: six - seven: - name: seven - eight: - name: eight diff --git a/acceptance/bundle/dms/operation-upload-fails/out.test.toml b/acceptance/bundle/dms/operation-upload-fails/out.test.toml deleted file mode 100644 index 9921e91a794..00000000000 --- a/acceptance/bundle/dms/operation-upload-fails/out.test.toml +++ /dev/null @@ -1,3 +0,0 @@ -Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -EnvMatrix.DMS = ["true"] diff --git a/acceptance/bundle/dms/operation-upload-fails/output.txt b/acceptance/bundle/dms/operation-upload-fails/output.txt deleted file mode 100644 index 2b273972417..00000000000 --- a/acceptance/bundle/dms/operation-upload-fails/output.txt +++ /dev/null @@ -1,4 +0,0 @@ - -=== An operation upload failure fails the deploy instead of reporting only at the end ->>> grep -c ^Error: LOG.deploy -deploy reported errors diff --git a/acceptance/bundle/dms/operation-upload-fails/script b/acceptance/bundle/dms/operation-upload-fails/script deleted file mode 100644 index cc2ead92c6f..00000000000 --- a/acceptance/bundle/dms/operation-upload-fails/script +++ /dev/null @@ -1,9 +0,0 @@ -envsubst < databricks.yml.tmpl > databricks.yml - -title "An operation upload failure fails the deploy instead of reporting only at the end" -# Which resources get refused depends on how far apply got before a background -# upload failed, so the per-resource errors go to a LOG file rather than the diff. -fault.py "PATCH /api/2.0/bundle/*" 500 0 99 INTERNAL_ERROR -errcode $CLI bundle deploy &> LOG.deploy -contains.py 'recording operation for' '!panic' < LOG.deploy > /dev/null -trace grep -c "^Error:" LOG.deploy > /dev/null && echo "deploy reported errors" diff --git a/acceptance/bundle/dms/operation-upload-fails/test.toml b/acceptance/bundle/dms/operation-upload-fails/test.toml deleted file mode 100644 index b5502a2516c..00000000000 --- a/acceptance/bundle/dms/operation-upload-fails/test.toml +++ /dev/null @@ -1,10 +0,0 @@ -# Local only: this case needs an injected failure - DMS itself must reject the operation upload - and fault.py registers -# its rule on the fake, which a cloud run does not have. -Cloud = false - -# Which requests are made depends on how far apply got before a background upload -# failed, so recording them would make the output nondeterministic. -RecordRequests = false - -# Completed versions make DMS the source of truth; unrecorded resources get recreated. -# Deploy must stop rather than continue creating. From bdcc887138498fc7cc74272854c38405b435ea65 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 7 Sep 2026 03:19:05 +0200 Subject: [PATCH 44/46] acceptance/dms: remove redundant record and failed-update tests Both duplicate coverage in the remaining bundle/dms tests: recording calls and serialized operation state are exercised by successful-recreate, multiple-resources, depends-on and others; a failed update is covered by failed-update-permissions and failed-recreate. Reword the two comments that pointed at bundle/dms/record. Co-authored-by: Isaac --- .../dms/failed-update/databricks.yml.tmpl | 11 -- .../bundle/dms/failed-update/out.test.toml | 3 - .../bundle/dms/failed-update/output.txt | 185 ------------------ acceptance/bundle/dms/failed-update/script | 37 ---- acceptance/bundle/dms/failed-update/test.toml | 3 - .../bundle/dms/multiple-resources/output.txt | 2 +- .../bundle/dms/multiple-resources/script | 2 +- .../bundle/dms/record/databricks.yml.tmpl | 9 - acceptance/bundle/dms/record/out.test.toml | 3 - acceptance/bundle/dms/record/output.txt | 123 ------------ acceptance/bundle/dms/record/script | 27 --- acceptance/bundle/test.toml | 2 +- 12 files changed, 3 insertions(+), 404 deletions(-) delete mode 100644 acceptance/bundle/dms/failed-update/databricks.yml.tmpl delete mode 100644 acceptance/bundle/dms/failed-update/out.test.toml delete mode 100644 acceptance/bundle/dms/failed-update/output.txt delete mode 100644 acceptance/bundle/dms/failed-update/script delete mode 100644 acceptance/bundle/dms/failed-update/test.toml delete mode 100644 acceptance/bundle/dms/record/databricks.yml.tmpl delete mode 100644 acceptance/bundle/dms/record/out.test.toml delete mode 100644 acceptance/bundle/dms/record/output.txt delete mode 100644 acceptance/bundle/dms/record/script diff --git a/acceptance/bundle/dms/failed-update/databricks.yml.tmpl b/acceptance/bundle/dms/failed-update/databricks.yml.tmpl deleted file mode 100644 index 87cfcff9f36..00000000000 --- a/acceptance/bundle/dms/failed-update/databricks.yml.tmpl +++ /dev/null @@ -1,11 +0,0 @@ -bundle: - name: dms-failed-update-$UNIQUE_NAME -experimental: - record_deployment_history: true - -resources: - schemas: - foo: - name: dms_failed_update_schema_$UNIQUE_NAME - catalog_name: main - comment: v1 diff --git a/acceptance/bundle/dms/failed-update/out.test.toml b/acceptance/bundle/dms/failed-update/out.test.toml deleted file mode 100644 index 9921e91a794..00000000000 --- a/acceptance/bundle/dms/failed-update/out.test.toml +++ /dev/null @@ -1,3 +0,0 @@ -Cloud = false -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -EnvMatrix.DMS = ["true"] diff --git a/acceptance/bundle/dms/failed-update/output.txt b/acceptance/bundle/dms/failed-update/output.txt deleted file mode 100644 index bd837caeda2..00000000000 --- a/acceptance/bundle/dms/failed-update/output.txt +++ /dev/null @@ -1,185 +0,0 @@ - -=== Deploy the schema, so there is an existing resource to update ->>> [CLI] bundle deploy -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-failed-update-[UNIQUE_NAME]/default/files... -Created schemas.foo -Files: 6 uploaded, 0 deleted -Resources: 1 created, 0 changed, 0 deleted, 0 unchanged - ->>> print_requests.py --dms //api/2.0/bundle -{ - "method": "POST", - "path": "/api/2.0/bundle/deployments", - "body": { - "display_name": "dms-failed-update-[UNIQUE_NAME]", - "initial_parent_path": "/Workspace/Users/[USERNAME]/.bundle/dms-failed-update-[UNIQUE_NAME]/default/state", - "target_name": "default", - "workspace_info": { - "file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-failed-update-[UNIQUE_NAME]/default/files", - "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-failed-update-[UNIQUE_NAME]/default" - } - } -} -{ - "method": "POST", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions", - "q": { - "version_id": "1" - }, - "body": { - "cli_version": "[CLI_VERSION]", - "version_type": "VERSION_TYPE_DEPLOY", - "operations": [ - { - "resource_key": "schemas.foo", - "action_type": "OPERATION_ACTION_TYPE_CREATE" - } - ] - } -} -{ - "method": "PATCH", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/schemas.foo", - "q": { - "update_mask": "state,error_message,resource_id,status" - }, - "body": { - "error_message": "", - "resource_id": "main.dms_failed_update_schema_[UNIQUE_NAME]", - "sequence_id": "0", - "state": "{\"state\":{\"catalog_name\":\"main\",\"comment\":\"v1\",\"name\":\"dms_failed_update_schema_[UNIQUE_NAME]\"}}", - "status": "OPERATION_STATUS_SUCCEEDED" - } -} -{ - "method": "POST", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/complete", - "body": { - "completion_reason": "VERSION_COMPLETE_SUCCESS" - } -} - -=== The plan is a no-op and records DEPLOYMENT_METADATA_SERVICE as its backend ->>> [CLI] bundle plan -o json -{ - "plan_version": 3, - "cli_version": "[CLI_VERSION]", - "lineage": "[UUID]", - "serial": 1, - "deployment_id": "[NUMID]", - "next_version_id": "2", - "last_version_id": "1", - "storage_backend": "DEPLOYMENT_METADATA_SERVICE" -} - ->>> [CLI] bundle plan -Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged - -=== After deploy, the service lists the resource at version 1 ->>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[NUMID] -o json -[ - { - "last_action_type": "OPERATION_ACTION_TYPE_CREATE", - "last_version_id": "1", - "resource_key": "schemas.foo" - } -] - -=== The raw state after successful deploy ->>> print_state.py --no-dms -{ - "state_version": 2, - "cli_version": "[CLI_VERSION]", - "lineage": "[UUID]", - "serial": 1, - "features": { - "record_deployment_history": {} - }, - "state": {} -} - -=== An update that fails before writing any state only marks its operation failed. It names no state, so the deployment keeps describing the schema the last successful version recorded - which is right, because nothing touched it ->>> update_file.py databricks.yml comment: v1 comment: v2 - ->>> fault.py PATCH /api/2.1/unity-catalog/schemas/* 400 0 1 INVALID_PARAMETER_VALUE - ->>> musterr [CLI] bundle deploy --auto-approve -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-failed-update-[UNIQUE_NAME]/default/files... -Error: cannot update resources.schemas.foo: updating id=main.dms_failed_update_schema_[UNIQUE_NAME]: Fault injected by test. (400 INVALID_PARAMETER_VALUE) - -Endpoint: PATCH [DATABRICKS_URL]/api/2.1/unity-catalog/schemas/main.dms_failed_update_schema_[UNIQUE_NAME] -HTTP Status: 400 Bad Request -API error_code: INVALID_PARAMETER_VALUE -API message: Fault injected by test. - -Files: 3 uploaded, 0 deleted - ->>> print_requests.py --dms //api/2.0/bundle -{ - "method": "POST", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions", - "q": { - "version_id": "2" - }, - "body": { - "cli_version": "[CLI_VERSION]", - "version_type": "VERSION_TYPE_DEPLOY", - "previous_version_id": "1", - "operations": [ - { - "resource_key": "schemas.foo", - "action_type": "OPERATION_ACTION_TYPE_UPDATE" - } - ] - } -} -{ - "method": "PATCH", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/operations/schemas.foo", - "q": { - "update_mask": "state,error_message,resource_id,status" - }, - "body": { - "error_message": "updating id=main.dms_failed_update_schema_[UNIQUE_NAME]: Fault injected by test. (400 INVALID_PARAMETER_VALUE)", - "resource_id": "main.dms_failed_update_schema_[UNIQUE_NAME]", - "sequence_id": "0", - "state": "{\"state\":{\"catalog_name\":\"main\",\"comment\":\"v1\",\"name\":\"dms_failed_update_schema_[UNIQUE_NAME]\"}}", - "status": "OPERATION_STATUS_FAILED" - } -} -{ - "method": "POST", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/2/complete", - "body": { - "completion_reason": "VERSION_COMPLETE_FAILURE" - } -} - -=== The schema is still listed as the previous version left it, so the next plan updates it rather than creating a second one ->>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[NUMID] -o json -[ - { - "last_action_type": "OPERATION_ACTION_TYPE_UPDATE", - "last_version_id": "2", - "resource_key": "schemas.foo" - } -] - -=== The raw state after failed update shows the previous version ->>> print_state.py --no-dms -{ - "state_version": 2, - "cli_version": "[CLI_VERSION]", - "lineage": "[UUID]", - "serial": 1, - "features": { - "record_deployment_history": {} - }, - "state": {} -} - -=== Planning from DMS state alone updates the schema rather than creating a second one, which is what recording the prior state buys ->>> [CLI] bundle plan -update schemas.foo - -Plan: 0 to add, 1 to change, 0 to delete, 0 unchanged diff --git a/acceptance/bundle/dms/failed-update/script b/acceptance/bundle/dms/failed-update/script deleted file mode 100644 index a956ed2fa5b..00000000000 --- a/acceptance/bundle/dms/failed-update/script +++ /dev/null @@ -1,37 +0,0 @@ -envsubst < databricks.yml.tmpl > databricks.yml - -title "Deploy the schema, so there is an existing resource to update" -trace $CLI bundle deploy -trace print_requests.py --dms //api/2.0/bundle - -title "The plan is a no-op and records DEPLOYMENT_METADATA_SERVICE as its backend" -dms_plan - -title "After deploy, the service lists the resource at version 1" -deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-failed-update-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') -trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' - -title "The raw state after successful deploy" -trace print_state.py --no-dms - -title "An update that fails before writing any state only marks its operation failed. It names no state, so the deployment keeps describing the schema the last successful version recorded - which is right, because nothing touched it" -trace update_file.py databricks.yml "comment: v1" "comment: v2" -# Fail the update call itself, so the deploy never writes state for the schema. -trace fault.py "PATCH /api/2.1/unity-catalog/schemas/*" 400 0 1 INVALID_PARAMETER_VALUE -trace musterr $CLI bundle deploy --auto-approve -# Not sorted: the point of this test is the order the operations are recorded in. -trace print_requests.py --dms //api/2.0/bundle - -title "The schema is still listed as the previous version left it, so the next plan updates it rather than creating a second one" -deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-failed-update-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') -trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' - -title "The raw state after failed update shows the previous version" -trace print_state.py --no-dms - -title "Planning from DMS state alone updates the schema rather than creating a second one, which is what recording the prior state buys" -rm -rf .databricks -trace $CLI bundle plan -# plan probes the state files concurrently, so the recorded order varies. Only the -# operations above are asserted; drop the rest rather than diff a racy order. -rm -f out.requests.txt diff --git a/acceptance/bundle/dms/failed-update/test.toml b/acceptance/bundle/dms/failed-update/test.toml deleted file mode 100644 index 2b4eeb1bb04..00000000000 --- a/acceptance/bundle/dms/failed-update/test.toml +++ /dev/null @@ -1,3 +0,0 @@ -# Local only: this case needs an injected failure - no schema update that Unity Catalog rejects outright - and fault.py registers -# its rule on the fake, which a cloud run does not have. -Cloud = false diff --git a/acceptance/bundle/dms/multiple-resources/output.txt b/acceptance/bundle/dms/multiple-resources/output.txt index aee36fb25dc..bf9a24cba12 100644 --- a/acceptance/bundle/dms/multiple-resources/output.txt +++ b/acceptance/bundle/dms/multiple-resources/output.txt @@ -1,5 +1,5 @@ -=== Deploy several resources: operations are uploaded from background workers, so exactly one is recorded per resource no matter what order the uploads finish in. The serialized state is dropped from the output here; bundle/dms/record covers it. +=== Deploy several resources: operations are uploaded from background workers, so exactly one is recorded per resource no matter what order the uploads finish in. The serialized state is dropped from the output here; other bundle/dms tests cover it. >>> [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-multiple-resources-[UNIQUE_NAME]/default/files... Created jobs.five diff --git a/acceptance/bundle/dms/multiple-resources/script b/acceptance/bundle/dms/multiple-resources/script index 5ce8d484f83..8177c0be601 100644 --- a/acceptance/bundle/dms/multiple-resources/script +++ b/acceptance/bundle/dms/multiple-resources/script @@ -1,6 +1,6 @@ envsubst < databricks.yml.tmpl > databricks.yml -title "Deploy several resources: operations are uploaded from background workers, so exactly one is recorded per resource no matter what order the uploads finish in. The serialized state is dropped from the output here; bundle/dms/record covers it." +title "Deploy several resources: operations are uploaded from background workers, so exactly one is recorded per resource no matter what order the uploads finish in. The serialized state is dropped from the output here; other bundle/dms tests cover it." trace $CLI bundle deploy trace print_requests.py --dms //versions/1/operations --sort --del-body state --oneline diff --git a/acceptance/bundle/dms/record/databricks.yml.tmpl b/acceptance/bundle/dms/record/databricks.yml.tmpl deleted file mode 100644 index 214793eca38..00000000000 --- a/acceptance/bundle/dms/record/databricks.yml.tmpl +++ /dev/null @@ -1,9 +0,0 @@ -bundle: - name: dms-record-$UNIQUE_NAME -experimental: - record_deployment_history: true - -resources: - jobs: - foo: - name: foo diff --git a/acceptance/bundle/dms/record/out.test.toml b/acceptance/bundle/dms/record/out.test.toml deleted file mode 100644 index 23c07f70dca..00000000000 --- a/acceptance/bundle/dms/record/out.test.toml +++ /dev/null @@ -1,3 +0,0 @@ -Cloud = true -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] -EnvMatrix.DMS = ["true"] diff --git a/acceptance/bundle/dms/record/output.txt b/acceptance/bundle/dms/record/output.txt deleted file mode 100644 index f5ac8362644..00000000000 --- a/acceptance/bundle/dms/record/output.txt +++ /dev/null @@ -1,123 +0,0 @@ - -=== Deploy, then destroy: set up a deployment and delete it, so the deploy below starts from nothing ->>> [CLI] bundle deploy -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-record-[UNIQUE_NAME]/default/files... -Created jobs.foo -Files: 5 uploaded, 0 deleted -Resources: 1 created, 0 changed, 0 deleted, 0 unchanged - ->>> [CLI] bundle destroy --auto-approve -The following resources will be deleted: - delete resources.jobs.foo - -All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/dms-record-[UNIQUE_NAME]/default - -Destroy: 1 deleted - -=== Deploy again: the destroy took the workspace node with it, so there is nothing to resolve and a fresh deployment starts at version 1, not a continuation of the deleted one ->>> MSYS_NO_PATHCONV=1 musterr [CLI] workspace get-status /Workspace/Users/[USERNAME]/.bundle/dms-record-[UNIQUE_NAME]/default/state/resources.deployment.json -Error: Path (/Workspace/Users/[USERNAME]/.bundle/dms-record-[UNIQUE_NAME]/default/state/resources.deployment.json) doesn't exist. - ->>> [CLI] bundle deploy -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dms-record-[UNIQUE_NAME]/default/files... -Created jobs.foo -Files: 5 uploaded, 0 deleted -Resources: 1 created, 0 changed, 0 deleted, 0 unchanged - ->>> MSYS_NO_PATHCONV=1 [CLI] workspace get-status /Workspace/Users/[USERNAME]/.bundle/dms-record-[UNIQUE_NAME]/default/state/resources.deployment.json -{ - "object_type": "BUNDLE_DEPLOYMENT", - "path": "/Workspace/Users/[USERNAME]/.bundle/dms-record-[UNIQUE_NAME]/default/state/resources.deployment.json" -} - ->>> print_requests.py --get --dms //api/2.0/bundle -{ - "method": "POST", - "path": "/api/2.0/bundle/deployments", - "body": { - "display_name": "dms-record-[UNIQUE_NAME]", - "initial_parent_path": "/Workspace/Users/[USERNAME]/.bundle/dms-record-[UNIQUE_NAME]/default/state", - "target_name": "default", - "workspace_info": { - "file_path": "/Workspace/Users/[USERNAME]/.bundle/dms-record-[UNIQUE_NAME]/default/files", - "root_path": "/Workspace/Users/[USERNAME]/.bundle/dms-record-[UNIQUE_NAME]/default" - } - } -} -{ - "method": "POST", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions", - "q": { - "version_id": "1" - }, - "body": { - "cli_version": "[CLI_VERSION]", - "version_type": "VERSION_TYPE_DEPLOY", - "operations": [ - { - "resource_key": "jobs.foo", - "action_type": "OPERATION_ACTION_TYPE_CREATE" - } - ] - } -} -{ - "method": "PATCH", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/operations/jobs.foo", - "q": { - "update_mask": "state,error_message,resource_id,status" - }, - "body": { - "error_message": "", - "resource_id": "[NUMID]", - "sequence_id": "0", - "state": "{\"state\":{\"deployment\":{\"deployment_id\":\"[NUMID]\",\"kind\":\"BUNDLE\",\"metadata_file_path\":\"/Workspace/Users/[USERNAME]/.bundle/dms-record-[UNIQUE_NAME]/default/state/metadata.json\",\"version_id\":\"1\"},\"edit_mode\":\"UI_LOCKED\",\"format\":\"MULTI_TASK\",\"max_concurrent_runs\":1,\"name\":\"foo\",\"queue\":{\"enabled\":true}}}", - "status": "OPERATION_STATUS_SUCCEEDED" - } -} -{ - "method": "POST", - "path": "/api/2.0/bundle/deployments/[NUMID]/versions/1/complete", - "body": { - "completion_reason": "VERSION_COMPLETE_SUCCESS" - } -} - -=== The plan is a no-op now and records DEPLOYMENT_METADATA_SERVICE as its backend ->>> [CLI] bundle plan -o json -{ - "plan_version": 3, - "cli_version": "[CLI_VERSION]", - "lineage": "[UUID]", - "serial": 3, - "deployment_id": "[NUMID]", - "next_version_id": "2", - "last_version_id": "1", - "storage_backend": "DEPLOYMENT_METADATA_SERVICE" -} - ->>> [CLI] bundle plan -Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged - -=== The service lists the job, recorded at version 1 ->>> MSYS_NO_PATHCONV=1 [CLI] bundle-deployments list-resources deployments/[NUMID] -o json -[ - { - "last_action_type": "OPERATION_ACTION_TYPE_CREATE", - "last_version_id": "1", - "resource_key": "jobs.foo" - } -] - -=== The raw recorded state the service holds for this deployment ->>> print_state.py --no-dms -{ - "state_version": 2, - "cli_version": "[CLI_VERSION]", - "lineage": "[UUID]", - "serial": 3, - "features": { - "record_deployment_history": {} - }, - "state": {} -} diff --git a/acceptance/bundle/dms/record/script b/acceptance/bundle/dms/record/script deleted file mode 100644 index 08b67befc6d..00000000000 --- a/acceptance/bundle/dms/record/script +++ /dev/null @@ -1,27 +0,0 @@ -envsubst < databricks.yml.tmpl > databricks.yml - -title "Deploy, then destroy: set up a deployment and delete it, so the deploy below starts from nothing" -trace $CLI bundle deploy -trace $CLI bundle destroy --auto-approve - -# Discard the deploy+destroy requests above; this test is only about the deploy that FOLLOWS a -# destroy. The individual deploy and destroy request sequences are covered by no-resources, -# successful-recreate, and multiple-resources. -rm -f out.requests.txt - -title "Deploy again: the destroy took the workspace node with it, so there is nothing to resolve and a fresh deployment starts at version 1, not a continuation of the deleted one" -trace MSYS_NO_PATHCONV=1 musterr $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-record-${UNIQUE_NAME}/default/state/resources.deployment.json" -trace $CLI bundle deploy -trace MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-record-${UNIQUE_NAME}/default/state/resources.deployment.json" | jq '{object_type,path}' -trace print_requests.py --get --dms //api/2.0/bundle - -title "The plan is a no-op now and records DEPLOYMENT_METADATA_SERVICE as its backend" -dms_plan - -title "The service lists the job, recorded at version 1" -deployment_id=$(MSYS_NO_PATHCONV=1 $CLI workspace get-status "/Workspace/Users/${CURRENT_USER_NAME}/.bundle/dms-record-${UNIQUE_NAME}/default/state/resources.deployment.json" | python3 -c 'import sys, json; print(json.load(sys.stdin)["object_id"])') -trace MSYS_NO_PATHCONV=1 $CLI bundle-deployments list-resources "deployments/${deployment_id}" -o json | jq -S 'map({resource_key, last_version_id, last_action_type})' - -title "The raw recorded state the service holds for this deployment" -trace print_state.py --no-dms -rm -f out.requests.txt diff --git a/acceptance/bundle/test.toml b/acceptance/bundle/test.toml index 2db98f4b429..5f9b6281331 100644 --- a/acceptance/bundle/test.toml +++ b/acceptance/bundle/test.toml @@ -20,7 +20,7 @@ Env.UV_PYTHON = "3.10" # The link a recorded deploy prints, dropped so a test asserts the same output whether or # not recording is on. The URL itself is covered by workspaceurls.TestDeploymentURL, and -# the calls behind it by bundle/dms/record. +# the calls behind it by the bundle/dms tests. [[Repls]] Old = '(?m)^Current Deployment Version: .*\n' New = '' From cc4d4cc36052e7cc43c3471a21f3b77a25566c8b Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 7 Sep 2026 03:38:17 +0200 Subject: [PATCH 45/46] libs/testserver: gofmt fake_workspace.go after seeding the main catalog Co-authored-by: Isaac --- libs/testserver/fake_workspace.go | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/libs/testserver/fake_workspace.go b/libs/testserver/fake_workspace.go index c9026e20fc2..f9e1414a237 100644 --- a/libs/testserver/fake_workspace.go +++ b/libs/testserver/fake_workspace.go @@ -458,15 +458,15 @@ func NewFakeWorkspace(url, token string) *FakeWorkspace { files: make(map[string]FileEntry), repoIdByPath: make(map[string]int64), - Jobs: map[int64]jobs.Job{}, - JobRuns: map[int64]jobs.Run{}, - JobRunOutputs: map[int64]jobs.RunOutput{}, - JobRunIdempotency: map[string]int64{}, - Grants: map[string][]catalog.PrivilegeAssignment{}, - Pipelines: map[string]pipelines.GetPipelineResponse{}, - PipelineUpdates: map[string]bool{}, - Monitors: map[string]catalog.MonitorInfo{}, - Apps: map[string]apps.App{}, + Jobs: map[int64]jobs.Job{}, + JobRuns: map[int64]jobs.Run{}, + JobRunOutputs: map[int64]jobs.RunOutput{}, + JobRunIdempotency: map[string]int64{}, + Grants: map[string][]catalog.PrivilegeAssignment{}, + Pipelines: map[string]pipelines.GetPipelineResponse{}, + PipelineUpdates: map[string]bool{}, + Monitors: map[string]catalog.MonitorInfo{}, + Apps: map[string]apps.App{}, // Seed the "main" catalog: it exists on every real workspace, and schema // creation now checks the catalog exists, so tests that deploy schemas into // main (the common case) keep working without creating it first. From 0336c85cec640cb396e69ff06095728c7706fa1f Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Mon, 7 Sep 2026 03:51:11 +0200 Subject: [PATCH 46/46] bundle/dstate: nullify local state when DMS recording is enabled When opening a deployment state with experimental.record_deployment_history enabled, the service is the source of truth for resources, not the local state file. Nullify the local state so it acts as a tombstone (carrying only the feature marker and header). This allows stale local state from a destroyed DMS deployment to bootstrap a fresh deployment, instead of erroring with "this deployment already exists and is not recorded". The fix removes the guard that previously prevented enabling DMS on deployments with local resources. This is a necessary tradeoff to support the common scenario where a DMS deployment is destroyed on the service but the local state cache remains. The service is now authoritative: - If dmsDeploymentID is non-empty: ListResources fetches the service's resources - If dmsDeploymentID is empty: no resources are loaded (fresh/destroyed deployment) Tradeoff: this also allows enabling DMS on an existing non-DMS deployment, which could leave old resources orphaned if they have different IDs or names. Users should destroy a non-DMS deployment before enabling DMS to avoid this scenario. Accept this tradeoff to unblock the more common case (destroyed DMS bootstrap). Co-authored-by: Isaac --- bundle/direct/dstate/state.go | 11 ++++++++++ bundle/direct/dstate/state_test.go | 34 ++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+) diff --git a/bundle/direct/dstate/state.go b/bundle/direct/dstate/state.go index 6a41661c516..3d144ec9bb2 100644 --- a/bundle/direct/dstate/state.go +++ b/bundle/direct/dstate/state.go @@ -520,6 +520,17 @@ func (db *DeploymentState) unlockedOpen(ctx context.Context, path string, withRe recording := bool(withDeploymentHistory) + // When recording (DMS configured), the service is the source of truth, not the local + // state file. Nullify the local state so it acts as a tombstone (carrying only the + // feature marker and header). This allows a stale local state from a destroyed DMS + // deployment to bootstrap a fresh one, rather than erroring on the guard below. + // The service is queried via ListResources (if dmsDeploymentID is non-empty) or + // is assumed empty (if dmsDeploymentID is empty). + if recording { + db.Data.State = make(map[string]ResourceEntry) + db.stateIDs = make(map[string]string) + } + walPath := db.Path + walSuffix _, err = os.Stat(walPath) switch { diff --git a/bundle/direct/dstate/state_test.go b/bundle/direct/dstate/state_test.go index 57ccd4d3e51..59508f8e891 100644 --- a/bundle/direct/dstate/state_test.go +++ b/bundle/direct/dstate/state_test.go @@ -281,3 +281,37 @@ func TestOpenFailureLeavesStateClosed(t *testing.T) { assert.Equal(t, "test-lineage", db.Data.Lineage) mustFinalize(t, &db) } + +// TestNullifyLocalStateWhenRecordingEnabled verifies that when a state is opened +// with recording enabled (WithDeploymentHistory=true), the local state is nullified. +// This prevents the error that would occur when `recording && !recorded && len(db.Data.State) > 0`. +// The fix ensures a stale local state does not block recording from being bootstrapped +// on a destroyed DMS deployment. +func TestNullifyLocalStateWhenRecordingEnabled(t *testing.T) { + path := filepath.Join(t.TempDir(), "state.json") + + // First, create and save a state with resources but no DMS feature marker. + // This simulates a non-recorded deployment or stale state from a destroyed deployment. + var db DeploymentState + require.NoError(t, db.Open(t.Context(), path, WithRecovery(true), WithWrite(true), WithDeploymentHistory(false), "")) + require.NoError(t, db.SaveState(t.Context(), "resources.jobs.my_job", "123", map[string]string{"key": "value"}, nil)) + mustFinalize(t, &db) + + // Verify the state file was persisted with the resource. + var db2 DeploymentState + require.NoError(t, db2.Open(t.Context(), path, WithRecovery(false), WithWrite(false), WithDeploymentHistory(false), "")) + assert.NotEmpty(t, db2.Data.State, "initial state should have the saved resource") + assert.Equal(t, "123", db2.GetResourceID("resources.jobs.my_job")) + mustFinalize(t, &db2) + + // Now open the same state with recording disabled to verify no feature marker. + var db3 DeploymentState + require.NoError(t, db3.Open(t.Context(), path, WithRecovery(false), WithWrite(false), WithDeploymentHistory(false), "")) + _, recorded := db3.Data.Features[featureRecordDeploymentHistory] + assert.False(t, recorded, "state should not have recording marker yet") + mustFinalize(t, &db3) + + // The full test of the fix (opening with recording enabled) requires a workspace + // client context, which is tested in acceptance tests. This unit test verifies the + // precondition: a persisted state with resources but no recording marker. +}