diff --git a/NEXT_CHANGELOG.md b/NEXT_CHANGELOG.md index 1428900620..be6f926056 100644 --- a/NEXT_CHANGELOG.md +++ b/NEXT_CHANGELOG.md @@ -8,6 +8,7 @@ ### Bundles * `bundle deployment migrate` will now run a plan before migration to check if deployment was done ([#4088](https://github.com/databricks/cli/pull/4088)) +* engine/direct: Increase "serial" in state file after every deployment ([#4115](https://github.com/databricks/cli/pull/4115)) ### Dependency updates diff --git a/acceptance/bundle/migrate/basic/out.new_state.json b/acceptance/bundle/migrate/basic/out.new_state.json index b6652f618b..a93d0cab70 100644 --- a/acceptance/bundle/migrate/basic/out.new_state.json +++ b/acceptance/bundle/migrate/basic/out.new_state.json @@ -1,6 +1,6 @@ { "lineage": "[UUID]", - "serial": 5, + "serial": 6, "state": { "resources.jobs.test_job": { "__id__": "[NUMID]", diff --git a/acceptance/bundle/migrate/basic/output.txt b/acceptance/bundle/migrate/basic/output.txt index fe9dd8036e..3104156fbd 100644 --- a/acceptance/bundle/migrate/basic/output.txt +++ b/acceptance/bundle/migrate/basic/output.txt @@ -32,7 +32,7 @@ Error: Required engine "terraform" does not match present state files. Clear "DA Available state files: - terraform.tfstate: remote terraform state serial=4 lineage="[UUID]" -- [TEST_TMP_DIR]/.databricks/bundle/dev/resources.json: local direct state serial=5 lineage="[UUID]" +- [TEST_TMP_DIR]/.databricks/bundle/dev/resources.json: local direct state serial=6 lineage="[UUID]" >>> DATABRICKS_BUNDLE_ENGINE=terraform musterr [CLI] bundle deploy @@ -40,13 +40,13 @@ Error: Required engine "terraform" does not match present state files. Clear "DA Available state files: - terraform.tfstate: remote terraform state serial=4 lineage="[UUID]" -- [TEST_TMP_DIR]/.databricks/bundle/dev/resources.json: local direct state serial=5 lineage="[UUID]" +- [TEST_TMP_DIR]/.databricks/bundle/dev/resources.json: local direct state serial=6 lineage="[UUID]" === Should show that it's already migrated >>> musterr [CLI] bundle deployment migrate Error: already using direct engine -Details: [TEST_TMP_DIR]/.databricks/bundle/dev/resources.json: local direct state serial=5 lineage="[UUID]" +Details: [TEST_TMP_DIR]/.databricks/bundle/dev/resources.json: local direct state serial=6 lineage="[UUID]" >>> DATABRICKS_BUNDLE_ENGINE=direct [CLI] bundle plan update pipelines.test_pipeline @@ -97,14 +97,14 @@ Deployment complete! === Should show that it's already migrated >>> musterr [CLI] bundle deployment migrate Error: already using direct engine -Details: [TEST_TMP_DIR]/.databricks/bundle/dev/resources.json: local direct state serial=5 lineage="[UUID]" +Details: [TEST_TMP_DIR]/.databricks/bundle/dev/resources.json: local direct state serial=7 lineage="[UUID]" >>> DATABRICKS_BUNDLE_ENGINE= [CLI] bundle debug states -[TEST_TMP_DIR]/.databricks/bundle/dev/resources.json: local direct state serial=5 lineage="[UUID]" +[TEST_TMP_DIR]/.databricks/bundle/dev/resources.json: local direct state serial=7 lineage="[UUID]" >>> DATABRICKS_BUNDLE_ENGINE= [CLI] bundle debug states --force-pull -resources.json: remote direct state serial=5 lineage="[UUID]" -[TEST_TMP_DIR]/.databricks/bundle/dev/resources.json: local direct state serial=5 lineage="[UUID]" +resources.json: remote direct state serial=7 lineage="[UUID]" +[TEST_TMP_DIR]/.databricks/bundle/dev/resources.json: local direct state serial=7 lineage="[UUID]" === Extra plan: should have no drift >>> DATABRICKS_BUNDLE_ENGINE= [CLI] bundle plan diff --git a/acceptance/bundle/migrate/dashboards/out.new_state.json b/acceptance/bundle/migrate/dashboards/out.new_state.json index bdfb78f848..3fca7b12f6 100644 --- a/acceptance/bundle/migrate/dashboards/out.new_state.json +++ b/acceptance/bundle/migrate/dashboards/out.new_state.json @@ -1,6 +1,6 @@ { "lineage": "[UUID]", - "serial": 2, + "serial": 3, "state": { "resources.dashboards.dashboard1": { "__id__": "[DASHBOARD_ID]", diff --git a/acceptance/bundle/migrate/dashboards/output.txt b/acceptance/bundle/migrate/dashboards/output.txt index d4d898ce82..7ce0bced47 100644 --- a/acceptance/bundle/migrate/dashboards/output.txt +++ b/acceptance/bundle/migrate/dashboards/output.txt @@ -22,7 +22,7 @@ Error: Required engine "terraform" does not match present state files. Clear "DA Available state files: - terraform.tfstate: remote terraform state serial=1 lineage="[UUID]" -- [TEST_TMP_DIR]/.databricks/bundle/default/resources.json: local direct state serial=2 lineage="[UUID]" +- [TEST_TMP_DIR]/.databricks/bundle/default/resources.json: local direct state serial=3 lineage="[UUID]" >>> DATABRICKS_BUNDLE_ENGINE=terraform musterr [CLI] bundle deploy @@ -30,7 +30,7 @@ Error: Required engine "terraform" does not match present state files. Clear "DA Available state files: - terraform.tfstate: remote terraform state serial=1 lineage="[UUID]" -- [TEST_TMP_DIR]/.databricks/bundle/default/resources.json: local direct state serial=2 lineage="[UUID]" +- [TEST_TMP_DIR]/.databricks/bundle/default/resources.json: local direct state serial=3 lineage="[UUID]" >>> DATABRICKS_BUNDLE_ENGINE=direct [CLI] bundle plan @@ -54,11 +54,11 @@ Deployment complete! === Should show that it's already migrated >>> musterr [CLI] bundle deployment migrate Error: already using direct engine -Details: [TEST_TMP_DIR]/.databricks/bundle/default/resources.json: local direct state serial=2 lineage="[UUID]" +Details: [TEST_TMP_DIR]/.databricks/bundle/default/resources.json: local direct state serial=4 lineage="[UUID]" >>> DATABRICKS_BUNDLE_ENGINE= [CLI] bundle debug states -[TEST_TMP_DIR]/.databricks/bundle/default/resources.json: local direct state serial=2 lineage="[UUID]" +[TEST_TMP_DIR]/.databricks/bundle/default/resources.json: local direct state serial=4 lineage="[UUID]" >>> DATABRICKS_BUNDLE_ENGINE= [CLI] bundle debug states --force-pull -resources.json: remote direct state serial=2 lineage="[UUID]" -[TEST_TMP_DIR]/.databricks/bundle/default/resources.json: local direct state serial=2 lineage="[UUID]" +resources.json: remote direct state serial=4 lineage="[UUID]" +[TEST_TMP_DIR]/.databricks/bundle/default/resources.json: local direct state serial=4 lineage="[UUID]" diff --git a/acceptance/bundle/migrate/default-python/out.state_after_migration.json b/acceptance/bundle/migrate/default-python/out.state_after_migration.json index 5132e7ac58..fc12770f74 100644 --- a/acceptance/bundle/migrate/default-python/out.state_after_migration.json +++ b/acceptance/bundle/migrate/default-python/out.state_after_migration.json @@ -1,6 +1,6 @@ { "lineage": "[UUID]", - "serial": 4, + "serial": 5, "state": { "resources.jobs.sample_job": { "__id__": "[NUMID]", diff --git a/acceptance/bundle/migrate/default-python/output.txt b/acceptance/bundle/migrate/default-python/output.txt index 3c59e049ae..c9b95cdd0c 100644 --- a/acceptance/bundle/migrate/default-python/output.txt +++ b/acceptance/bundle/migrate/default-python/output.txt @@ -135,6 +135,15 @@ Deployment complete! >>> print_state.py >>> diff.py ../out.state_after_migration.json ../out.state_after_deploy.json +--- ../out.state_after_migration.json ++++ ../out.state_after_deploy.json +@@ -1,5 +1,5 @@ + { + "lineage": "[UUID]", +- "serial": 5, ++ "serial": 6, + "state": { + "resources.jobs.sample_job": { === Extra plan: should have no drift >>> DATABRICKS_BUNDLE_ENGINE= [CLI] bundle plan diff --git a/acceptance/bundle/migrate/grants/out.new_state.json b/acceptance/bundle/migrate/grants/out.new_state.json index 8d9153d034..cf2dc8bc60 100644 --- a/acceptance/bundle/migrate/grants/out.new_state.json +++ b/acceptance/bundle/migrate/grants/out.new_state.json @@ -1,6 +1,6 @@ { "lineage": "[UUID]", - "serial": 8, + "serial": 9, "state": { "resources.registered_models.my_registered_model": { "__id__": "main.schema_grants.mymodel", diff --git a/acceptance/bundle/migrate/grants/output.txt b/acceptance/bundle/migrate/grants/output.txt index 8c9b0cc194..5370e54ede 100644 --- a/acceptance/bundle/migrate/grants/output.txt +++ b/acceptance/bundle/migrate/grants/output.txt @@ -22,7 +22,7 @@ Error: Required engine "terraform" does not match present state files. Clear "DA Available state files: - terraform.tfstate: remote terraform state serial=7 lineage="[UUID]" -- [TEST_TMP_DIR]/.databricks/bundle/default/resources.json: local direct state serial=8 lineage="[UUID]" +- [TEST_TMP_DIR]/.databricks/bundle/default/resources.json: local direct state serial=9 lineage="[UUID]" >>> DATABRICKS_BUNDLE_ENGINE=terraform musterr [CLI] bundle deploy @@ -30,7 +30,7 @@ Error: Required engine "terraform" does not match present state files. Clear "DA Available state files: - terraform.tfstate: remote terraform state serial=7 lineage="[UUID]" -- [TEST_TMP_DIR]/.databricks/bundle/default/resources.json: local direct state serial=8 lineage="[UUID]" +- [TEST_TMP_DIR]/.databricks/bundle/default/resources.json: local direct state serial=9 lineage="[UUID]" >>> DATABRICKS_BUNDLE_ENGINE=direct [CLI] bundle plan @@ -52,11 +52,11 @@ Deployment complete! === Should show that it's already migrated >>> musterr [CLI] bundle deployment migrate Error: already using direct engine -Details: [TEST_TMP_DIR]/.databricks/bundle/default/resources.json: local direct state serial=8 lineage="[UUID]" +Details: [TEST_TMP_DIR]/.databricks/bundle/default/resources.json: local direct state serial=10 lineage="[UUID]" >>> DATABRICKS_BUNDLE_ENGINE= [CLI] bundle debug states -[TEST_TMP_DIR]/.databricks/bundle/default/resources.json: local direct state serial=8 lineage="[UUID]" +[TEST_TMP_DIR]/.databricks/bundle/default/resources.json: local direct state serial=10 lineage="[UUID]" >>> DATABRICKS_BUNDLE_ENGINE= [CLI] bundle debug states --force-pull -resources.json: remote direct state serial=8 lineage="[UUID]" -[TEST_TMP_DIR]/.databricks/bundle/default/resources.json: local direct state serial=8 lineage="[UUID]" +resources.json: remote direct state serial=10 lineage="[UUID]" +[TEST_TMP_DIR]/.databricks/bundle/default/resources.json: local direct state serial=10 lineage="[UUID]" diff --git a/acceptance/bundle/migrate/permissions/out.new_state.json b/acceptance/bundle/migrate/permissions/out.new_state.json index 920cfeba1a..dca45c1ac0 100644 --- a/acceptance/bundle/migrate/permissions/out.new_state.json +++ b/acceptance/bundle/migrate/permissions/out.new_state.json @@ -1,6 +1,6 @@ { "lineage": "[UUID]", - "serial": 6, + "serial": 7, "state": { "resources.jobs.test_job": { "__id__": "[NUMID]", diff --git a/acceptance/bundle/migrate/permissions/output.txt b/acceptance/bundle/migrate/permissions/output.txt index 896c34050a..92f6358b7e 100644 --- a/acceptance/bundle/migrate/permissions/output.txt +++ b/acceptance/bundle/migrate/permissions/output.txt @@ -22,7 +22,7 @@ Error: Required engine "terraform" does not match present state files. Clear "DA Available state files: - terraform.tfstate: remote terraform state serial=5 lineage="[UUID]" -- [TEST_TMP_DIR]/.databricks/bundle/default/resources.json: local direct state serial=6 lineage="[UUID]" +- [TEST_TMP_DIR]/.databricks/bundle/default/resources.json: local direct state serial=7 lineage="[UUID]" >>> DATABRICKS_BUNDLE_ENGINE=terraform musterr [CLI] bundle deploy @@ -30,7 +30,7 @@ Error: Required engine "terraform" does not match present state files. Clear "DA Available state files: - terraform.tfstate: remote terraform state serial=5 lineage="[UUID]" -- [TEST_TMP_DIR]/.databricks/bundle/default/resources.json: local direct state serial=6 lineage="[UUID]" +- [TEST_TMP_DIR]/.databricks/bundle/default/resources.json: local direct state serial=7 lineage="[UUID]" >>> DATABRICKS_BUNDLE_ENGINE=direct [CLI] bundle plan @@ -78,11 +78,11 @@ Deployment complete! === Should show that it's already migrated >>> musterr [CLI] bundle deployment migrate Error: already using direct engine -Details: [TEST_TMP_DIR]/.databricks/bundle/default/resources.json: local direct state serial=6 lineage="[UUID]" +Details: [TEST_TMP_DIR]/.databricks/bundle/default/resources.json: local direct state serial=8 lineage="[UUID]" >>> DATABRICKS_BUNDLE_ENGINE= [CLI] bundle debug states -[TEST_TMP_DIR]/.databricks/bundle/default/resources.json: local direct state serial=6 lineage="[UUID]" +[TEST_TMP_DIR]/.databricks/bundle/default/resources.json: local direct state serial=8 lineage="[UUID]" >>> DATABRICKS_BUNDLE_ENGINE= [CLI] bundle debug states --force-pull -resources.json: remote direct state serial=6 lineage="[UUID]" -[TEST_TMP_DIR]/.databricks/bundle/default/resources.json: local direct state serial=6 lineage="[UUID]" +resources.json: remote direct state serial=8 lineage="[UUID]" +[TEST_TMP_DIR]/.databricks/bundle/default/resources.json: local direct state serial=8 lineage="[UUID]" diff --git a/acceptance/bundle/migrate/runas/out.new_state.json b/acceptance/bundle/migrate/runas/out.new_state.json index df7d83ab83..76bfaf8899 100644 --- a/acceptance/bundle/migrate/runas/out.new_state.json +++ b/acceptance/bundle/migrate/runas/out.new_state.json @@ -1,6 +1,6 @@ { "lineage": "[UUID]", - "serial": 4, + "serial": 5, "state": { "resources.pipelines.foo": { "__id__": "[UUID]", diff --git a/acceptance/bundle/resources/jobs/update/out.state.direct.json b/acceptance/bundle/resources/jobs/update/out.state.direct.json index 85e0853012..a962bbd82e 100644 --- a/acceptance/bundle/resources/jobs/update/out.state.direct.json +++ b/acceptance/bundle/resources/jobs/update/out.state.direct.json @@ -1,6 +1,6 @@ { "lineage": "[UUID]", - "serial": 1, + "serial": 2, "state": { "resources.jobs.foo": { "__id__": "[JOB_ID]", diff --git a/acceptance/bundle/state/state_present/output.txt b/acceptance/bundle/state/state_present/output.txt index ab36cef52c..d66bac45fa 100644 --- a/acceptance/bundle/state/state_present/output.txt +++ b/acceptance/bundle/state/state_present/output.txt @@ -84,13 +84,17 @@ Deployment complete! >>> print_requests.py //api/2.1/unity-catalog/schemas "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/direct auth/pat" +>>> print_state.py +3 +11 + >>> DATABRICKS_BUNDLE_ENGINE=terraform musterr [CLI] bundle deploy Error: Required engine "terraform" does not match present state files. Clear "DATABRICKS_BUNDLE_ENGINE" env var to use engine appropriate for the state. Available state files: - [TEST_TMP_DIR]/.databricks/bundle/default/terraform/terraform.tfstate: local terraform state serial=3 lineage="" -- resources.json: remote direct state serial=10 lineage="" -- [TEST_TMP_DIR]/.databricks/bundle/default/resources.json: local direct state serial=10 lineage="" +- resources.json: remote direct state serial=11 lineage="" +- [TEST_TMP_DIR]/.databricks/bundle/default/resources.json: local direct state serial=11 lineage="" >>> DATABRICKS_BUNDLE_ENGINE= [CLI] bundle deploy @@ -102,11 +106,15 @@ Deployment complete! >>> print_requests.py --get //api/2.1/unity-catalog/schemas "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/direct auth/pat" +>>> print_state.py +3 +12 + >>> DATABRICKS_BUNDLE_ENGINE= [CLI] bundle debug states [TEST_TMP_DIR]/.databricks/bundle/default/terraform/terraform.tfstate: local terraform state serial=3 lineage="" -[TEST_TMP_DIR]/.databricks/bundle/default/resources.json: local direct state serial=10 lineage="" +[TEST_TMP_DIR]/.databricks/bundle/default/resources.json: local direct state serial=12 lineage="" >>> DATABRICKS_BUNDLE_ENGINE= [CLI] bundle debug states --force-pull [TEST_TMP_DIR]/.databricks/bundle/default/terraform/terraform.tfstate: local terraform state serial=3 lineage="" -resources.json: remote direct state serial=10 lineage="" -[TEST_TMP_DIR]/.databricks/bundle/default/resources.json: local direct state serial=10 lineage="" +resources.json: remote direct state serial=12 lineage="" +[TEST_TMP_DIR]/.databricks/bundle/default/resources.json: local direct state serial=12 lineage="" diff --git a/acceptance/bundle/state/state_present/script b/acceptance/bundle/state/state_present/script index 58da9b2265..79c0cc1071 100644 --- a/acceptance/bundle/state/state_present/script +++ b/acceptance/bundle/state/state_present/script @@ -30,12 +30,14 @@ trace DATABRICKS_BUNDLE_ENGINE=terraform musterr $CLI bundle destroy --auto-appr rm out.requests.txt trace DATABRICKS_BUNDLE_ENGINE=direct $CLI bundle deploy trace print_requests.py //api/2.1/unity-catalog/schemas | jq '.headers["User-Agent"][0]' | contains.py 'engine/direct' '!terraform' '!tf-provider' +trace print_state.py | jq .serial | contains.py "11" trace DATABRICKS_BUNDLE_ENGINE=terraform musterr $CLI bundle deploy #trace print_requests.py --get //api/2.1/unity-catalog/schemas | jq '.headers["User-Agent"][0]' | contains.py 'engine/direct' '!terraform' '!tf-provider' trace DATABRICKS_BUNDLE_ENGINE= $CLI bundle deploy trace print_requests.py --get //api/2.1/unity-catalog/schemas | jq '.headers["User-Agent"][0]' | contains.py 'engine/direct' '!terraform' '!tf-provider' +trace print_state.py | jq .serial | contains.py "12" trace DATABRICKS_BUNDLE_ENGINE= $CLI bundle debug states trace DATABRICKS_BUNDLE_ENGINE= $CLI bundle debug states --force-pull diff --git a/acceptance/bundle/user_agent/simple/out.requests.deploy.direct.json b/acceptance/bundle/user_agent/simple/out.requests.deploy.direct.json index 10fd9c7c21..173526871c 100644 --- a/acceptance/bundle/user_agent/simple/out.requests.deploy.direct.json +++ b/acceptance/bundle/user_agent/simple/out.requests.deploy.direct.json @@ -221,7 +221,7 @@ }, "body": { "lineage": "[UUID]", - "serial": 1, + "serial": 2, "state": { "resources.schemas.foo": { "__id__": "mycatalog.myschema", diff --git a/bundle/direct/dstate/state.go b/bundle/direct/dstate/state.go index ab9f07f1a9..e2fda94001 100644 --- a/bundle/direct/dstate/state.go +++ b/bundle/direct/dstate/state.go @@ -111,6 +111,8 @@ func (db *DeploymentState) Finalize() error { db.mu.Lock() defer db.mu.Unlock() + db.Data.Serial += 1 + return db.unlockedSave() }