direct: increase "serial" field after every deployment

NEXT_CHANGELOG.md

@@ -8,6 +8,7 @@
 
 ### Bundles
 * `bundle deployment migrate` will now run a plan before migration to check if deployment was done ([#4088](https://github.com/databricks/cli/pull/4088))
+* engine/direct: Increase "serial" in state file after every deployment ([#4115](https://github.com/databricks/cli/pull/4115))
 
 ### Dependency updates
 

acceptance/bundle/migrate/basic/out.new_state.json

@@ -1,6 +1,6 @@
 {
  "lineage": "[UUID]",
- "serial": 5,
+ "serial": 6,
  "state": {
   "resources.jobs.test_job": {
    "__id__": "[NUMID]",

acceptance/bundle/migrate/basic/output.txt

@@ -32,21 +32,21 @@ Error: Required engine "terraform" does not match present state files. Clear "DA
 
 Available state files:
 - terraform.tfstate: remote terraform state serial=4 lineage="[UUID]"
-- [TEST_TMP_DIR]/.databricks/bundle/dev/resources.json: local direct state serial=5 lineage="[UUID]"
+- [TEST_TMP_DIR]/.databricks/bundle/dev/resources.json: local direct state serial=6 lineage="[UUID]"
 
 
 >>> DATABRICKS_BUNDLE_ENGINE=terraform musterr [CLI] bundle deploy
 Error: Required engine "terraform" does not match present state files. Clear "DATABRICKS_BUNDLE_ENGINE" env var to use engine appropriate for the state.
 
 Available state files:
 - terraform.tfstate: remote terraform state serial=4 lineage="[UUID]"
-- [TEST_TMP_DIR]/.databricks/bundle/dev/resources.json: local direct state serial=5 lineage="[UUID]"
+- [TEST_TMP_DIR]/.databricks/bundle/dev/resources.json: local direct state serial=6 lineage="[UUID]"
 
 
 === Should show that it's already migrated
 >>> musterr [CLI] bundle deployment migrate
 Error: already using direct engine
-Details: [TEST_TMP_DIR]/.databricks/bundle/dev/resources.json: local direct state serial=5 lineage="[UUID]"
+Details: [TEST_TMP_DIR]/.databricks/bundle/dev/resources.json: local direct state serial=6 lineage="[UUID]"
 
 >>> DATABRICKS_BUNDLE_ENGINE=direct [CLI] bundle plan
 update pipelines.test_pipeline
@@ -97,14 +97,14 @@ Deployment complete!
 === Should show that it's already migrated
 >>> musterr [CLI] bundle deployment migrate
 Error: already using direct engine
-Details: [TEST_TMP_DIR]/.databricks/bundle/dev/resources.json: local direct state serial=5 lineage="[UUID]"
+Details: [TEST_TMP_DIR]/.databricks/bundle/dev/resources.json: local direct state serial=7 lineage="[UUID]"
 
 >>> DATABRICKS_BUNDLE_ENGINE= [CLI] bundle debug states
-[TEST_TMP_DIR]/.databricks/bundle/dev/resources.json: local direct state serial=5 lineage="[UUID]"
+[TEST_TMP_DIR]/.databricks/bundle/dev/resources.json: local direct state serial=7 lineage="[UUID]"
 
 >>> DATABRICKS_BUNDLE_ENGINE= [CLI] bundle debug states --force-pull
-resources.json: remote direct state serial=5 lineage="[UUID]"
-[TEST_TMP_DIR]/.databricks/bundle/dev/resources.json: local direct state serial=5 lineage="[UUID]"
+resources.json: remote direct state serial=7 lineage="[UUID]"
+[TEST_TMP_DIR]/.databricks/bundle/dev/resources.json: local direct state serial=7 lineage="[UUID]"
 
 === Extra plan: should have no drift
 >>> DATABRICKS_BUNDLE_ENGINE= [CLI] bundle plan

acceptance/bundle/migrate/dashboards/out.new_state.json

@@ -1,6 +1,6 @@
 {
  "lineage": "[UUID]",
- "serial": 2,
+ "serial": 3,
  "state": {
   "resources.dashboards.dashboard1": {
    "__id__": "[DASHBOARD_ID]",

acceptance/bundle/migrate/dashboards/output.txt

@@ -22,15 +22,15 @@ Error: Required engine "terraform" does not match present state files. Clear "DA
 
 Available state files:
 - terraform.tfstate: remote terraform state serial=1 lineage="[UUID]"
-- [TEST_TMP_DIR]/.databricks/bundle/default/resources.json: local direct state serial=2 lineage="[UUID]"
+- [TEST_TMP_DIR]/.databricks/bundle/default/resources.json: local direct state serial=3 lineage="[UUID]"
 
 
 >>> DATABRICKS_BUNDLE_ENGINE=terraform musterr [CLI] bundle deploy
 Error: Required engine "terraform" does not match present state files. Clear "DATABRICKS_BUNDLE_ENGINE" env var to use engine appropriate for the state.
 
 Available state files:
 - terraform.tfstate: remote terraform state serial=1 lineage="[UUID]"
-- [TEST_TMP_DIR]/.databricks/bundle/default/resources.json: local direct state serial=2 lineage="[UUID]"
+- [TEST_TMP_DIR]/.databricks/bundle/default/resources.json: local direct state serial=3 lineage="[UUID]"
 
 
 >>> DATABRICKS_BUNDLE_ENGINE=direct [CLI] bundle plan
@@ -54,11 +54,11 @@ Deployment complete!
 === Should show that it's already migrated
 >>> musterr [CLI] bundle deployment migrate
 Error: already using direct engine
-Details: [TEST_TMP_DIR]/.databricks/bundle/default/resources.json: local direct state serial=2 lineage="[UUID]"
+Details: [TEST_TMP_DIR]/.databricks/bundle/default/resources.json: local direct state serial=4 lineage="[UUID]"
 
 >>> DATABRICKS_BUNDLE_ENGINE= [CLI] bundle debug states
-[TEST_TMP_DIR]/.databricks/bundle/default/resources.json: local direct state serial=2 lineage="[UUID]"
+[TEST_TMP_DIR]/.databricks/bundle/default/resources.json: local direct state serial=4 lineage="[UUID]"
 
 >>> DATABRICKS_BUNDLE_ENGINE= [CLI] bundle debug states --force-pull
-resources.json: remote direct state serial=2 lineage="[UUID]"
-[TEST_TMP_DIR]/.databricks/bundle/default/resources.json: local direct state serial=2 lineage="[UUID]"
+resources.json: remote direct state serial=4 lineage="[UUID]"
+[TEST_TMP_DIR]/.databricks/bundle/default/resources.json: local direct state serial=4 lineage="[UUID]"

acceptance/bundle/migrate/default-python/out.state_after_migration.json

@@ -1,6 +1,6 @@
 {
  "lineage": "[UUID]",
- "serial": 4,
+ "serial": 5,
  "state": {
   "resources.jobs.sample_job": {
    "__id__": "[NUMID]",

acceptance/bundle/migrate/default-python/output.txt

@@ -135,6 +135,15 @@ Deployment complete!
 >>> print_state.py
 
 >>> diff.py ../out.state_after_migration.json ../out.state_after_deploy.json
+--- ../out.state_after_migration.json
++++ ../out.state_after_deploy.json
+@@ -1,5 +1,5 @@
+ {
+  "lineage": "[UUID]",
+- "serial": 5,
++ "serial": 6,
+  "state": {
+   "resources.jobs.sample_job": {
 
 === Extra plan: should have no drift
 >>> DATABRICKS_BUNDLE_ENGINE= [CLI] bundle plan

acceptance/bundle/migrate/grants/out.new_state.json

@@ -1,6 +1,6 @@
 {
  "lineage": "[UUID]",
- "serial": 8,
+ "serial": 9,
  "state": {
   "resources.registered_models.my_registered_model": {
    "__id__": "main.schema_grants.mymodel",

acceptance/bundle/migrate/grants/output.txt

@@ -22,15 +22,15 @@ Error: Required engine "terraform" does not match present state files. Clear "DA
 
 Available state files:
 - terraform.tfstate: remote terraform state serial=7 lineage="[UUID]"
-- [TEST_TMP_DIR]/.databricks/bundle/default/resources.json: local direct state serial=8 lineage="[UUID]"
+- [TEST_TMP_DIR]/.databricks/bundle/default/resources.json: local direct state serial=9 lineage="[UUID]"
 
 
 >>> DATABRICKS_BUNDLE_ENGINE=terraform musterr [CLI] bundle deploy
 Error: Required engine "terraform" does not match present state files. Clear "DATABRICKS_BUNDLE_ENGINE" env var to use engine appropriate for the state.
 
 Available state files:
 - terraform.tfstate: remote terraform state serial=7 lineage="[UUID]"
-- [TEST_TMP_DIR]/.databricks/bundle/default/resources.json: local direct state serial=8 lineage="[UUID]"
+- [TEST_TMP_DIR]/.databricks/bundle/default/resources.json: local direct state serial=9 lineage="[UUID]"
 
 
 >>> DATABRICKS_BUNDLE_ENGINE=direct [CLI] bundle plan
@@ -52,11 +52,11 @@ Deployment complete!
 === Should show that it's already migrated
 >>> musterr [CLI] bundle deployment migrate
 Error: already using direct engine
-Details: [TEST_TMP_DIR]/.databricks/bundle/default/resources.json: local direct state serial=8 lineage="[UUID]"
+Details: [TEST_TMP_DIR]/.databricks/bundle/default/resources.json: local direct state serial=10 lineage="[UUID]"
 
 >>> DATABRICKS_BUNDLE_ENGINE= [CLI] bundle debug states
-[TEST_TMP_DIR]/.databricks/bundle/default/resources.json: local direct state serial=8 lineage="[UUID]"
+[TEST_TMP_DIR]/.databricks/bundle/default/resources.json: local direct state serial=10 lineage="[UUID]"
 
 >>> DATABRICKS_BUNDLE_ENGINE= [CLI] bundle debug states --force-pull
-resources.json: remote direct state serial=8 lineage="[UUID]"
-[TEST_TMP_DIR]/.databricks/bundle/default/resources.json: local direct state serial=8 lineage="[UUID]"
+resources.json: remote direct state serial=10 lineage="[UUID]"
+[TEST_TMP_DIR]/.databricks/bundle/default/resources.json: local direct state serial=10 lineage="[UUID]"

acceptance/bundle/migrate/permissions/out.new_state.json

@@ -1,6 +1,6 @@
 {
  "lineage": "[UUID]",
- "serial": 6,
+ "serial": 7,
  "state": {
   "resources.jobs.test_job": {
    "__id__": "[NUMID]",

acceptance/bundle/migrate/permissions/output.txt

@@ -22,15 +22,15 @@ Error: Required engine "terraform" does not match present state files. Clear "DA
 
 Available state files:
 - terraform.tfstate: remote terraform state serial=5 lineage="[UUID]"
-- [TEST_TMP_DIR]/.databricks/bundle/default/resources.json: local direct state serial=6 lineage="[UUID]"
+- [TEST_TMP_DIR]/.databricks/bundle/default/resources.json: local direct state serial=7 lineage="[UUID]"
 
 
 >>> DATABRICKS_BUNDLE_ENGINE=terraform musterr [CLI] bundle deploy
 Error: Required engine "terraform" does not match present state files. Clear "DATABRICKS_BUNDLE_ENGINE" env var to use engine appropriate for the state.
 
 Available state files:
 - terraform.tfstate: remote terraform state serial=5 lineage="[UUID]"
-- [TEST_TMP_DIR]/.databricks/bundle/default/resources.json: local direct state serial=6 lineage="[UUID]"
+- [TEST_TMP_DIR]/.databricks/bundle/default/resources.json: local direct state serial=7 lineage="[UUID]"
 
 
 >>> DATABRICKS_BUNDLE_ENGINE=direct [CLI] bundle plan
@@ -78,11 +78,11 @@ Deployment complete!
 === Should show that it's already migrated
 >>> musterr [CLI] bundle deployment migrate
 Error: already using direct engine
-Details: [TEST_TMP_DIR]/.databricks/bundle/default/resources.json: local direct state serial=6 lineage="[UUID]"
+Details: [TEST_TMP_DIR]/.databricks/bundle/default/resources.json: local direct state serial=8 lineage="[UUID]"
 
 >>> DATABRICKS_BUNDLE_ENGINE= [CLI] bundle debug states
-[TEST_TMP_DIR]/.databricks/bundle/default/resources.json: local direct state serial=6 lineage="[UUID]"
+[TEST_TMP_DIR]/.databricks/bundle/default/resources.json: local direct state serial=8 lineage="[UUID]"
 
 >>> DATABRICKS_BUNDLE_ENGINE= [CLI] bundle debug states --force-pull
-resources.json: remote direct state serial=6 lineage="[UUID]"
-[TEST_TMP_DIR]/.databricks/bundle/default/resources.json: local direct state serial=6 lineage="[UUID]"
+resources.json: remote direct state serial=8 lineage="[UUID]"
+[TEST_TMP_DIR]/.databricks/bundle/default/resources.json: local direct state serial=8 lineage="[UUID]"

acceptance/bundle/migrate/runas/out.new_state.json

@@ -1,6 +1,6 @@
 {
  "lineage": "[UUID]",
- "serial": 4,
+ "serial": 5,
  "state": {
   "resources.pipelines.foo": {
    "__id__": "[UUID]",

acceptance/bundle/resources/jobs/update/out.state.direct.json

@@ -1,6 +1,6 @@
 {
  "lineage": "[UUID]",
- "serial": 1,
+ "serial": 2,
  "state": {
   "resources.jobs.foo": {
    "__id__": "[JOB_ID]",

acceptance/bundle/state/state_present/output.txt

@@ -84,13 +84,17 @@ Deployment complete!
 >>> print_requests.py //api/2.1/unity-catalog/schemas
 "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/direct auth/pat"
 
+>>> print_state.py
+3
+11
+
 >>> DATABRICKS_BUNDLE_ENGINE=terraform musterr [CLI] bundle deploy
 Error: Required engine "terraform" does not match present state files. Clear "DATABRICKS_BUNDLE_ENGINE" env var to use engine appropriate for the state.
 
 Available state files:
 - [TEST_TMP_DIR]/.databricks/bundle/default/terraform/terraform.tfstate: local terraform state serial=3 lineage=""
-- resources.json: remote direct state serial=10 lineage=""
-- [TEST_TMP_DIR]/.databricks/bundle/default/resources.json: local direct state serial=10 lineage=""
+- resources.json: remote direct state serial=11 lineage=""
+- [TEST_TMP_DIR]/.databricks/bundle/default/resources.json: local direct state serial=11 lineage=""
 
 
 >>> DATABRICKS_BUNDLE_ENGINE= [CLI] bundle deploy
@@ -102,11 +106,15 @@ Deployment complete!
 >>> print_requests.py --get //api/2.1/unity-catalog/schemas
 "cli/[DEV_VERSION] databricks-sdk-go/[SDK_VERSION] go/[GO_VERSION] os/[OS] cmd/bundle_deploy cmd-exec-id/[UUID] engine/direct auth/pat"
 
+>>> print_state.py
+3
+12
+
 >>> DATABRICKS_BUNDLE_ENGINE= [CLI] bundle debug states
 [TEST_TMP_DIR]/.databricks/bundle/default/terraform/terraform.tfstate: local terraform state serial=3 lineage=""
-[TEST_TMP_DIR]/.databricks/bundle/default/resources.json: local direct state serial=10 lineage=""
+[TEST_TMP_DIR]/.databricks/bundle/default/resources.json: local direct state serial=12 lineage=""
 
 >>> DATABRICKS_BUNDLE_ENGINE= [CLI] bundle debug states --force-pull
 [TEST_TMP_DIR]/.databricks/bundle/default/terraform/terraform.tfstate: local terraform state serial=3 lineage=""
-resources.json: remote direct state serial=10 lineage=""
-[TEST_TMP_DIR]/.databricks/bundle/default/resources.json: local direct state serial=10 lineage=""
+resources.json: remote direct state serial=12 lineage=""
+[TEST_TMP_DIR]/.databricks/bundle/default/resources.json: local direct state serial=12 lineage=""

acceptance/bundle/state/state_present/script

@@ -30,12 +30,14 @@ trace DATABRICKS_BUNDLE_ENGINE=terraform musterr $CLI bundle destroy --auto-appr
 rm out.requests.txt
 trace DATABRICKS_BUNDLE_ENGINE=direct $CLI bundle deploy
 trace print_requests.py //api/2.1/unity-catalog/schemas | jq '.headers["User-Agent"][0]' | contains.py 'engine/direct' '!terraform' '!tf-provider'
+trace print_state.py | jq .serial | contains.py "11"
 
 trace DATABRICKS_BUNDLE_ENGINE=terraform musterr $CLI bundle deploy
 #trace print_requests.py --get //api/2.1/unity-catalog/schemas | jq '.headers["User-Agent"][0]' | contains.py 'engine/direct' '!terraform' '!tf-provider'
 
 trace DATABRICKS_BUNDLE_ENGINE= $CLI bundle deploy
 trace print_requests.py --get //api/2.1/unity-catalog/schemas | jq '.headers["User-Agent"][0]' | contains.py 'engine/direct' '!terraform' '!tf-provider'
+trace print_state.py | jq .serial | contains.py "12"
 
 trace DATABRICKS_BUNDLE_ENGINE= $CLI bundle debug states
 trace DATABRICKS_BUNDLE_ENGINE= $CLI bundle debug states --force-pull

acceptance/bundle/user_agent/simple/out.requests.deploy.direct.json

@@ -221,7 +221,7 @@
   },
   "body": {
     "lineage": "[UUID]",
-    "serial": 1,
+    "serial": 2,
     "state": {
       "resources.schemas.foo": {
         "__id__": "mycatalog.myschema",

bundle/direct/dstate/state.go

@@ -111,6 +111,8 @@ func (db *DeploymentState) Finalize() error {
 	db.mu.Lock()
 	defer db.mu.Unlock()
 
+	db.Data.Serial += 1
+
 	return db.unlockedSave()
 }