infra udpates

Author: jeffreyaven

.github/workflows/databricks-dab.yml

@@ -31,9 +31,6 @@ env:
   DATABRICKS_CLIENT_ID: ${{ secrets.DATABRICKS_CLIENT_ID }}
   DATABRICKS_CLIENT_SECRET: ${{ secrets.DATABRICKS_CLIENT_SECRET }}
 
-  # # Databricks Workspace Credentials (set after infrastructure provisioning)
-  # DATABRICKS_HOST: ${{ secrets.DATABRICKS_HOST }}
-  # DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN }}
 
 jobs:
 

infrastructure/resources/aws/s3/s3_bucket.iql

@@ -36,9 +36,14 @@ WHERE
 region = '{{ region }}' 
 AND data__Identifier = '{{ bucket_name }}'
 
-/*+ statecheck, retries=3, retry_delay=5 */
-SELECT COUNT(*) as count FROM (
+/*+ exports, retries=3, retry_delay=5 */
+SELECT 
+arn,
+bucket_name
+FROM (
     SELECT 
+        arn,
+        bucket_name,
         JSON_EQUAL(ownership_controls, '{{ ownership_controls }}') as test_ownership_controls,
         JSON_EQUAL(bucket_encryption, '{{ bucket_encryption }}') as test_encryption,
         JSON_EQUAL(public_access_block_configuration, '{{ public_access_block_configuration }}') as test_public_access_block_configuration,
@@ -51,11 +56,3 @@ WHERE test_ownership_controls = 1
 AND test_encryption = 1
 AND test_public_access_block_configuration = 1
 AND test_versioning_configuration = 1
-
-/*+ exports, retries=3, retry_delay=5 */
-SELECT 
-arn,
-bucket_name
-FROM aws.s3.buckets
-WHERE region = '{{ region }}'
-AND data__Identifier = '{{ bucket_name }}'

infrastructure/resources/databricks_account/credentials.iql

@@ -15,25 +15,15 @@ SELECT
 '{{ credentials_name }}',
 '{{ aws_credentials }}'
 
-/*+ statecheck, retries=3, retry_delay=5 */
-SELECT COUNT(*) as count FROM
-(
-SELECT 
-credentials_id
-FROM databricks_account.provisioning.credentials
-WHERE account_id = '{{ databricks_account_id }}'
-AND credentials_name = '{{ credentials_name }}'
-AND JSON_EXTRACT(aws_credentials, '$.sts_role.role_arn') = '{{ aws_iam_cross_account_role_arn }}'
-) t
-
-/*+ exports */
+/*+ exports, retries=3, retry_delay=5 */
 SELECT 
 '{{ credentials_name }}' as databricks_credentials_name,
 credentials_id as databricks_credentials_id,
 JSON_EXTRACT(aws_credentials, '$.sts_role.external_id') as databricks_role_external_id
 FROM databricks_account.provisioning.credentials
 WHERE account_id = '{{ databricks_account_id }}'
 AND credentials_name = '{{ credentials_name }}'
+AND JSON_EXTRACT(aws_credentials, '$.sts_role.role_arn') = '{{ aws_iam_cross_account_role_arn }}'
 
 /*+ delete */
 DELETE FROM databricks_account.provisioning.credentials

infrastructure/resources/databricks_account/network.iql

@@ -19,26 +19,21 @@ SELECT
 '{{ subnet_ids }}',
 '{{ security_group_ids }}'
 
-/*+ statecheck, retries=3, retry_delay=5 */
-SELECT COUNT(*) as count FROM
-(
+/*+ exports, retries=3, retry_delay=5 */
+SELECT
+network_id as databricks_network_id
+FROM (
 SELECT 
+network_id,
 JSON_EQUAL(subnet_ids, '{{ subnet_ids }}') as subnet_test,
 JSON_EQUAL(security_group_ids, '{{ security_group_ids }}') as sg_test
 FROM databricks_account.provisioning.networks
 WHERE account_id = '{{ databricks_account_id }}'
 AND network_name = '{{ databricks_network_name }}'
 AND vpc_id = '{{ vpc_id }}'
-AND subnet_test = 1
-AND sg_test = 1
 )t
-
-/*+ exports */
-SELECT
-network_id as databricks_network_id
-FROM databricks_account.provisioning.networks
-WHERE account_id = '{{ databricks_account_id }}' AND
-network_name = '{{ databricks_network_name }}'
+WHERE subnet_test = 1
+AND sg_test = 1
 
 /*+ delete */
 DELETE FROM databricks_account.provisioning.networks

infrastructure/resources/databricks_account/storage_configuration.iql

@@ -15,19 +15,13 @@ SELECT
 '{{ storage_configuration_name }}',
 '{{ root_bucket_info }}'
 
-/*+ statecheck, retries=3, retry_delay=5 */
-SELECT COUNT(*) as count 
-FROM databricks_account.provisioning.storage
-WHERE account_id = '{{ databricks_account_id }}'
-AND storage_configuration_name = '{{ storage_configuration_name }}'
-AND JSON_EXTRACT(root_bucket_info, '$.bucket_name') = '{{ aws_s3_workspace_bucket_name }}'
-
-/*+ exports */
+/*+ exports, retries=3, retry_delay=5 */
 SELECT
 storage_configuration_id as databricks_storage_configuration_id
 FROM databricks_account.provisioning.storage
 WHERE account_id = '{{ databricks_account_id }}'
 AND storage_configuration_name = '{{ storage_configuration_name }}'
+AND JSON_EXTRACT(root_bucket_info, '$.bucket_name') = '{{ aws_s3_workspace_bucket_name }}'
 
 /*+ delete */
 DELETE FROM databricks_account.provisioning.storage

infrastructure/resources/databricks_account/workspace.iql

@@ -21,24 +21,18 @@ SELECT
 '{{ storage_configuration_id }}',
 '{{ pricing_tier }}'
 
-/*+ statecheck, retries=3, retry_delay=5 */
-SELECT COUNT(*) as count 
-FROM databricks_account.provisioning.workspaces
-WHERE account_id = '{{ databricks_account_id }}'
-AND workspace_name = '{{ workspace_name }}'
-AND aws_region = '{{ aws_region }}'
-AND credentials_id = '{{ credentials_id }}'
-AND storage_configuration_id = '{{ storage_configuration_id }}'
-AND pricing_tier = '{{ pricing_tier }}'
-
-/*+ exports */
+/*+ exports, retries=3, retry_delay=5 */
 SELECT 
 '{{ workspace_name }}' AS databricks_workspace_name,
 workspace_id AS databricks_workspace_id,
 deployment_name AS databricks_deployment_name
 FROM databricks_account.provisioning.workspaces
 WHERE account_id = '{{ databricks_account_id }}'
 AND workspace_name = '{{ workspace_name }}'
+AND aws_region = '{{ aws_region }}'
+AND credentials_id = '{{ credentials_id }}'
+AND storage_configuration_id = '{{ storage_configuration_id }}'
+AND pricing_tier = '{{ pricing_tier }}'
 
 /*+ delete */
 DELETE FROM databricks_account.provisioning.workspaces

infrastructure/resources/databricks_account/workspace_group.iql

@@ -13,13 +13,7 @@ SELECT
 '{{ databricks_account_id }}',
 '{{ display_name }}'
 
-/*+ statecheck, retries=3, retry_delay=5 */
-SELECT COUNT(*) as count
-FROM databricks_account.iam.groups
-WHERE account_id = '{{ databricks_account_id }}'
-AND displayName = '{{ display_name }}'
-
-/*+ exports */
+/*+ exports, retries=3, retry_delay=5 */
 SELECT id AS databricks_group_id,
 displayName AS databricks_group_name
 FROM databricks_account.iam.groups

infrastructure/resources/databricks_workspace/external_location.iql

@@ -24,21 +24,15 @@ SELECT
 {{ skip_validation }}
 ;
 
-/*+ statecheck, retries=3, retry_delay=5 */
-SELECT COUNT(*) as count
+/*+ exports, retries=3, retry_delay=5 */
+SELECT name as external_location_name
 FROM databricks_workspace.unitycatalog.external_locations
 WHERE name = '{{ name | replace('-', '_') }}' AND
 deployment_name = '{{ databricks_deployment_name }}'
 AND url = '{{ url }}' AND
 credential_name = '{{ credential_name | replace('-', '_') }}' AND
 read_only = {{ read_only }} AND
 comment = '{{ comment }}';
- 
-/*+ exports */
-SELECT name as external_location_name
-FROM databricks_workspace.unitycatalog.external_locations
-WHERE name = '{{ name | replace('-', '_') }}' AND
-deployment_name = '{{ databricks_deployment_name }}'
 
 /*+ delete */
 DELETE FROM databricks_workspace.unitycatalog.external_locations

infrastructure/resources/databricks_workspace/storage_credential.iql

@@ -22,20 +22,14 @@ SELECT
 '{{ skip_validation }}'
 ;
 
-/*+ statecheck, retries=3, retry_delay=5 */
-SELECT COUNT(*) as count
-FROM databricks_workspace.unitycatalog.storage_credentials
-WHERE name = '{{ name | replace('-', '_') | upper }}' AND
-deployment_name = '{{ databricks_deployment_name }}' AND
-JSON_EXTRACT(aws_iam_role, '$.role_arn') = '{{ metastore_access_role_arn }}'; 
-
-/*+ exports */
+/*+ exports, retries=3, retry_delay=5 */
 SELECT 
 name as storage_credential_name,
 JSON_EXTRACT(aws_iam_role, '$.external_id') as storage_credential_external_id
 FROM databricks_workspace.unitycatalog.storage_credentials
 WHERE name = '{{ name | replace('-', '_') | upper }}' AND
-deployment_name = '{{ databricks_deployment_name }}';
+deployment_name = '{{ databricks_deployment_name }}' AND
+JSON_EXTRACT(aws_iam_role, '$.role_arn') = '{{ metastore_access_role_arn }}';
 
 /*+ delete */
 DELETE FROM databricks_workspace.unitycatalog.storage_credentials

infrastructure/stackql_manifest.yml

@@ -188,7 +188,6 @@ resources:
       - bucket_name: aws_s3_workspace_bucket_name
 
   - name: aws/s3/workspace_bucket_policy
-    if: "'env' == '{{ stack_env }}'"
     file: aws/s3/s3_bucket_policy.iql
     props:
       - name: policy_document
@@ -225,7 +224,6 @@ resources:
 # ====================================================================================
 
   - name: aws/s3/metastore_bucket
-    if: "'env' == '{{ stack_env }}'"
     file: aws/s3/s3_bucket.iql
     props:
       - name: bucket_name
@@ -254,7 +252,6 @@ resources:
       - bucket_name: aws_s3_metastore_bucket_name
 
   - name: aws/iam/metastore_access_role
-    if: "'env' == '{{ stack_env }}'"
     file: aws/iam/iam_role.iql
     props:
       - name: role_name
@@ -387,7 +384,6 @@ resources:
       - databricks_deployment_name        
 
   - name: databricks_account/workspace_group
-    if: "'env' == '{{ stack_env }}'"
     props:
     - name: display_name
       value: "{{ stack_name }}-{{ stack_env }}-workspace-admins"
@@ -396,7 +392,6 @@ resources:
       - databricks_group_name
 
   - name: databricks_account/get_users
-    if: "'env' == '{{ stack_env }}'"
     type: query
     props:
     - name: users
@@ -407,16 +402,13 @@ resources:
       - databricks_workspace_group_members  
 
   - name: databricks_account/update_group_membership
-    if: "'env' == '{{ stack_env }}'"
     type: command
     props: []
 
   - name: databricks_account/workspace_permission_assignments
-    if: "'env' == '{{ stack_env }}'"
     props: []
 
   - name: databricks_workspace/storage_credential
-    if: "'env' == '{{ stack_env }}'"
     props:
     - name: name
       value: "{{ stack_name }}_{{ stack_env }}_storage_credential"
@@ -434,7 +426,6 @@ resources:
     - storage_credential_external_id
 
   - name: aws/iam/update_metastore_access_role
-    if: "'env' == '{{ stack_env }}'"
     type: command
     props:
       - name: role_name
@@ -454,7 +445,6 @@ resources:
                   sts:ExternalId: "{{ storage_credential_external_id }}"
 
   - name: databricks_workspace/unitycatalog/credential_grants
-    if: "'env' == '{{ stack_env }}'"
     type: command
     props: 
       - name: privileges
@@ -469,7 +459,6 @@ resources:
       deployment_name = '{{ databricks_deployment_name }}';
 
   - name: databricks_workspace/external_location
-    if: "'env' == '{{ stack_env }}'"
     props:
     - name: name
       value: "{{ stack_name }}_{{ stack_env }}_external_location"
@@ -487,7 +476,6 @@ resources:
     - external_location_name
 
   - name: databricks_workspace/unitycatalog/location_grants
-    if: "'env' == '{{ stack_env }}'"
     type: command
     props: 
       - name: privileges