Merge pull request #262 from microsoft/v1.9.1

Addressing issues - #260, #258, #257, #256, #254, #250, #239 issues

Author: prdpsvs

dbt/adapters/fabric/__version__.py

@@ -1 +1 @@
-version = "1.9.0"
+version = "1.9.1"

dbt/adapters/fabric/relation_configs/policies.py

@@ -7,6 +7,7 @@
 class FabricRelationType(StrEnum):
     Table = "table"
     View = "view"
+    Ephemeral = "ephemeral"
     CTE = "cte"
 
 

dbt/include/fabric/macros/adapters/columns.sql

@@ -1,19 +1,17 @@
 {% macro fabric__get_empty_subquery_sql(select_sql, select_sql_header=none) %}
-    {% if select_sql.strip().lower().startswith('with') %}
-        {{ select_sql }}
-    {% else -%}
-        select * from (
+
+    with __dbt_sbq_tmp as (
         {{ select_sql }}
-    ) dbt_sbq_tmp
-    where 1 = 0
-    {%- endif -%}
+    )
+    select * from __dbt_sbq_tmp
+    where 0 = 1
 
 {% endmacro %}
 
 {% macro fabric__get_columns_in_relation(relation) -%}
     {% set query_label = apply_label() %}
     {% call statement('get_columns_in_relation', fetch_result=True) %}
-
+        {{ get_use_database_sql(relation.database) }}
         with mapping as (
             select
                 row_number() over (partition by object_name(c.object_id) order by c.column_id) as ordinal_position,
@@ -22,7 +20,7 @@
                 c.max_length as character_maximum_length,
                 c.precision as numeric_precision,
                 c.scale as numeric_scale
-            from [{{ 'tempdb' if '#' in relation.identifier else relation.database }}].sys.columns c {{ information_schema_hints() }}
+            from sys.columns c {{ information_schema_hints() }}
             inner join sys.types t {{ information_schema_hints() }}
             on c.user_type_id = t.user_type_id
             where c.object_id = object_id('{{ 'tempdb..' ~ relation.include(database=false, schema=false) if '#' in relation.identifier else relation }}')
@@ -46,11 +44,16 @@
 {% macro fabric__get_columns_in_query(select_sql) %}
     {% set query_label = apply_label() %}
     {% call statement('get_columns_in_query', fetch_result=True, auto_begin=False) -%}
-        select TOP 0 * from (
+
+        with __dbt_sbq as
+        (
             {{ select_sql }}
-        ) as __dbt_sbq
+        )
+        select top 0 *
+        from __dbt_sbq
         where 0 = 1
         {{ query_label }}
+
     {% endcall %}
 
     {{ return(load_result('get_columns_in_query').table.columns | map(attribute='name') | list) }}

dbt/include/fabric/macros/materializations/models/table/create_table_as.sql

@@ -1,26 +1,38 @@
 {% macro fabric__create_table_as(temporary, relation, sql) -%}
-    {% set query_label = apply_label() %}
-    {% set tmp_vw_relation = relation.incorporate(path={"identifier": relation.identifier ~ '__dbt_tmp_vw'}, type='view')-%}
-    {% do adapter.drop_relation(tmp_vw_relation) %}
-    {{ get_create_view_as_sql(tmp_vw_relation, sql) }}
 
+    {% set query_label = apply_label() %}
     {% set contract_config = config.get('contract') %}
-    {% if contract_config.enforced %}
+
+    {% if sql.strip().lower().startswith('with') and contract_config.enforced %}
+
+        {{ exceptions.raise_compiler_error(
+            "As contract is enforced and the model is using CTE, INSERT INTO is not supported with CTE. Either do not enforce contract or change the model"
+        ) }}
+
+    {%- elif not sql.strip().lower().startswith('with') and contract_config.enforced %}
 
         CREATE TABLE {{relation}}
         {{ build_columns_constraints(relation) }}
         {{ get_assert_columns_equivalent(sql)  }}
+
         {% set listColumns %}
             {% for column in model['columns'] %}
                 {{ "["~column~"]" }}{{ ", " if not loop.last }}
             {% endfor %}
         {%endset%}
 
+        {% set tmp_vw_relation = relation.incorporate(path={"identifier": relation.identifier ~ '__dbt_tmp_vw'}, type='view')-%}
+        {% do adapter.drop_relation(tmp_vw_relation) %}
+        {{ get_create_view_as_sql(tmp_vw_relation, sql) }}
+
         INSERT INTO {{relation}} ({{listColumns}})
         SELECT {{listColumns}} FROM {{tmp_vw_relation}} {{ query_label }}
 
     {%- else %}
+
         {%- set query_label_option = query_label.replace("'", "''") -%}
-        EXEC('CREATE TABLE {{relation}} AS SELECT * FROM {{tmp_vw_relation}} {{ query_label_option }}');
+        {%- set sql_with_quotes = sql.replace("'", "''") -%}
+        EXEC('CREATE TABLE {{relation}} AS {{sql_with_quotes}} {{ query_label_option }}');
+
     {% endif %}
 {% endmacro %}

dbt/include/fabric/macros/materializations/snapshots/snapshot.sql

@@ -30,7 +30,7 @@
           type='view') -%}
 
   -- Create a temporary view to manage if user SQl uses CTE
-  {% set temp_snapshot_relation_sql = model['compiled_code'].replace("'", "''") %}
+  {% set temp_snapshot_relation_sql = model['compiled_code'] %}
   {{ adapter.drop_relation(temp_snapshot_relation) }}
 
   {% call statement('create temp_snapshot_relation') -%}

dbt/include/fabric/macros/materializations/tests/helpers.sql

@@ -1,28 +1,59 @@
-{% macro get_fabric_test_sql(database, schema, main_sql, fail_calc, warn_if, error_if, limit) -%}
-  {{ log ("local_md5(model.name): "~ local_md5(model.name))}}
-  {{ log ("local_md5(invocation_id): "~ local_md5(invocation_id))}}
-  {% set testview %}
+{% macro fabric__get_test_sql(main_sql, fail_calc, warn_if, error_if, limit) -%}
 
-    [{{ schema }}.testview_{{ local_md5(model.name ~ invocation_id) }}]
-  {% endset %}
-
-  {% set sql = main_sql.replace("'", "''")%}
-  {{ get_use_database_sql(database) }}
-  EXEC('create view {{testview}} as {{ sql }};')
+  with test_main_sql as (
+  {{ main_sql }}
+  ),
+  dbt_internal_test as (
+    select {{ "top (" ~ limit ~ ')' if limit != none }} * from test_main_sql
+  )
   select
     {{ fail_calc }} as failures,
     case when {{ fail_calc }} {{ warn_if }}
       then 'true' else 'false' end as should_warn,
     case when {{ fail_calc }} {{ error_if }}
       then 'true' else 'false' end as should_error
-  from (
-    select {{ "top (" ~ limit ~ ')' if limit != none }} * from {{testview}}
-  ) dbt_internal_test;
+  from dbt_internal_test
+
+{%- endmacro %}
+
+
+{% macro fabric__get_unit_test_sql(main_sql, expected_fixture_sql, expected_column_names) -%}
+  -- Build actual result given inputs
+  WITH dbt_internal_unit_test_actual AS (
+
+    WITH main_sql AS (
+      {{ main_sql }}
+    )
+    SELECT
+      {% for expected_column_name in expected_column_names %}
+        {{ expected_column_name }}{% if not loop.last -%},{% endif %}
+      {%- endfor -%},
+      {{ dbt.string_literal("actual") }} AS {{ adapter.quote("actual_or_expected") }}
+    FROM main_sql
+  ),
+
+  -- Build expected result
+  dbt_internal_unit_test_expected AS (
+
+    WITH expected_fixture_sql AS (
+      {{ expected_fixture_sql }}
+    )
+    SELECT
+      {% for expected_column_name in expected_column_names %}
+        {{ expected_column_name }}{% if not loop.last -%}, {% endif %}
+      {%- endfor -%},
+      {{ dbt.string_literal("expected") }} AS {{ adapter.quote("actual_or_expected") }}
+    FROM expected_fixture_sql
+  )
+
+  -- Union actual and expected results
+  SELECT * FROM dbt_internal_unit_test_actual
+  UNION ALL
+  SELECT * FROM dbt_internal_unit_test_expected
 
-  {{ get_use_database_sql(database) }}
-  EXEC('drop view {{testview}};')
 {%- endmacro %}
 
+
 {% macro fabric__generate_schema_name(custom_schema_name, node) -%}
 
     {%- set default_schema = target.schema -%}

dbt/include/fabric/macros/materializations/tests/test.sql

@@ -0,0 +1,8 @@
+{% macro fabric__split_part(string_text, delimiter_text, part_number) %}
+    {% if part_number >= 0 %}
+        (select value from string_split({{ string_text }}, {{ delimiter_text }}, 1) where ordinal = {{ part_number }})
+    {% else %}
+        (select value from string_split({{ string_text }}, {{ delimiter_text }}, 1)
+        where ordinal = len(replace({{ string_text }}, {{delimiter_text}}, '')) + 1 + {{ part_number }})
+    {% endif %}
+{% endmacro %}

dbt/include/fabric/macros/utils/split_part.sql

@@ -1,7 +1,7 @@
 # install latest changes in dbt-core
 # TODO: how to automate switching from develop to version branches?
 git+https://github.com/dbt-labs/dbt-core.git#egg=dbt-core&subdirectory=core
-git+https://github.com/dbt-labs/dbt-adapters.git
+git+https://github.com/dbt-labs/dbt-adapters.git#egg=dbt-adapters&subdirectory=dbt-adapters
 git+https://github.com/dbt-labs/dbt-adapters.git#subdirectory=dbt-tests-adapter
 
 pytest==8.0.1

dev_requirements.txt

@@ -480,7 +480,7 @@ def models(self):
     @pytest.fixture(scope="class")
     def expected_sql(self):
         return """
-EXEC('create view <model_identifier> as -- depends_on: <foreign_key_model_identifier> select ''blue'' as color, 1 as id, ''2019-01-01'' as date_day;'); CREATE TABLE <model_identifier> ( id int not null, color varchar(100), date_day varchar(100) ) INSERT INTO <model_identifier> ( [id], [color], [date_day] ) SELECT [id], [color], [date_day] FROM <model_identifier>
+create table <model_identifier>(id int not null,color varchar(100),date_day varchar(100))exec('create view <model_identifier> as -- depends_on: <foreign_key_model_identifier> select ''blue'' as color,1 as id,''2019-01-01'' as date_day;'); insert into <model_identifier>([id],[color],[date_day])select [id],[color],[date_day] from <model_identifier>
 """
 
     # EXEC('DROP view IF EXISTS <model_identifier>
@@ -549,7 +549,7 @@ def models(self):
     @pytest.fixture(scope="class")
     def expected_sql(self):
         return """
-EXEC('create view <model_identifier> as -- depends_on: <foreign_key_model_identifier> select ''blue'' as color, 1 as id, ''2019-01-01'' as date_day;'); CREATE TABLE <model_identifier> ( id int not null, color varchar(100), date_day varchar(100) ) INSERT INTO <model_identifier> ( [id], [color], [date_day] ) SELECT [id], [color], [date_day] FROM <model_identifier>
+create table <model_identifier>(id int not null,color varchar(100),date_day varchar(100))exec('create view <model_identifier> as -- depends_on: <foreign_key_model_identifier> select ''blue'' as color,1 as id,''2019-01-01'' as date_day;'); insert into <model_identifier>([id],[color],[date_day])select [id],[color],[date_day] from <model_identifier>
 """
 
     # EXEC('DROP view IF EXISTS <model_identifier>