Merge pull request #56 from microsoft/v1.5.0rc2

V1.5.0rc2

Author: prdpsvs

.github/workflows/integration-tests-azure.yml

@@ -12,7 +12,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python_version: ["3.7", "3.8", "3.9", "3.10", "3.11"]
+        python_version: ["3.8", "3.9", "3.10", "3.11"]
         profile: ["ci_azure_auto"]
         msodbc_version: ["17", "18"]
       max-parallel: 1

.github/workflows/publish-docker.yml

@@ -13,7 +13,7 @@ jobs:
   publish-docker-client:
     strategy:
       matrix:
-        python_version: ["3.7", "3.8", "3.9", "3.10", "3.11"]
+        python_version: ["3.8", "3.9", "3.10", "3.11"]
         docker_target: ["msodbc17", "msodbc18"]
     runs-on: ubuntu-latest
     permissions:
@@ -24,7 +24,7 @@ jobs:
         uses: actions/checkout@v3
 
       - name: Log in to the Container registry
-        uses: docker/login-action@v2.1.0
+        uses: docker/login-action@v2.2.0
         with:
           registry: ghcr.io
           username: ${{ github.actor }}

.github/workflows/unit-tests.yml

@@ -16,7 +16,7 @@ jobs:
     name: Unit tests
     strategy:
       matrix:
-        python_version: ["3.7", "3.8", "3.9", "3.10", "3.11"]
+        python_version: ["3.8", "3.9", "3.10", "3.11"]
     runs-on: ubuntu-latest
     permissions:
       contents: read

.pre-commit-config.yaml

@@ -17,7 +17,7 @@ repos:
       - id: check-builtin-literals
       - id: check-merge-conflict
       - id: no-commit-to-branch
-      - id: fix-byte-order-marker
+      # - id: fix-byte-order-marker
       - id: mixed-line-ending
       - id: check-docstring-first
   - repo: 'https://github.com/adrienverge/yamllint'
@@ -66,14 +66,14 @@ repos:
           - '--check'
           - '--diff'
   - repo: 'https://github.com/pycqa/flake8'
-    rev: 6.0.0
+    rev: 6.1.0
     hooks:
       - id: flake8
         args:
-          - '--max-line-length=99'
+          - '--max-line-length=1000'
       - id: flake8
         args:
-          - '--max-line-length=99'
+          - '--max-line-length=1000'
         alias: flake8-check
         stages:
           - manual

CHANGELOG.md

@@ -1,8 +1,19 @@
 # Changelog
 
+### v1.5.0-rc1
+
+* Upgraded dbt-fabric adapter to match dbt-core & dbt-tests-adapter version 1.5.2.
+* Added constraint support to dbt-fabric adapter.
+    * Check constraints are not supported.
+    * Column & model constraints are not supported in CREATE TABLE command by Microsoft Fabric Data Warehouse. Column and model constraints are implemented by ALTER TABLE ADD Constraints command.
+    * user-defined names for constraints are not currently supported. naming is handled by the adapter, until `SP_RENAME` is supported in Fabric
+    * Added tests related to constraints.
+* Bumped wheel, precommit, docker package versions.
+
+
 ### v1.4.0-rc3
 
-Updated connection property to track dbt telemetry
+Updated connection property to track dbt telemetry by Microsoft.
 
 ### v1.4.0-rc2
 

dbt/adapters/fabric/__version__.py

@@ -1 +1 @@
-version = "1.4.0-rc3"
+version = "1.5.0-rc1"

dbt/adapters/fabric/fabric_adapter.py

@@ -1,10 +1,19 @@
 from typing import List, Optional
 
 import agate
+from dbt.adapters.base import Column as BaseColumn
+
+# from dbt.events.functions import fire_event
+# from dbt.events.types import SchemaCreation
+from dbt.adapters.base.impl import ConstraintSupport
+from dbt.adapters.base.meta import available
 from dbt.adapters.base.relation import BaseRelation
-from dbt.adapters.cache import _make_ref_key_msg
+from dbt.adapters.cache import _make_ref_key_dict
+
+# from dbt.adapters.cache import _make_ref_key_msg
 from dbt.adapters.sql import SQLAdapter
 from dbt.adapters.sql.impl import CREATE_SCHEMA_MACRO_NAME
+from dbt.contracts.graph.nodes import ColumnLevelConstraint, ConstraintType, ModelLevelConstraint
 from dbt.events.functions import fire_event
 from dbt.events.types import SchemaCreation
 
@@ -18,9 +27,44 @@ class FabricAdapter(SQLAdapter):
     Column = FabricColumn
     AdapterSpecificConfigs = FabricConfigs
 
+    CONSTRAINT_SUPPORT = {
+        ConstraintType.check: ConstraintSupport.NOT_SUPPORTED,
+        ConstraintType.not_null: ConstraintSupport.ENFORCED,
+        ConstraintType.unique: ConstraintSupport.ENFORCED,
+        ConstraintType.primary_key: ConstraintSupport.ENFORCED,
+        ConstraintType.foreign_key: ConstraintSupport.ENFORCED,
+    }
+
+    @available.parse(lambda *a, **k: [])
+    def get_column_schema_from_query(self, sql: str) -> List[BaseColumn]:
+        """Get a list of the Columns with names and data types from the given sql."""
+        _, cursor = self.connections.add_select_query(sql)
+
+        columns = [
+            self.Column.create(
+                column_name, self.connections.data_type_code_to_name(column_type_code)
+            )
+            # https://peps.python.org/pep-0249/#description
+            for column_name, column_type_code, *_ in cursor.description
+        ]
+        return columns
+
+    @classmethod
+    def convert_boolean_type(cls, agate_table, col_idx):
+        return "bit"
+
+    @classmethod
+    def convert_datetime_type(cls, agate_table, col_idx):
+        return "datetime2(6)"
+
+    @classmethod
+    def convert_number_type(cls, agate_table, col_idx):
+        decimals = agate_table.aggregate(agate.MaxPrecision(col_idx))
+        return "float" if decimals else "int"
+
     def create_schema(self, relation: BaseRelation) -> None:
         relation = relation.without_identifier()
-        fire_event(SchemaCreation(relation=_make_ref_key_msg(relation)))
+        fire_event(SchemaCreation(relation=_make_ref_key_dict(relation)))
         macro_name = CREATE_SCHEMA_MACRO_NAME
         kwargs = {
             "relation": relation,
@@ -33,10 +77,6 @@ def create_schema(self, relation: BaseRelation) -> None:
         self.execute_macro(macro_name, kwargs=kwargs)
         self.commit_if_has_connection()
 
-    @classmethod
-    def date_function(cls):
-        return "getdate()"
-
     @classmethod
     def convert_text_type(cls, agate_table, col_idx):
         column = agate_table.columns[col_idx]
@@ -46,23 +86,14 @@ def convert_text_type(cls, agate_table, col_idx):
         length = max_len if max_len > 16 else 16
         return "varchar({})".format(length)
 
-    @classmethod
-    def convert_datetime_type(cls, agate_table, col_idx):
-        return "datetime2(6)"
-
-    @classmethod
-    def convert_boolean_type(cls, agate_table, col_idx):
-        return "bit"
-
-    @classmethod
-    def convert_number_type(cls, agate_table, col_idx):
-        decimals = agate_table.aggregate(agate.MaxPrecision(col_idx))
-        return "float" if decimals else "int"
-
     @classmethod
     def convert_time_type(cls, agate_table, col_idx):
         return "time(6)"
 
+    @classmethod
+    def date_function(cls):
+        return "getdate()"
+
     # Methods used in adapter tests
     def timestamp_add_sql(self, add_to: str, number: int = 1, interval: str = "hour") -> str:
         # note: 'interval' is not supported for T-SQL
@@ -145,6 +176,45 @@ def run_sql_for_tests(self, sql, fetch, conn):
         finally:
             conn.transaction_open = False
 
+    @available
+    @classmethod
+    def render_column_constraint(cls, constraint: ColumnLevelConstraint) -> Optional[str]:
+        rendered_column_constraint = None
+        if constraint.type == ConstraintType.not_null:
+            rendered_column_constraint = "not null "
+        else:
+            rendered_column_constraint = ""
+
+        if rendered_column_constraint:
+            rendered_column_constraint = rendered_column_constraint.strip()
+
+        return rendered_column_constraint
+
+    @classmethod
+    def render_model_constraint(cls, constraint: ModelLevelConstraint) -> Optional[str]:
+        constraint_prefix = "add constraint "
+        column_list = ", ".join(constraint.columns)
+
+        if constraint.type == ConstraintType.unique:
+            return (
+                constraint_prefix
+                + f"{constraint.name} unique nonclustered({column_list}) not enforced"
+            )
+        elif constraint.type == ConstraintType.primary_key:
+            return (
+                constraint_prefix
+                + f"{constraint.name} primary key nonclustered({column_list}) not enforced"
+            )
+        elif constraint.type == ConstraintType.foreign_key and constraint.expression:
+            return (
+                constraint_prefix
+                + f"{constraint.name} foreign key({column_list}) references {constraint.expression} not enforced"
+            )
+        elif constraint.type == ConstraintType.custom and constraint.expression:
+            return f"{constraint_prefix}{constraint.expression}"
+        else:
+            return None
+
 
 COLUMNS_EQUAL_SQL = """
 with diff_count as (

dbt/adapters/fabric/fabric_connection_manager.py

@@ -3,7 +3,7 @@
 import time
 from contextlib import contextmanager
 from itertools import chain, repeat
-from typing import Any, Callable, Dict, Mapping, Optional, Tuple
+from typing import Any, Callable, Dict, Mapping, Optional, Tuple, Union
 
 import agate
 import dbt.exceptions
@@ -30,6 +30,27 @@
 
 logger = AdapterLogger("fabric")
 
+# https://github.com/mkleehammer/pyodbc/wiki/Data-Types
+datatypes = {
+    # "str": "char",
+    "str": "varchar",
+    "uuid.UUID": "uniqueidentifier",
+    "uuid": "uniqueidentifier",
+    # "float": "real",
+    # "float": "float",
+    "float": "bigint",
+    # "int": "smallint",
+    # "int": "tinyint",
+    "int": "int",
+    "bytes": "varbinary",
+    "bool": "bit",
+    "datetime.date": "date",
+    "datetime.datetime": "datetime2(6)",
+    "datetime.time": "time",
+    "decimal.Decimal": "decimal",
+    # "decimal.Decimal": "numeric",
+}
+
 
 def convert_bytes_to_mswindows_byte_string(value: bytes) -> bytes:
     """
@@ -469,6 +490,11 @@ def get_response(cls, cursor: Any) -> AdapterResponse:
             rows_affected=rows,
         )
 
+    @classmethod
+    def data_type_code_to_name(cls, type_code: Union[str, str]) -> str:
+        data_type = str(type_code)[str(type_code).index("'") + 1 : str(type_code).rindex("'")]
+        return datatypes[data_type]
+
     def execute(
         self, sql: str, auto_begin: bool = True, fetch: bool = False
     ) -> Tuple[AdapterResponse, agate.Table]:

dbt/include/fabric/macros/adapters/columns.sql

@@ -1,3 +1,15 @@
+{% macro fabric__get_empty_subquery_sql(select_sql, select_sql_header=none) %}
+    {% if sql.strip().lower().startswith('with') %}
+        {{ select_sql }}
+    {% else -%}
+        select * from (
+        {{ select_sql }}
+    ) dbt_sbq_tmp
+    where 1 = 0
+    {%- endif -%}
+
+{% endmacro %}
+
 {% macro fabric__get_columns_in_relation(relation) -%}
   {% call statement('get_columns_in_relation', fetch_result=True) %}
 
@@ -29,7 +41,6 @@
   {{ return(sql_convert_columns_in_relation(table)) }}
 {% endmacro %}
 
-
 {% macro fabric__get_columns_in_query(select_sql) %}
     {% call statement('get_columns_in_query', fetch_result=True, auto_begin=False) -%}
         select TOP 0 * from (