Consolidate pre-commit config

.github/actions/latest-wrangler/main.py

@@ -1,9 +1,10 @@
 import os
-from packaging.version import Version, parse
-import requests
 import sys
 from typing import List
 
+import requests
+from packaging.version import Version, parse
+
 
 def main():
     package_name: str = os.environ["INPUT_PACKAGE_NAME"]

.pre-commit-config.yaml

@@ -8,73 +8,84 @@ default_language_version:
   python: python3
 
 repos:
-- repo: https://github.com/pre-commit/pre-commit-hooks
-  rev: v3.2.0
-  hooks:
-  - id: check-yaml
-    args: [--unsafe]
-  - id: check-json
-  - id: end-of-file-fixer
-    exclude: schemas/dbt/manifest/
-  - id: trailing-whitespace
-    exclude_types:
-      - "markdown"
-  - id: check-case-conflict
-- repo: https://github.com/pycqa/isort
-  # rev must match what's in dev-requirements.txt
-  rev: 5.13.2
-  hooks:
-    - id: isort
-- repo: https://github.com/psf/black
-  # rev must match what's in dev-requirements.txt
-  rev: 24.3.0
-  hooks:
-  - id: black
-  - id: black
-    alias: black-check
-    stages: [manual]
-    args:
-    - "--check"
-    - "--diff"
-- repo: https://github.com/pycqa/flake8
-  # rev must match what's in dev-requirements.txt
-  rev: 4.0.1
-  hooks:
-  - id: flake8
-  - id: flake8
-    alias: flake8-check
-    stages: [manual]
-- repo: https://github.com/pre-commit/mirrors-mypy
-  # rev must match what's in dev-requirements.txt
-  rev: v1.4.1
-  hooks:
-  - id: mypy
-    # N.B.: Mypy is... a bit fragile.
-    #
-    # By using `language: system` we run this hook in the local
-    # environment instead of a pre-commit isolated one.  This is needed
-    # to ensure mypy correctly parses the project.
-
-    # It may cause trouble
-    # in that it adds environmental variables out of our control to the
-    # mix.  Unfortunately, there's nothing we can do about per pre-commit's
-    # author.
-    # See https://github.com/pre-commit/pre-commit/issues/730 for details.
-    args: [--show-error-codes]
-    files: ^core/dbt/
-    language: system
-  - id: mypy
-    alias: mypy-check
-    stages: [manual]
-    args: [--show-error-codes, --pretty]
-    files: ^core/dbt/
-    language: system
-- repo: local
-  hooks:
-    - id: no_versioned_artifact_resource_imports
-      name: no_versioned_artifact_resource_imports
-      entry: python custom-hooks/no_versioned_artifact_resource_imports.py
-      language: system
-      files: ^core/dbt/
-      types: [python]
-      pass_filenames: true
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v3.2.0
+    hooks:
+      - id: check-yaml
+        args: [--unsafe]
+      - id: check-json
+      - id: end-of-file-fixer
+        exclude: schemas/dbt/manifest/
+      - id: trailing-whitespace
+        exclude_types:
+          - "markdown"
+      - id: check-case-conflict
+  # local hooks are used to run the hooks in the local environment instead of a pre-commit isolated one.
+  # This ensures that the hooks are run with the same version of the dependencies as the local environment
+  # without having to manually keep them in sync.
+  - repo: local
+    hooks:
+      # Formatter/linter/type-checker pins live in the pyproject.dev optional dependency.
+      - id: isort
+        name: isort
+        entry: python -m isort
+        language: system
+        types: [python]
+      - id: black
+        name: black
+        entry: python -m black
+        language: system
+        types: [python]
+      - id: black-check
+        name: black-check
+        entry: python -m black
+        args:
+          - "--check"
+          - "--diff"
+        language: system
+        stages: [manual]
+        types: [python]
+      - id: flake8
+        name: flake8
+        entry: python -m flake8
+        language: system
+        types: [python]
+      - id: flake8-check
+        name: flake8-check
+        entry: python -m flake8
+        language: system
+        stages: [manual]
+        types: [python]
+      # N.B.: Mypy is... a bit fragile.
+      #
+      # By using `language: system` we run this hook in the local
+      # environment instead of a pre-commit isolated one.  This is needed
+      # to ensure mypy correctly parses the project.
+      #
+      # It may cause trouble
+      # in that it adds environmental variables out of our control to the
+      # mix.  Unfortunately, there's nothing we can do about per pre-commit's
+      # author.
+      # See https://github.com/pre-commit/pre-commit/issues/730 for details.
+      - id: mypy
+        name: mypy
+        entry: python -m mypy
+        args: [--show-error-codes]
+        files: ^core/dbt/
+        language: system
+        types: [python]
+      - id: mypy-check
+        name: mypy-check
+        entry: python -m mypy
+        args: [--show-error-codes, --pretty]
+        files: ^core/dbt/
+        language: system
+        stages: [manual]
+        types: [python]
+      - id: no_versioned_artifact_resource_imports
+        name: no_versioned_artifact_resource_imports
+        entry: python custom-hooks/no_versioned_artifact_resource_imports.py
+        language: system
+        files: ^core/dbt/
+        types: [python]
+        pass_filenames: true

dev-requirements.txt

@@ -2,20 +2,16 @@ git+https://github.com/dbt-labs/dbt-adapters.git@main#subdirectory=dbt-adapters
 git+https://github.com/dbt-labs/dbt-adapters.git@main#subdirectory=dbt-tests-adapter
 git+https://github.com/dbt-labs/dbt-common.git@main
 git+https://github.com/dbt-labs/dbt-adapters.git@main#subdirectory=dbt-postgres
-# black must match what's in .pre-commit-config.yaml to be sure local env matches CI
 black==24.3.0
 bumpversion
 ddtrace==2.21.3
 docutils
-# flake8 must match what's in .pre-commit-config.yaml to be sure local env matches CI
 flake8==4.0.1
 flaky
 freezegun>=1.5.1
 hypothesis
 ipdb
-# isort must match what's in .pre-commit-config.yaml to be sure local env matches CI
 isort==5.13.2
-# mypy must match what's in .pre-commit-config.yaml to be sure local env matches CI
 mypy==1.4.1
 pip-tools
 pre-commit

scripts/check_libyaml.py

@@ -1,8 +1,10 @@
 #!/usr/bin/env python
 try:
-    from yaml import CLoader as Loader, CSafeLoader as SafeLoader, CDumper as Dumper  # noqa: F401
+    from yaml import CDumper as Dumper
+    from yaml import CLoader as Loader  # noqa: F401
+    from yaml import CSafeLoader as SafeLoader
 except ImportError:
-    from yaml import Loader, SafeLoader, Dumper  # noqa: F401
+    from yaml import Dumper, Loader, SafeLoader  # noqa: F401
 
 if Loader.__name__ == "CLoader":
     print("libyaml is working")

scripts/collect-artifact-schema.py

@@ -1,15 +1,15 @@
 #!/usr/bin/env python
-from dataclasses import dataclass
+import json
 from argparse import ArgumentParser
+from dataclasses import dataclass
 from pathlib import Path
-import json
-from typing import Type, Dict, Any
+from typing import Any, Dict, Type
 
-from dbt.contracts.graph.manifest import WritableManifest
+from dbt.artifacts.schemas.base import VersionedSchema
 from dbt.artifacts.schemas.catalog import CatalogArtifact
-from dbt.artifacts.schemas.run import RunResultsArtifact
 from dbt.artifacts.schemas.freshness import FreshnessExecutionResultArtifact
-from dbt.artifacts.schemas.base import VersionedSchema
+from dbt.artifacts.schemas.run import RunResultsArtifact
+from dbt.contracts.graph.manifest import WritableManifest
 from dbt_common.clients.system import write_file
 
 

scripts/collect-dbt-contexts.py

@@ -3,14 +3,12 @@
 import inspect
 import json
 from dataclasses import dataclass
-from typing import List, Optional, Iterable, Union, Dict, Any
-from dbt_common.dataclass_schema import dbtClassMixin
-
+from typing import Any, Dict, Iterable, List, Optional, Union
 
 from dbt.context.base import BaseContext
+from dbt.context.providers import MacroContext, ModelContext
 from dbt.context.target import TargetContext
-from dbt.context.providers import ModelContext, MacroContext
-
+from dbt_common.dataclass_schema import dbtClassMixin
 
 CONTEXTS_MAP = {
     "base": BaseContext,