fix: configure pre-commit.ci and clean up codebase formatting

.github/pull_request_template.md

@@ -16,11 +16,11 @@
 
 ## Database Migrations
 
-- 
+-
 
 ## Env Config
 
-- 
+-
 
 ## Relevant Docs
 
@@ -42,4 +42,4 @@
 
 ## Checklist
 
-I have read and understood the [Contribution Guidelines]().
+I have read and understood the [Contribution Guidelines]().

.gitignore

@@ -213,4 +213,4 @@ backend/backend/utils/load_models/yaml_models.yaml
 
 # macOS
 .DS_Store
-**/.DS_Store
+**/.DS_Store

.pre-commit-config.yaml

@@ -7,7 +7,17 @@ ci:
     - mypy # Uses language: system, not available in pre-commit.ci sandbox
     - protolint-docker # Needs Docker, not available in pre-commit.ci
     - hadolint-docker # Needs Docker, not available in pre-commit.ci
-  autofix_prs: true
+    - pycln # Path resolution bug in pre-commit.ci sandbox
+    - end-of-file-fixer # Inconsistent behavior between local and CI environments
+    - flake8 # Pre-existing violations — will clean up separately
+    - markdownlint # Pre-existing violations — will clean up separately
+    - absolufy-imports # Incorrectly rewrites relative imports in monorepo structure
+    - black # Large-scale reformatting — will clean up in dedicated PR
+    - pyupgrade # Pre-existing across 43 files
+    - isort # Pre-existing import ordering across codebase
+    - yesqa # Pre-existing unnecessary noqa comments
+    - docformatter # Pre-existing docstring formatting
+  autofix_prs: false
   autoupdate_schedule: monthly
 
 # Force all unspecified python hooks to run python 3.10
@@ -22,7 +32,10 @@ repos:
       - id: trailing-whitespace
         exclude_types:
           - "markdown"
+          - "svg"
       - id: end-of-file-fixer
+        exclude_types:
+          - "svg"
       - id: check-yaml
         args: [--unsafe]
       - id: check-added-large-files
@@ -36,6 +49,7 @@ repos:
       - id: check-toml
       - id: debug-statements
       - id: detect-private-key
+        exclude: sample\.env$
       # - id: detect-aws-credentials
       #   args: ["--allow-missing-credentials"]
       - id: check-merge-conflict
@@ -54,7 +68,7 @@ repos:
     rev: 24.1.1
     hooks:
       - id: black
-        args: [--config=pyproject.toml]
+        args: [--config=backend/pyproject.toml]
   # - id: black
   #   alias: black-check
   #   stages: [manual]
@@ -82,13 +96,13 @@ repos:
     rev: v2.4.0
     hooks:
       - id: pycln
-        args: [--config=pyproject.toml]
+        args: [--config=backend/pyproject.toml]
   - repo: https://github.com/pycqa/isort
     rev: 5.13.2
     hooks:
       - id: isort
         files: "\\.(py)$"
-        args: [--settings-path=pyproject.toml]
+        args: [--settings-path=backend/pyproject.toml]
   - repo: https://github.com/pycqa/flake8
     rev: 7.0.0
     hooks:
@@ -161,7 +175,7 @@ repos:
     - id: markdownlint
       args: ["--config", "markdownlint.yaml"]
   - repo: https://github.com/pycqa/docformatter
-    rev: v1.7.5
+    rev: v1.7.7
     hooks:
       - id: docformatter
   - repo: https://github.com/adrienverge/yamllint

backend/backend/application/config_parser/config_parser.py

@@ -91,7 +91,7 @@ def unique_keys(self) -> list[str]:
     @property
     def delta_strategy(self) -> dict[str, Any]:
         return self.incremental_config.get("delta_strategy", {})
-    
+
     @property
     def reference(self) -> list[str]:
         if not self._reference:

backend/backend/application/config_parser/transformation_parser.py

@@ -65,19 +65,19 @@ def transform_orders(self) -> list[str]:
     def get_transforms(self) -> list[BaseParser]:
         """
         Generate and yield transformation parsers in the order defined by the configuration.
-    
-        This method processes the `transform_order` list and corresponding `transform` dictionary 
+
+        This method processes the `transform_order` list and corresponding `transform` dictionary
         from the configuration to create parser instances of appropriate types for each transformation.
-    
+
         - It iterates through the `transform_order` to ensure transformations are applied sequentially.
         - For each transformation, it determines the type and maps it to the corresponding parser class.
-        - Certain transformation types (`combine_columns`, `group`, `find_and_replace`, and `distinct`) 
-          require special handling for their configuration. These are instantiated with a modified 
+        - Certain transformation types (`combine_columns`, `group`, `find_and_replace`, and `distinct`)
+          require special handling for their configuration. These are instantiated with a modified
           configuration structure.
         - Other transformations are instantiated normally with their respective configuration data.
-    
+
         Yields:
-            BaseParser: An instance of the transformation parser for each transformation in the order 
+            BaseParser: An instance of the transformation parser for each transformation in the order
             defined by `transform_order`.
         """
         if self._transforms:

backend/backend/application/config_parser/transformation_parsers/rename_parser.py

@@ -33,4 +33,4 @@ def column_names(self) -> list[str]:
 
     @property
     def new_column_names(self) -> list[str]:
-        return [rp.new_name for rp in self.get_rename_parsers()]
+        return [rp.new_name for rp in self.get_rename_parsers()]

backend/backend/application/context/chat_ai_context.py

@@ -272,7 +272,7 @@ def _process_completed(self, *args, **kwargs):
         content = kwargs.get("content")
         token_usage_data = {}
         processing_time_ms = 0
-        
+
         # Check if content is a dictionary and contains token_info
         if isinstance(content, dict):
             token_usage_data = content.get("token_info", {})

backend/backend/application/context/chat_message_context.py

@@ -248,7 +248,7 @@ def persist_response(
             fields_to_update.append("discussion_type")
             if discussion_status == 'GENERATE':
                 chat_message.transformation_type = 'TRANSFORM'
-                fields_to_update.append('transformation_type') 
+                fields_to_update.append('transformation_type')
         if response:
             if is_append_response:
                 chat_message.response = (chat_message.response or "") + response

backend/backend/application/context/environment.py

@@ -36,7 +36,7 @@ def create_environment(self, environment_details: dict) -> dict[str, Any]:
             logging.exception("Failed to decrypt environment creation data")
             # Continue with original data if decryption fails
             decrypted_environment_details = environment_details
-        
+
         env_model = self.env_session.create_environment(environment_details=decrypted_environment_details)
         response_data = {
             "id": env_model.environment_id,
@@ -58,7 +58,7 @@ def update_environment(self, environment_id: str, environment_details: dict[str,
             logging.exception("Failed to decrypt environment update data")
             # Continue with original data if decryption fails
             decrypted_environment_details = environment_details
-        
+
         env_model = self.env_session.update_environment(
             environment_id=environment_id, environment_details=decrypted_environment_details
         )

backend/backend/application/context/no_code_model.py

@@ -196,7 +196,7 @@ def delete_model_transformation(self, model_name: str, transformation_id: str, i
 
     def set_model_presentation(self, no_code_data: dict[str, Any], model_name: str):
         """
-        Updates the 'presentation' configuration of the model. Only updates keys present in 'no_code_data' 
+        Updates the 'presentation' configuration of the model. Only updates keys present in 'no_code_data'
         without affecting other keys.
         """
         model_data = self.session.fetch_model_data(model_name=model_name)
@@ -235,4 +235,4 @@ def get_transformation_columns(
             model_name=model_name,
             transformation_id=transformation_id,
             transformation_type=transformation_type
-        )
+        )

backend/backend/application/interpreter/python_templates/destination_table.jinja

@@ -28,4 +28,4 @@ class {{class_name}}({{previous_class}}):
         if self.delta_strategy.get("type"):
             return self._execute_delta_strategy()
         return self.select()
-    {% endif %}
+    {% endif %}

backend/backend/application/interpreter/python_templates/transformations_template/combine_column.jinja

@@ -27,4 +27,4 @@ except Exception as error:
         model_name=MODEL_NAME,
         error_message=str(error)
     ) from error
-self.save_table_columns(transformation_id="{{ transformation_id }}_transformed", table_obj=source_table)
+self.save_table_columns(transformation_id="{{ transformation_id }}_transformed", table_obj=source_table)

backend/backend/application/interpreter/python_templates/transformations_template/groups_and_aggregation.jinja

@@ -23,4 +23,4 @@ except Exception as error:
         model_name=MODEL_NAME,
         error_message=str(error)
     ) from error
-self.save_table_columns(transformation_id="{{ transformation_id }}_transformed", table_obj=source_table)
+self.save_table_columns(transformation_id="{{ transformation_id }}_transformed", table_obj=source_table)

backend/backend/application/interpreter/python_templates/transformations_template/pivot.jinja

@@ -23,4 +23,4 @@ except Exception as error:
         model_name=MODEL_NAME,
         error_message=str(error)
     ) from error
-self.save_table_columns(transformation_id="{{ transformation_id }}_transformed", table_obj=source_table)
+self.save_table_columns(transformation_id="{{ transformation_id }}_transformed", table_obj=source_table)

backend/backend/application/interpreter/python_templates/transformations_template/unions.jinja

@@ -13,4 +13,4 @@ except IbisTypeError as error:
     raise TransformationFailed(transformation_name="merge", model_name=MODEL_NAME, error_message=str(error)) from error
 except Exception as error:
     raise TransformationFailed(transformation_name="merge", model_name=MODEL_NAME, error_message=str(error)) from error
-self.save_table_columns(transformation_id="{{ transformation_id }}_transformed", table_obj=source_table)
+self.save_table_columns(transformation_id="{{ transformation_id }}_transformed", table_obj=source_table)

backend/backend/application/interpreter/transformations/groups_and_aggregation.py

@@ -753,4 +753,4 @@ def construct_code(self) -> str:
         return self._transformed_code
 
     def transform(self) -> str:
-        return self.construct_code()
+        return self.construct_code()

backend/backend/application/interpreter/transformations/joins.py

@@ -110,7 +110,7 @@ def parse_joins(self, join_list: list[JoinParser]):
         for join_parser in join_list:
             # Parse the left join
             join_successful, class_name = self.__parse_left_joins(join_parser=join_parser)
-            
+
             # If join parsing failed, store parent class
             if not join_successful:
                 self.parent_classes.append(class_name)

backend/backend/application/interpreter/transformations/pivot.py

@@ -19,7 +19,7 @@ def get_values_fill(self) -> str:
                     return f"values_fill={int(fill_null)})"
                 except (ValueError, TypeError):
                     """
-                    suppress the exception since the column type doesn't support string types 
+                    suppress the exception since the column type doesn't support string types
                     """
                     pass
             else:

backend/backend/application/model_validator/transformations/synthesis_validator.py

@@ -19,4 +19,4 @@ def check_column_usage(self, columns: list[str]) -> list[str]:
             for col in columns:
                 if col == expr or col in expr:
                     still_used.add(col)
-        return list(still_used)
+        return list(still_used)

backend/backend/application/session/connection_session.py

@@ -71,7 +71,7 @@ def get_all_connections(page: int, limit: int, filter_condition: dict[str, Any])
                     "is_connection_valid": con_model.is_connection_valid,
                     "connection_flag": con_model.connection_flag,
                     "is_sample_project": is_sample_project,
-                    # "connection_details": con_model.connection_details, # skipping connection_details 
+                    # "connection_details": con_model.connection_details, # skipping connection_details
                 }
             )
 

backend/backend/application/session/session.py

@@ -58,10 +58,10 @@ def update_project_details(self, project_details: dict[str, Any]):
 
     def update_project_connection(self, connection_details: dict[str, Any]) -> dict[str, Any]:
         # TODO - Need to remove the project_connection update from project level
-        
+
         # Decrypt sensitive fields from frontend encrypted data
         decrypted_connection_details = decrypt_sensitive_fields(connection_details)
-        
+
         connection_model = self.project_instance.connection_model
         connection_model.connection_details = decrypted_connection_details
         connection_model.save()

backend/backend/application/utils.py

@@ -256,4 +256,4 @@ def replace_in(match):
     parts = [p.strip() for p in match.group(1).split(",")]
     col, *values = parts
     conditions = [f"{col} = {v}" for v in values]
-    return f"OR({', '.join(conditions)})"
+    return f"OR({', '.join(conditions)})"

backend/backend/application/visitran_backend_context.py

@@ -414,10 +414,10 @@ def test_connection_data(self, connection_data: dict[str, Any], db_type: str) ->
         db_type = db_type or self.database_type
         if not connection_data:
             connection_data = {"file_path": f"{self.project_path}{os.path.sep}models/local.db"}
-        
+
         # Decrypt sensitive fields from frontend encrypted data
         decrypted_connection_data = decrypt_sensitive_fields(connection_data)
-        
+
         connection_cls: type[BaseConnection] = get_adapter_connection_cls(db_type)
         old_connection = self._conn_details
         self._conn_details = decrypted_connection_data