diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index e3c709f..d4e3ff4 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -16,11 +16,11 @@ ## Database Migrations -- +- ## Env Config -- +- ## Relevant Docs @@ -42,4 +42,4 @@ ## Checklist -I have read and understood the [Contribution Guidelines](). \ No newline at end of file +I have read and understood the [Contribution Guidelines](). diff --git a/.github/workflows/core-backend-full-tests-parallel.yaml b/.github/workflows/core-backend-full-tests-parallel.yaml deleted file mode 100644 index d85895a..0000000 --- a/.github/workflows/core-backend-full-tests-parallel.yaml +++ /dev/null @@ -1,87 +0,0 @@ -#--- -# name: Run Tests Parallely -# -# on: -# workflow_dispatch: -# push: -# branches: ["main"] -# pull_request: -# branches: ["main"] -# types: [labeled] -# -# -# concurrency: -# group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} -# cancel-in-progress: true -# env: -# FORCE_COLOR: "1" -# -# jobs: -# run_tests_parallely: -# name: Run Tests Parallely -# runs-on: ubuntu-latest -# if: contains(github.event.pull_request.labels.*.name, 'Run Tests Parallely') || -# contains(github.event_name, 'workflow_dispatch') -# strategy: -# fail-fast: true -# matrix: -# python-version: ["3.11"] #,"3.9","3.10"] -# steps: -# #---------------------------------------------- -# # check-out repo and set-up python -# #---------------------------------------------- -# - uses: actions/checkout@v4 -# with: -# lfs: true -# - name: Setup Python -# uses: ./.github/actions/setup-python/ -# with: -# python-version: ${{ matrix.python-version }} -# - name: Login to Docker Hub -# uses: docker/login-action@v3 -# with: -# username: ${{ secrets.DOCKERHUB_USERNAME }} -# password: ${{ secrets.DOCKERHUB_TOKEN }} -# continue-on-error: true -# - name: Start visitran test containers -# run: | -# docker compose up --wait -# - id: 'auth' -# uses: 'google-github-actions/auth@v2' -# with: -# credentials_json: '${{ secrets.GCP_BIGQUERY_SECRET }}' -# - name: 'Set up Cloud SDK' -# uses: 'google-github-actions/setup-gcloud@v2' -# - name: 'Use gcloud CLI' -# run: 'gcloud info' -# -# - name: Run tests -# env: -# SNOWFLAKE_USERNAME: ${{ secrets.SNOWFLAKE_USERNAME }} -# SNOWFLAKE_PASSWORD: ${{ secrets.SNOWFLAKE_PASSWORD }} -# SNOWFLAKE_ACCOUNT: ${{ secrets.SNOWFLAKE_ACCOUNT }} -# run: | -# uv run pytest -vv --cov=visitran --cov=visitran_cli --cov=visitran_adapters --cov=visitran_backend \ -# --cov-report=xml --cov-config=pyproject.toml --dist loadgroup -n 5 tests visitran_backend -# -# - name: Check code coverage -# run: | -# uv run coverage report -m -# coverage=$(uv run coverage report --format=total) -# echo "Coverage is $coverage" -# - name: Stop test containers -# run: | -# docker compose down -# - name: Git fetch unshallow -# run: | -# git fetch --unshallow -# - name: Core SonarCloud Scan -# uses: SonarSource/sonarcloud-github-action@master -# if: ${{ github.actor != 'dependabot[bot]' }} -# env: -# GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Needed to get PR information, if any -# SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} -# with: -# projectBaseDir: ./ -# args: > -# -Dproject.settings=./sonar-project.properties diff --git a/.gitignore b/.gitignore index 8d49727..d62c433 100644 --- a/.gitignore +++ b/.gitignore @@ -213,4 +213,4 @@ backend/backend/utils/load_models/yaml_models.yaml # macOS .DS_Store -**/.DS_Store \ No newline at end of file +**/.DS_Store diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 708c772..9c7b966 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -7,7 +7,17 @@ ci: - mypy # Uses language: system, not available in pre-commit.ci sandbox - protolint-docker # Needs Docker, not available in pre-commit.ci - hadolint-docker # Needs Docker, not available in pre-commit.ci - autofix_prs: true + - pycln # Path resolution bug in pre-commit.ci sandbox + - end-of-file-fixer # Inconsistent behavior between local and CI environments + - flake8 # Pre-existing violations — will clean up separately + - markdownlint # Pre-existing violations — will clean up separately + - absolufy-imports # Incorrectly rewrites relative imports in monorepo structure + - black # Large-scale reformatting — will clean up in dedicated PR + - pyupgrade # Pre-existing across 43 files + - isort # Pre-existing import ordering across codebase + - yesqa # Pre-existing unnecessary noqa comments + - docformatter # Pre-existing docstring formatting + autofix_prs: false autoupdate_schedule: monthly # Force all unspecified python hooks to run python 3.10 @@ -22,7 +32,10 @@ repos: - id: trailing-whitespace exclude_types: - "markdown" + - "svg" - id: end-of-file-fixer + exclude_types: + - "svg" - id: check-yaml args: [--unsafe] - id: check-added-large-files @@ -36,6 +49,7 @@ repos: - id: check-toml - id: debug-statements - id: detect-private-key + exclude: sample\.env$ # - id: detect-aws-credentials # args: ["--allow-missing-credentials"] - id: check-merge-conflict @@ -54,7 +68,7 @@ repos: rev: 24.1.1 hooks: - id: black - args: [--config=pyproject.toml] + args: [--config=backend/pyproject.toml] # - id: black # alias: black-check # stages: [manual] @@ -82,13 +96,13 @@ repos: rev: v2.4.0 hooks: - id: pycln - args: [--config=pyproject.toml] + args: [--config=backend/pyproject.toml] - repo: https://github.com/pycqa/isort rev: 5.13.2 hooks: - id: isort files: "\\.(py)$" - args: [--settings-path=pyproject.toml] + args: [--settings-path=backend/pyproject.toml] - repo: https://github.com/pycqa/flake8 rev: 7.0.0 hooks: @@ -161,7 +175,7 @@ repos: - id: markdownlint args: ["--config", "markdownlint.yaml"] - repo: https://github.com/pycqa/docformatter - rev: v1.7.5 + rev: v1.7.7 hooks: - id: docformatter - repo: https://github.com/adrienverge/yamllint diff --git a/backend/backend/application/config_parser/config_parser.py b/backend/backend/application/config_parser/config_parser.py index 08d7f41..0d06a50 100644 --- a/backend/backend/application/config_parser/config_parser.py +++ b/backend/backend/application/config_parser/config_parser.py @@ -91,7 +91,7 @@ def unique_keys(self) -> list[str]: @property def delta_strategy(self) -> dict[str, Any]: return self.incremental_config.get("delta_strategy", {}) - + @property def reference(self) -> list[str]: if not self._reference: diff --git a/backend/backend/application/config_parser/transformation_parser.py b/backend/backend/application/config_parser/transformation_parser.py index 8ff1df6..41a00e1 100644 --- a/backend/backend/application/config_parser/transformation_parser.py +++ b/backend/backend/application/config_parser/transformation_parser.py @@ -65,19 +65,19 @@ def transform_orders(self) -> list[str]: def get_transforms(self) -> list[BaseParser]: """ Generate and yield transformation parsers in the order defined by the configuration. - - This method processes the `transform_order` list and corresponding `transform` dictionary + + This method processes the `transform_order` list and corresponding `transform` dictionary from the configuration to create parser instances of appropriate types for each transformation. - + - It iterates through the `transform_order` to ensure transformations are applied sequentially. - For each transformation, it determines the type and maps it to the corresponding parser class. - - Certain transformation types (`combine_columns`, `group`, `find_and_replace`, and `distinct`) - require special handling for their configuration. These are instantiated with a modified + - Certain transformation types (`combine_columns`, `group`, `find_and_replace`, and `distinct`) + require special handling for their configuration. These are instantiated with a modified configuration structure. - Other transformations are instantiated normally with their respective configuration data. - + Yields: - BaseParser: An instance of the transformation parser for each transformation in the order + BaseParser: An instance of the transformation parser for each transformation in the order defined by `transform_order`. """ if self._transforms: diff --git a/backend/backend/application/config_parser/transformation_parsers/rename_parser.py b/backend/backend/application/config_parser/transformation_parsers/rename_parser.py index b32c1f0..1a27dd5 100644 --- a/backend/backend/application/config_parser/transformation_parsers/rename_parser.py +++ b/backend/backend/application/config_parser/transformation_parsers/rename_parser.py @@ -33,4 +33,4 @@ def column_names(self) -> list[str]: @property def new_column_names(self) -> list[str]: - return [rp.new_name for rp in self.get_rename_parsers()] \ No newline at end of file + return [rp.new_name for rp in self.get_rename_parsers()] diff --git a/backend/backend/application/context/chat_ai_context.py b/backend/backend/application/context/chat_ai_context.py index a0cafd3..8506dca 100644 --- a/backend/backend/application/context/chat_ai_context.py +++ b/backend/backend/application/context/chat_ai_context.py @@ -272,7 +272,7 @@ def _process_completed(self, *args, **kwargs): content = kwargs.get("content") token_usage_data = {} processing_time_ms = 0 - + # Check if content is a dictionary and contains token_info if isinstance(content, dict): token_usage_data = content.get("token_info", {}) diff --git a/backend/backend/application/context/chat_message_context.py b/backend/backend/application/context/chat_message_context.py index b5b8f58..aeb41d9 100644 --- a/backend/backend/application/context/chat_message_context.py +++ b/backend/backend/application/context/chat_message_context.py @@ -248,7 +248,7 @@ def persist_response( fields_to_update.append("discussion_type") if discussion_status == 'GENERATE': chat_message.transformation_type = 'TRANSFORM' - fields_to_update.append('transformation_type') + fields_to_update.append('transformation_type') if response: if is_append_response: chat_message.response = (chat_message.response or "") + response diff --git a/backend/backend/application/context/environment.py b/backend/backend/application/context/environment.py index 906c05a..3b87f74 100644 --- a/backend/backend/application/context/environment.py +++ b/backend/backend/application/context/environment.py @@ -36,7 +36,7 @@ def create_environment(self, environment_details: dict) -> dict[str, Any]: logging.exception("Failed to decrypt environment creation data") # Continue with original data if decryption fails decrypted_environment_details = environment_details - + env_model = self.env_session.create_environment(environment_details=decrypted_environment_details) response_data = { "id": env_model.environment_id, @@ -58,7 +58,7 @@ def update_environment(self, environment_id: str, environment_details: dict[str, logging.exception("Failed to decrypt environment update data") # Continue with original data if decryption fails decrypted_environment_details = environment_details - + env_model = self.env_session.update_environment( environment_id=environment_id, environment_details=decrypted_environment_details ) diff --git a/backend/backend/application/context/no_code_model.py b/backend/backend/application/context/no_code_model.py index 9230c07..1d49e60 100644 --- a/backend/backend/application/context/no_code_model.py +++ b/backend/backend/application/context/no_code_model.py @@ -196,7 +196,7 @@ def delete_model_transformation(self, model_name: str, transformation_id: str, i def set_model_presentation(self, no_code_data: dict[str, Any], model_name: str): """ - Updates the 'presentation' configuration of the model. Only updates keys present in 'no_code_data' + Updates the 'presentation' configuration of the model. Only updates keys present in 'no_code_data' without affecting other keys. """ model_data = self.session.fetch_model_data(model_name=model_name) @@ -235,4 +235,4 @@ def get_transformation_columns( model_name=model_name, transformation_id=transformation_id, transformation_type=transformation_type - ) \ No newline at end of file + ) diff --git a/backend/backend/application/interpreter/python_templates/destination_table.jinja b/backend/backend/application/interpreter/python_templates/destination_table.jinja index 38d5a00..76ec713 100644 --- a/backend/backend/application/interpreter/python_templates/destination_table.jinja +++ b/backend/backend/application/interpreter/python_templates/destination_table.jinja @@ -28,4 +28,4 @@ class {{class_name}}({{previous_class}}): if self.delta_strategy.get("type"): return self._execute_delta_strategy() return self.select() - {% endif %} \ No newline at end of file + {% endif %} diff --git a/backend/backend/application/interpreter/python_templates/transformations_template/combine_column.jinja b/backend/backend/application/interpreter/python_templates/transformations_template/combine_column.jinja index 1e29c84..da061c3 100644 --- a/backend/backend/application/interpreter/python_templates/transformations_template/combine_column.jinja +++ b/backend/backend/application/interpreter/python_templates/transformations_template/combine_column.jinja @@ -27,4 +27,4 @@ except Exception as error: model_name=MODEL_NAME, error_message=str(error) ) from error -self.save_table_columns(transformation_id="{{ transformation_id }}_transformed", table_obj=source_table) \ No newline at end of file +self.save_table_columns(transformation_id="{{ transformation_id }}_transformed", table_obj=source_table) diff --git a/backend/backend/application/interpreter/python_templates/transformations_template/groups_and_aggregation.jinja b/backend/backend/application/interpreter/python_templates/transformations_template/groups_and_aggregation.jinja index d42b18f..ab6a90f 100644 --- a/backend/backend/application/interpreter/python_templates/transformations_template/groups_and_aggregation.jinja +++ b/backend/backend/application/interpreter/python_templates/transformations_template/groups_and_aggregation.jinja @@ -23,4 +23,4 @@ except Exception as error: model_name=MODEL_NAME, error_message=str(error) ) from error -self.save_table_columns(transformation_id="{{ transformation_id }}_transformed", table_obj=source_table) \ No newline at end of file +self.save_table_columns(transformation_id="{{ transformation_id }}_transformed", table_obj=source_table) diff --git a/backend/backend/application/interpreter/python_templates/transformations_template/pivot.jinja b/backend/backend/application/interpreter/python_templates/transformations_template/pivot.jinja index 14433b3..6cd5429 100644 --- a/backend/backend/application/interpreter/python_templates/transformations_template/pivot.jinja +++ b/backend/backend/application/interpreter/python_templates/transformations_template/pivot.jinja @@ -23,4 +23,4 @@ except Exception as error: model_name=MODEL_NAME, error_message=str(error) ) from error -self.save_table_columns(transformation_id="{{ transformation_id }}_transformed", table_obj=source_table) \ No newline at end of file +self.save_table_columns(transformation_id="{{ transformation_id }}_transformed", table_obj=source_table) diff --git a/backend/backend/application/interpreter/python_templates/transformations_template/unions.jinja b/backend/backend/application/interpreter/python_templates/transformations_template/unions.jinja index 8f492d1..d5ccf78 100644 --- a/backend/backend/application/interpreter/python_templates/transformations_template/unions.jinja +++ b/backend/backend/application/interpreter/python_templates/transformations_template/unions.jinja @@ -13,4 +13,4 @@ except IbisTypeError as error: raise TransformationFailed(transformation_name="merge", model_name=MODEL_NAME, error_message=str(error)) from error except Exception as error: raise TransformationFailed(transformation_name="merge", model_name=MODEL_NAME, error_message=str(error)) from error -self.save_table_columns(transformation_id="{{ transformation_id }}_transformed", table_obj=source_table) \ No newline at end of file +self.save_table_columns(transformation_id="{{ transformation_id }}_transformed", table_obj=source_table) diff --git a/backend/backend/application/interpreter/transformations/groups_and_aggregation.py b/backend/backend/application/interpreter/transformations/groups_and_aggregation.py index 2b7f030..5a09e67 100644 --- a/backend/backend/application/interpreter/transformations/groups_and_aggregation.py +++ b/backend/backend/application/interpreter/transformations/groups_and_aggregation.py @@ -753,4 +753,4 @@ def construct_code(self) -> str: return self._transformed_code def transform(self) -> str: - return self.construct_code() \ No newline at end of file + return self.construct_code() diff --git a/backend/backend/application/interpreter/transformations/joins.py b/backend/backend/application/interpreter/transformations/joins.py index 66ee992..1649938 100644 --- a/backend/backend/application/interpreter/transformations/joins.py +++ b/backend/backend/application/interpreter/transformations/joins.py @@ -110,7 +110,7 @@ def parse_joins(self, join_list: list[JoinParser]): for join_parser in join_list: # Parse the left join join_successful, class_name = self.__parse_left_joins(join_parser=join_parser) - + # If join parsing failed, store parent class if not join_successful: self.parent_classes.append(class_name) diff --git a/backend/backend/application/interpreter/transformations/pivot.py b/backend/backend/application/interpreter/transformations/pivot.py index ea8f925..a966228 100644 --- a/backend/backend/application/interpreter/transformations/pivot.py +++ b/backend/backend/application/interpreter/transformations/pivot.py @@ -19,7 +19,7 @@ def get_values_fill(self) -> str: return f"values_fill={int(fill_null)})" except (ValueError, TypeError): """ - suppress the exception since the column type doesn't support string types + suppress the exception since the column type doesn't support string types """ pass else: diff --git a/backend/backend/application/model_validator/transformations/synthesis_validator.py b/backend/backend/application/model_validator/transformations/synthesis_validator.py index def79ca..b4d428b 100644 --- a/backend/backend/application/model_validator/transformations/synthesis_validator.py +++ b/backend/backend/application/model_validator/transformations/synthesis_validator.py @@ -19,4 +19,4 @@ def check_column_usage(self, columns: list[str]) -> list[str]: for col in columns: if col == expr or col in expr: still_used.add(col) - return list(still_used) \ No newline at end of file + return list(still_used) diff --git a/backend/backend/application/session/connection_session.py b/backend/backend/application/session/connection_session.py index 1d1edf4..991a839 100644 --- a/backend/backend/application/session/connection_session.py +++ b/backend/backend/application/session/connection_session.py @@ -71,7 +71,7 @@ def get_all_connections(page: int, limit: int, filter_condition: dict[str, Any]) "is_connection_valid": con_model.is_connection_valid, "connection_flag": con_model.connection_flag, "is_sample_project": is_sample_project, - # "connection_details": con_model.connection_details, # skipping connection_details + # "connection_details": con_model.connection_details, # skipping connection_details } ) diff --git a/backend/backend/application/session/session.py b/backend/backend/application/session/session.py index 6024a5e..20c4a4d 100644 --- a/backend/backend/application/session/session.py +++ b/backend/backend/application/session/session.py @@ -58,10 +58,10 @@ def update_project_details(self, project_details: dict[str, Any]): def update_project_connection(self, connection_details: dict[str, Any]) -> dict[str, Any]: # TODO - Need to remove the project_connection update from project level - + # Decrypt sensitive fields from frontend encrypted data decrypted_connection_details = decrypt_sensitive_fields(connection_details) - + connection_model = self.project_instance.connection_model connection_model.connection_details = decrypted_connection_details connection_model.save() diff --git a/backend/backend/application/utils.py b/backend/backend/application/utils.py index 148fe88..0c35b51 100644 --- a/backend/backend/application/utils.py +++ b/backend/backend/application/utils.py @@ -256,4 +256,4 @@ def replace_in(match): parts = [p.strip() for p in match.group(1).split(",")] col, *values = parts conditions = [f"{col} = {v}" for v in values] - return f"OR({', '.join(conditions)})" \ No newline at end of file + return f"OR({', '.join(conditions)})" diff --git a/backend/backend/application/visitran_backend_context.py b/backend/backend/application/visitran_backend_context.py index 762788c..09e2608 100644 --- a/backend/backend/application/visitran_backend_context.py +++ b/backend/backend/application/visitran_backend_context.py @@ -414,10 +414,10 @@ def test_connection_data(self, connection_data: dict[str, Any], db_type: str) -> db_type = db_type or self.database_type if not connection_data: connection_data = {"file_path": f"{self.project_path}{os.path.sep}models/local.db"} - + # Decrypt sensitive fields from frontend encrypted data decrypted_connection_data = decrypt_sensitive_fields(connection_data) - + connection_cls: type[BaseConnection] = get_adapter_connection_cls(db_type) old_connection = self._conn_details self._conn_details = decrypted_connection_data diff --git a/backend/backend/core/constants/reserved_names.py b/backend/backend/core/constants/reserved_names.py index 6e6f0a9..236437d 100644 --- a/backend/backend/core/constants/reserved_names.py +++ b/backend/backend/core/constants/reserved_names.py @@ -5,11 +5,11 @@ class ProjectNameConstants(BaseConstant): """Constants for project name validation. - + Attributes: RESERVED_NAMES (set): Set of reserved project names that cannot be used. """ - + RESERVED_NAMES = { 'test', 'visitran', @@ -25,14 +25,14 @@ class ProjectNameConstants(BaseConstant): 'celery', 'time' } - + @classmethod def is_reserved_name(cls, name: str) -> bool: """Check if a name is reserved. - + Args: name: The name to check - + Returns: bool: True if the name is reserved, False otherwise """ diff --git a/backend/backend/core/models/ai_context_rules.py b/backend/backend/core/models/ai_context_rules.py index 54a7153..24a796e 100644 --- a/backend/backend/core/models/ai_context_rules.py +++ b/backend/backend/core/models/ai_context_rules.py @@ -7,19 +7,19 @@ class UserAIContextRules(models.Model): """Model for storing user's personal AI context rules""" user = models.OneToOneField( - User, + User, on_delete=models.CASCADE, related_name='ai_context_rules' ) context_rules = models.TextField(default='', blank=True) created_at = models.DateTimeField(auto_now_add=True) updated_at = models.DateTimeField(auto_now=True) - + class Meta: db_table = 'core_user_ai_context_rules' verbose_name = 'User AI Context Rules' verbose_name_plural = 'User AI Context Rules' - + def __str__(self): return f"AI Context Rules for {self.user.username}" @@ -43,11 +43,11 @@ class ProjectAIContextRules(models.Model): ) created_at = models.DateTimeField(auto_now_add=True) updated_at = models.DateTimeField(auto_now=True) - + class Meta: db_table = 'core_project_ai_context_rules' verbose_name = 'Project AI Context Rules' verbose_name_plural = 'Project AI Context Rules' - + def __str__(self): return f"AI Context Rules for {self.project.project_name}" diff --git a/backend/backend/core/models/chat_intent.py b/backend/backend/core/models/chat_intent.py index 095eee8..4f0d582 100644 --- a/backend/backend/core/models/chat_intent.py +++ b/backend/backend/core/models/chat_intent.py @@ -45,7 +45,7 @@ class ChatIntent(BaseModel): unique=True, help_text="User-facing display name for the intent." ) - + objects = models.Manager() def __str__(self) -> str: diff --git a/backend/backend/core/models/chat_message.py b/backend/backend/core/models/chat_message.py index 771a808..e4db10e 100644 --- a/backend/backend/core/models/chat_message.py +++ b/backend/backend/core/models/chat_message.py @@ -183,32 +183,32 @@ class ChatMessage(BaseModel): editable=True, help_text="String identifier of the developer LLM model used for this chat." ) - + # Feedback fields for response quality has_feedback = models.BooleanField( default=False, help_text="Indicates whether this message has received user feedback." ) - + FEEDBACK_CHOICES = [ ('0', 'Neutral'), ('P', 'Positive'), ('N', 'Negative') ] - + feedback = models.CharField( max_length=1, choices=FEEDBACK_CHOICES, default='0', help_text="Feedback value: 0=Neutral, P=Positive, N=Negative" ) - + feedback_timestamp = models.DateTimeField( null=True, blank=True, help_text="When the feedback was provided." ) - + feedback_comment = models.TextField( null=True, blank=True, diff --git a/backend/backend/core/models/environment_models.py b/backend/backend/core/models/environment_models.py index 5edba79..edf52b6 100644 --- a/backend/backend/core/models/environment_models.py +++ b/backend/backend/core/models/environment_models.py @@ -40,11 +40,11 @@ def decrypted_connection_data(self) -> dict: try: # First try the old Fernet decryption system decrypted_data = decrypt_connection_details(self.env_connection_data) - + # If Fernet decryption succeeds, return the data # (Don't try RSA decryption on already decrypted data) return decrypted_data - + except Exception as e: # If Fernet decryption fails, try RSA decryption try: diff --git a/backend/backend/core/models/onboarding.py b/backend/backend/core/models/onboarding.py index 992f766..ec85bf5 100644 --- a/backend/backend/core/models/onboarding.py +++ b/backend/backend/core/models/onboarding.py @@ -40,31 +40,31 @@ class ProjectOnboardingSessionManager(DefaultOrganizationManagerMixin, models.Ma class ProjectOnboardingSession(DefaultOrganizationMixin, BaseModel): """Active onboarding session for a project""" project = models.ForeignKey( - ProjectDetails, + ProjectDetails, on_delete=models.CASCADE, related_name="onboarding_sessions" ) user = models.ForeignKey( - User, + User, on_delete=models.CASCADE, related_name="onboarding_sessions" ) template = models.ForeignKey( - OnboardingTemplate, + OnboardingTemplate, on_delete=models.CASCADE, related_name="sessions" ) - + # Progress tracking (no more sequential ordering) completed_tasks = models.JSONField(default=list) # List of task IDs skipped_tasks = models.JSONField(default=list) # List of task IDs - + # Session state is_active = models.BooleanField(default=True) is_completed = models.BooleanField(default=False) started_at = models.DateTimeField(auto_now_add=True) completed_at = models.DateTimeField(null=True, blank=True) - + # Manager objects = ProjectOnboardingSessionManager() @@ -83,15 +83,15 @@ def progress_percentage(self) -> float: items = self.template.template_data.get('items', []) except: return 0.0 - + total_tasks = len(items) if total_tasks == 0: return 0.0 - + completed_count = len(self.completed_tasks) skipped_count = len(self.skipped_tasks) total_progress = completed_count + skipped_count - + return round((total_progress / total_tasks) * 100, 2) def reset_session(self): diff --git a/backend/backend/core/routers/ai_context/urls.py b/backend/backend/core/routers/ai_context/urls.py index b255926..13b1c26 100644 --- a/backend/backend/core/routers/ai_context/urls.py +++ b/backend/backend/core/routers/ai_context/urls.py @@ -4,7 +4,7 @@ urlpatterns = [ # Personal AI Context Rules path('user/ai-context-rules/', views.user_ai_context_rules, name='user-ai-context-rules'), - + # Project AI Context Rules path('project//ai-context-rules/', views.project_ai_context_rules, name='project-ai-context-rules'), ] diff --git a/backend/backend/core/routers/ai_context/views.py b/backend/backend/core/routers/ai_context/views.py index 464c74f..5dbd589 100644 --- a/backend/backend/core/routers/ai_context/views.py +++ b/backend/backend/core/routers/ai_context/views.py @@ -22,14 +22,14 @@ def user_ai_context_rules(request: Request) -> Response: """Get or update user's personal AI context rules""" try: user = request.user - + if request.method == HTTPMethods.GET: # Get or create user context rules context_rules, created = UserAIContextRules.objects.get_or_create( user=user, defaults={'context_rules': ''} ) - + return Response({ "success": True, "data": { @@ -39,20 +39,20 @@ def user_ai_context_rules(request: Request) -> Response: "updated_at": context_rules.updated_at.isoformat() } }, status=status.HTTP_200_OK) - + elif request.method == HTTPMethods.PUT: context_rules_text = request.data.get('context_rules', '') - + # Get or create user context rules context_rules, created = UserAIContextRules.objects.get_or_create( user=user, defaults={'context_rules': context_rules_text} ) - + if not created: context_rules.context_rules = context_rules_text context_rules.save() - + return Response({ "success": True, "message": BackendSuccessMessages.AI_CONTEXT_RULES_PERSONAL_UPDATED, @@ -62,7 +62,7 @@ def user_ai_context_rules(request: Request) -> Response: "updated_at": context_rules.updated_at.isoformat() } }, status=status.HTTP_200_OK) - + except Exception as e: logger.error(f"Error with user AI context rules: {str(e)}") return Response({ @@ -86,12 +86,12 @@ def project_ai_context_rules(request: Request, project_id: str) -> Response: "is_markdown": True, "severity": "error" }, status=status.HTTP_404_NOT_FOUND) - + if request.method == HTTPMethods.GET: # Get project context rules (single entry per project) try: context_rules = ProjectAIContextRules.objects.get(project=project) - + return Response({ "success": True, "data": { @@ -112,7 +112,7 @@ def project_ai_context_rules(request: Request, project_id: str) -> Response: "updated_at": context_rules.updated_at.isoformat() } }, status=status.HTTP_200_OK) - + except ProjectAIContextRules.DoesNotExist: # Return empty context rules if none exist yet return Response({ @@ -127,11 +127,11 @@ def project_ai_context_rules(request: Request, project_id: str) -> Response: "updated_at": None } }, status=status.HTTP_200_OK) - + elif request.method == HTTPMethods.PUT: user = request.user context_rules_text = request.data.get('context_rules', '') - + # Get or create project context rules (single entry per project) context_rules, created = ProjectAIContextRules.objects.get_or_create( project=project, @@ -141,12 +141,12 @@ def project_ai_context_rules(request: Request, project_id: str) -> Response: 'updated_by': user } ) - + if not created: context_rules.context_rules = context_rules_text context_rules.updated_by = user # Track who updated context_rules.save() - + return Response({ "success": True, "message": BackendSuccessMessages.AI_CONTEXT_RULES_PROJECT_UPDATED, @@ -161,7 +161,7 @@ def project_ai_context_rules(request: Request, project_id: str) -> Response: "updated_at": context_rules.updated_at.isoformat() } }, status=status.HTTP_200_OK) - + except Exception as e: logger.error(f"Error with project AI context rules: {str(e)}") return Response({ diff --git a/backend/backend/core/routers/chat_intent/serializers.py b/backend/backend/core/routers/chat_intent/serializers.py index 1e3aedd..f5beb3c 100644 --- a/backend/backend/core/routers/chat_intent/serializers.py +++ b/backend/backend/core/routers/chat_intent/serializers.py @@ -1,6 +1,6 @@ from rest_framework import serializers from backend.core.models.chat_intent import ChatIntent - + class ChatIntentSerializer(serializers.ModelSerializer): class Meta: model = ChatIntent diff --git a/backend/backend/core/routers/chat_message/constants.py b/backend/backend/core/routers/chat_message/constants.py index e03de75..13529e0 100644 --- a/backend/backend/core/routers/chat_message/constants.py +++ b/backend/backend/core/routers/chat_message/constants.py @@ -14,4 +14,4 @@ class ChatMessageStatus: MODEL_GENERATION_FAILED = "MODEL_CREATE_FAILED" MODEL_UPDATED = "MODEL_UPDATED" MODEL_UPDATE_FAILED = "MODEL_UPDATE_FAILED" - TRANSFORM_RETRY = "TRANSFORM_RETRY" \ No newline at end of file + TRANSFORM_RETRY = "TRANSFORM_RETRY" diff --git a/backend/backend/core/routers/chat_message/serializers/feedback_serializer.py b/backend/backend/core/routers/chat_message/serializers/feedback_serializer.py index 31eed4a..e8fafe7 100644 --- a/backend/backend/core/routers/chat_message/serializers/feedback_serializer.py +++ b/backend/backend/core/routers/chat_message/serializers/feedback_serializer.py @@ -16,16 +16,16 @@ def validate(self, attrs): Validates the feedback value. """ feedback_value = attrs.get('feedback', None) - + if not feedback_value: raise serializers.ValidationError( {"feedback": "This field is required for providing feedback."} ) - + # Validate the value matches our choices if feedback_value not in ['0', 'P', 'N']: raise serializers.ValidationError( {"feedback": "Must be one of '0' (neutral), 'P' (positive), or 'N' (negative)."} ) - + return attrs diff --git a/backend/backend/core/routers/chat_message/views/feedback_views.py b/backend/backend/core/routers/chat_message/views/feedback_views.py index aebb39a..d3cc7d6 100644 --- a/backend/backend/core/routers/chat_message/views/feedback_views.py +++ b/backend/backend/core/routers/chat_message/views/feedback_views.py @@ -21,7 +21,7 @@ class ChatMessageFeedbackView(APIView): def post(self, request, chat_message_id, project_id=None, chat_id=None, **kwargs): """ Submit feedback for a specific chat message. - + Args: request: The HTTP request org_id: Organization ID @@ -35,18 +35,18 @@ def post(self, request, chat_message_id, project_id=None, chat_id=None, **kwargs {"error": BackendErrorMessages.ORGANIZATION_REQUIRED}, status=status.HTTP_400_BAD_REQUEST ) - + # Find the chat message chat_message = ChatMessage.objects.filter( chat_message_id=chat_message_id ).first() - + if not chat_message: return Response( {"error": BackendErrorMessages.CHAT_MESSAGE_NOT_FOUND}, status=status.HTTP_404_NOT_FOUND ) - + # Validate and save feedback serializer = ChatMessageFeedbackSerializer(data=request.data) if serializer.is_valid(): @@ -61,23 +61,23 @@ def post(self, request, chat_message_id, project_id=None, chat_id=None, **kwargs 'feedback_comment', 'feedback_timestamp' ] ) - + logging.info( f"Feedback submitted for chat message {chat_message_id}: " f"feedback={chat_message.feedback}" ) - + return Response( {"success": True, "message": "Feedback submitted successfully"}, status=status.HTTP_200_OK ) - + # Use INVALID_FEEDBACK_FORMAT for serializer validation errors return Response( {"error": BackendErrorMessages.INVALID_FEEDBACK_FORMAT}, status=status.HTTP_400_BAD_REQUEST ) - + except Exception as e: logging.exception(f"Error submitting feedback for chat message {chat_message_id}") error_message = BackendErrorMessages.FEEDBACK_SUBMISSION_FAILED.format( @@ -87,11 +87,11 @@ def post(self, request, chat_message_id, project_id=None, chat_id=None, **kwargs {"error": error_message}, status=status.HTTP_500_INTERNAL_SERVER_ERROR ) - + def get(self, request, chat_message_id, project_id=None, chat_id=None, **kwargs): """ Retrieve feedback status for a specific chat message. - + Args: request: The HTTP request chat_message_id: UUID of the chat message to retrieve feedback for @@ -104,23 +104,23 @@ def get(self, request, chat_message_id, project_id=None, chat_id=None, **kwargs) {"error": BackendErrorMessages.ORGANIZATION_REQUIRED}, status=status.HTTP_400_BAD_REQUEST ) - + # Find the chat message - don't filter by organization_id which is causing the error chat_message = ChatMessage.objects.filter( chat_message_id=chat_message_id ).first() - + if not chat_message: return Response( {"error": BackendErrorMessages.CHAT_MESSAGE_NOT_FOUND}, status=status.HTTP_404_NOT_FOUND ) - + # Return feedback status response_data = { 'has_feedback': chat_message.has_feedback, } - + # Only include feedback details if feedback exists if chat_message.has_feedback: response_data.update({ @@ -128,9 +128,9 @@ def get(self, request, chat_message_id, project_id=None, chat_id=None, **kwargs) 'feedback_comment': chat_message.feedback_comment or '', 'feedback_timestamp': chat_message.feedback_timestamp }) - + return Response(response_data, status=status.HTTP_200_OK) - + except Exception as e: logging.exception(f"Error retrieving feedback for chat message {chat_message_id}") error_message = BackendErrorMessages.FEEDBACK_RETRIEVAL_FAILED.format( diff --git a/backend/backend/core/routers/environment/views.py b/backend/backend/core/routers/environment/views.py index fea0ff4..f302c54 100644 --- a/backend/backend/core/routers/environment/views.py +++ b/backend/backend/core/routers/environment/views.py @@ -128,7 +128,7 @@ def test_environment(request: Request): request_data: dict[str, Any] = request.data datasource: str = request_data.get("datasource") connection_data: dict[str, Any] = request_data.get("connection_details") - + # Decrypt sensitive fields from frontend encrypted data from backend.utils.decryption_utils import decrypt_sensitive_fields if connection_data: @@ -136,5 +136,5 @@ def test_environment(request: Request): test_connection_data(datasource=datasource, connection_data=decrypted_connection_data) else: test_connection_data(datasource=datasource, connection_data=connection_data) - + return Response(data={"status": "success"}, status=status.HTTP_200_OK) diff --git a/backend/backend/core/routers/execute/views.py b/backend/backend/core/routers/execute/views.py index 25fe3a7..742d277 100644 --- a/backend/backend/core/routers/execute/views.py +++ b/backend/backend/core/routers/execute/views.py @@ -138,4 +138,4 @@ def execute_sql_command(request: Request, project_id: str) -> Response: try: app.execute_sql_command(sql_command=drop_sql) except Exception as drop_err: - logger.warning(f"Failed to drop table {table}: {drop_err}") \ No newline at end of file + logger.warning(f"Failed to drop table {table}: {drop_err}") diff --git a/backend/backend/core/routers/onboarding/urls.py b/backend/backend/core/routers/onboarding/urls.py index 661b44b..c4b5191 100644 --- a/backend/backend/core/routers/onboarding/urls.py +++ b/backend/backend/core/routers/onboarding/urls.py @@ -52,7 +52,7 @@ urlpatterns = [ # Template management (no org required) path('templates//', onboarding_template, name='get_onboarding_template'), - + # Project-level onboarding management (org handled by middleware) path('status/', onboarding_status, name='get_project_onboarding_status'), path('start/', onboarding_start, name='start_onboarding'), @@ -62,4 +62,4 @@ path('reset/', onboarding_reset, name='reset_onboarding'), path('toggle/', onboarding_toggle, name='toggle_project_onboarding'), # End of urlpatterns -] \ No newline at end of file +] diff --git a/backend/backend/core/routers/onboarding/views.py b/backend/backend/core/routers/onboarding/views.py index 7b9576c..44bfb13 100644 --- a/backend/backend/core/routers/onboarding/views.py +++ b/backend/backend/core/routers/onboarding/views.py @@ -26,7 +26,7 @@ def get_onboarding_template(self, request: Request, template_id: str) -> Respons """Get onboarding template by ID - Global templates""" try: template = OnboardingTemplate.objects.get( - template_id=template_id, + template_id=template_id, is_active=True ) return Response({ @@ -90,7 +90,7 @@ def get_project_onboarding_status(self, request: Request, project_id: str) -> Re completed_count = len(onboarding_session.completed_tasks) skipped_count = len(onboarding_session.skipped_tasks) progress_percentage = int((completed_count + skipped_count) / total_tasks * 100) if total_tasks > 0 else 0 - + # Check if onboarding is completed (only check session status, not progress) is_completed = onboarding_session.is_completed @@ -196,7 +196,7 @@ def complete_task(self, request: Request, project_id: str) -> Response: # Get project and user project = ProjectDetails.objects.get(project_uuid=project_id) - + try: user = self._get_user_from_context() except ValueError as e: @@ -233,7 +233,7 @@ def complete_task(self, request: Request, project_id: str) -> Response: completed_count = len(onboarding_session.completed_tasks) skipped_count = len(onboarding_session.skipped_tasks) progress_percentage = int((completed_count + skipped_count) / total_tasks * 100) if total_tasks > 0 else 0 - + # Don't auto-complete onboarding when progress reaches 100% # Use separate API endpoint to mark as complete is_completed = onboarding_session.is_completed @@ -276,7 +276,7 @@ def skip_task(self, request: Request, project_id: str) -> Response: # Get project and user project = ProjectDetails.objects.get(project_uuid=project_id) - + try: user = self._get_user_from_context() except ValueError as e: @@ -313,7 +313,7 @@ def skip_task(self, request: Request, project_id: str) -> Response: completed_count = len(onboarding_session.completed_tasks) skipped_count = len(onboarding_session.skipped_tasks) progress_percentage = int((completed_count + skipped_count) / total_tasks * 100) if total_tasks > 0 else 0 - + # Don't auto-complete onboarding when progress reaches 100% # Use separate API endpoint to mark as complete is_completed = onboarding_session.is_completed @@ -441,7 +441,7 @@ def _get_template_for_project(self, project: ProjectDetails) -> OnboardingTempla template_id = "jaffleshop_starter" else: template_id = project.project_type - + try: return OnboardingTemplate.objects.get(template_id=template_id) except OnboardingTemplate.DoesNotExist: @@ -451,12 +451,12 @@ def _get_template_for_project(self, project: ProjectDetails) -> OnboardingTempla def _build_tasks_with_status(self, template: OnboardingTemplate, session: ProjectOnboardingSession) -> List[Dict]: """Build tasks list with individual status for each task""" tasks = [] - + for task in template.template_data.get('items', []): task_id = task.get("id") if not task_id: continue - + # Determine task status if task_id in session.completed_tasks: status = "completed" @@ -464,7 +464,7 @@ def _build_tasks_with_status(self, template: OnboardingTemplate, session: Projec status = "skipped" else: status = "pending" - + tasks.append({ "id": task_id, "title": task.get("title", ""), @@ -473,7 +473,7 @@ def _build_tasks_with_status(self, template: OnboardingTemplate, session: Projec "mode": task.get("mode", ""), "status": status }) - + return tasks @action(detail=False, methods=["POST"]) @@ -529,9 +529,9 @@ def _get_user_from_context(self) -> User: current_user = get_current_user() if not current_user or not current_user.get("username"): raise ValueError("User not found in context") - + username = current_user.get("username") try: return User.objects.get(email=username) except User.DoesNotExist: - raise ValueError(f"User with email {username} not found") \ No newline at end of file + raise ValueError(f"User with email {username} not found") diff --git a/backend/backend/core/routers/projects/views.py b/backend/backend/core/routers/projects/views.py index 0799bb2..9149221 100644 --- a/backend/backend/core/routers/projects/views.py +++ b/backend/backend/core/routers/projects/views.py @@ -38,39 +38,39 @@ def _create_starter_projects_if_needed(): """Create starter projects for new users if they haven't been created yet.""" logging.info("Starting starter projects creation check") - + try: # Get current user from pluggable_apps.tenant_account.organization_member_service import OrganizationMemberService current_user = get_current_user() - + if not current_user: logging.warning("No current user found in context - skipping starter projects creation") return - + if not current_user.get("username"): logging.warning("Current user has no username - skipping starter projects creation") return - + username = current_user.get("username") logging.info(f"Checking starter projects for user: {username}") - + # Check if starter projects have been created using service with cache if OrganizationMemberService.is_starter_projects_created(username): logging.info(f"Starter projects already created for user {username} - skipping creation") return - + logging.info(f"Starter projects not created for user {username} - proceeding with creation") - + # Create starter projects _create_starter_projects() - + # Mark as created using service (updates both DB and cache) OrganizationMemberService.mark_starter_projects_created(username) - + logging.info(f"Successfully created and marked starter projects for user {username}") - + except Exception as e: logging.error(f"Error creating starter projects: {str(e)}", exc_info=True) # Don't raise exception to avoid breaking the project list API @@ -79,46 +79,46 @@ def _create_starter_projects_if_needed(): def _create_starter_projects(): """Create starter projects using mapper.""" logging.info("Starting creation of starter projects") - + # Mapper for starter projects only starter_project_mapper = { "dvd_starter": DvdRentalProjectStarter, "jaffleshop_starter": JaffleShopProjectStarter, } - + logging.info(f"Will create {len(starter_project_mapper)} starter projects: {list(starter_project_mapper.keys())}") - + for project_key, project_class in starter_project_mapper.items(): logging.info(f"Creating {project_key} starter project") - + try: project_loader = project_class() sample_project_data = project_loader.load_sample_project() - + # Enable onboarding and set project type for this project from backend.core.models.project_details import ProjectDetails from backend.application.utils import get_filter - + project_id = sample_project_data.get("project_id") if not project_id: logging.warning(f"No project_id returned for {project_key} - skipping project configuration") continue - + logging.info(f"Configuring project {project_key} with ID: {project_id}") - + filter_condition = get_filter() filter_condition["project_uuid"] = project_id - + project = ProjectDetails.objects.get(**filter_condition) project.onboarding_enabled = True project.project_type = project_key project.save() - + logging.info(f"Successfully created and configured {project_key} starter project with ID: {project_id}") - + except Exception as e: logging.error(f"Error creating {project_key} starter project: {str(e)}", exc_info=True) - + logging.info("Completed starter projects creation process") @@ -189,7 +189,7 @@ def create_sample_project(request) -> Response: # create connection with postgres for project sample_project_data = sample_project.load_sample_project() - + # Set project_type for all sample projects; enable onboarding for starters from backend.core.models.project_details import ProjectDetails from backend.application.utils import get_filter diff --git a/backend/backend/core/routers/security/urls.py b/backend/backend/core/routers/security/urls.py index 476fced..4073f2d 100644 --- a/backend/backend/core/routers/security/urls.py +++ b/backend/backend/core/routers/security/urls.py @@ -4,4 +4,4 @@ urlpatterns = [ path("/public-key", get_public_key, name="get-public-key"), -] \ No newline at end of file +] diff --git a/backend/backend/core/routers/security/views.py b/backend/backend/core/routers/security/views.py index 220c2fe..9f93440 100644 --- a/backend/backend/core/routers/security/views.py +++ b/backend/backend/core/routers/security/views.py @@ -30,7 +30,7 @@ def get_public_key(request): {"status": "error", "message": "RSA public key not available"}, status=503 ) - + # Return public key in PEM format response_data = { "status": "success", @@ -43,11 +43,11 @@ def get_public_key(request): "algorithm": "RSA" } } - + return JsonResponse(data=response_data, status=200) - + except Exception as e: return JsonResponse( {"status": "error", "message": f"Error serving public key: {str(e)}"}, status=500 - ) \ No newline at end of file + ) diff --git a/backend/backend/core/utils.py b/backend/backend/core/utils.py index 1a267e1..90f60a9 100644 --- a/backend/backend/core/utils.py +++ b/backend/backend/core/utils.py @@ -131,7 +131,7 @@ def wrapper(*args, **kwargs): if not chat_message_id: return func(*args, **kwargs) - redis = RedisClient().redis_client + redis = RedisClient().redis_client key = f"transformation:{chat_message_id}:lock" diff --git a/backend/backend/core/views.py b/backend/backend/core/views.py index a2d5a12..888a71e 100644 --- a/backend/backend/core/views.py +++ b/backend/backend/core/views.py @@ -92,11 +92,11 @@ def get_user_profile(request: Request) -> Response: def get_datasource_list(request: Request) -> Response: """This method will return the list of adapters installed.""" adapters_list: list[str] = get_adapters_list() - + # Soft delete: Remove Trino from the list if "trino" in adapters_list: adapters_list.remove("trino") - + data = [] for adapter_name in adapters_list: icon = import_file(f"visitran.adapters.{adapter_name}").ICON diff --git a/backend/backend/errors/config_exceptions.py b/backend/backend/errors/config_exceptions.py index 6459f88..61844cc 100644 --- a/backend/backend/errors/config_exceptions.py +++ b/backend/backend/errors/config_exceptions.py @@ -83,4 +83,4 @@ def __init__(self, failure_reason: str): error_code=BackendErrorMessages.INVALID_MODEL_REFERENCE_DATA, http_status_code=status.HTTP_400_BAD_REQUEST, failure_reason=failure_reason, - ) \ No newline at end of file + ) diff --git a/backend/backend/errors/error_codes.py b/backend/backend/errors/error_codes.py index 253f248..a5075ca 100644 --- a/backend/backend/errors/error_codes.py +++ b/backend/backend/errors/error_codes.py @@ -31,17 +31,17 @@ class BackendSuccessMessages(BaseConstant): "**Personal Rules Updated!**\n" "Your personal AI context rules have been saved successfully and will apply to all future conversations." ) - + AI_CONTEXT_RULES_PROJECT_UPDATED = ( "**Project Rules Updated!**\n" "Project AI context rules have been saved successfully and are now shared with all team members." ) - + AI_CONTEXT_RULES_PERSONAL_RETRIEVED = ( "**Personal Rules Retrieved!**\n" "Your personal AI context rules have been loaded successfully." ) - + AI_CONTEXT_RULES_PROJECT_RETRIEVED = ( "**Project Rules Retrieved!**\n" "Project AI context rules have been loaded successfully." @@ -267,17 +267,17 @@ class BackendErrorMessages(BaseConstant): FEEDBACK_SUBMISSION_FAILED = ( "**Feedback Error!**\nCouldn't save feedback for message ID \"{chat_message_id}\". Please try again." ) - + ORGANIZATION_REQUIRED = "**Organization Required!**\nOrganization ID is required for this operation." - + INVALID_FEEDBACK_FORMAT = ( "**Invalid Feedback!**\nFeedback format is invalid. Use 'P' for positive, 'N' for negative, or '0' for neutral." ) - + FEEDBACK_RETRIEVAL_FAILED = ( "**Feedback Retrieval Failed!**\nUnable to retrieve feedback for message ID \"{chat_message_id}\". Please try again." ) - + INVALID_CHAT_MESSAGE_STATUS = ( '**Invalid Status!**\nStatus "{invalid_status}" is invalid. Valid statuses: {valid_status}.' ) @@ -291,7 +291,7 @@ class BackendErrorMessages(BaseConstant): ) SQL_EXTRACTION_FAILED = "**SQL Query Extraction Failed**\nUnable to extract SQL query from the provided text." - + LLM_SERVER_FAILURE = ( "**AI Server Error!**\n" "Failed while answering your prompt \n " @@ -326,23 +326,23 @@ class BackendErrorMessages(BaseConstant): "**Context Rules Fetch Failed!**\n" "Unable to retrieve AI context rules. Please try again or contact support if the issue persists." ) - + AI_CONTEXT_RULES_UPDATE_FAILED = ( "**Context Rules Update Failed!**\n" "Failed to save AI context rules. Please verify your input and try again." ) - + AI_CONTEXT_RULES_INVALID_PROJECT = ( '**Invalid Project!**\nProject with ID "{project_id}" not found or you don\'t have access to it. ' "Verify the project ID and your permissions." ) - + AI_CONTEXT_RULES_PERMISSION_DENIED = ( "**Permission Denied!**\n" "You don't have permission to modify AI context rules for this project. " "Contact your project administrator for access." ) - + AI_CONTEXT_RULES_INVALID_INPUT = ( "**Invalid Input!**\n" "The context rules format is invalid. Please check your input and try again." diff --git a/backend/backend/utils/decryption_utils.py b/backend/backend/utils/decryption_utils.py index b029e8e..9e40e2c 100644 --- a/backend/backend/utils/decryption_utils.py +++ b/backend/backend/utils/decryption_utils.py @@ -48,10 +48,10 @@ def decrypt_chunked_value(encrypted_value: str) -> str: """ Decrypt a value that was encrypted using chunked encryption. - + Args: encrypted_value: The encrypted value (may be chunked) - + Returns: Decrypted value """ @@ -60,7 +60,7 @@ def decrypt_chunked_value(encrypted_value: str) -> str: if '|' in encrypted_value: # Split into chunks chunks = encrypted_value.split('|') - + # Decrypt each chunk decrypted_chunks = [] for i, chunk in enumerate(chunks): @@ -69,7 +69,7 @@ def decrypt_chunked_value(encrypted_value: str) -> str: logging.error(f"Failed to decrypt chunk {i + 1}") return encrypted_value # Return original on error decrypted_chunks.append(decrypted_chunk) - + # Combine chunks result = ''.join(decrypted_chunks) return result @@ -84,29 +84,29 @@ def decrypt_chunked_value(encrypted_value: str) -> str: def decrypt_bigquery_credentials(credentials_json: str) -> str: """ Decrypt BigQuery credentials specifically. - + Args: credentials_json: The BigQuery credentials JSON string - + Returns: Decrypted credentials JSON string """ try: # Parse the credentials JSON credentials = json.loads(credentials_json) - + # Decrypt sensitive fields within the credentials decrypted_credentials = credentials.copy() - + # List of sensitive fields in BigQuery service account JSON bigquery_sensitive_fields = [ "private_key", "client_email", - "client_id", + "client_id", "private_key_id", "project_id" ] - + for field in bigquery_sensitive_fields: if field in decrypted_credentials and isinstance(decrypted_credentials[field], str): try: @@ -118,7 +118,7 @@ def decrypt_bigquery_credentials(credentials_json: str) -> str: logging.warning(f"Failed to decrypt BigQuery field '{field}': {e}") # Keep original value on error pass - + # Return the decrypted credentials as a JSON string return json.dumps(decrypted_credentials) except Exception as e: @@ -129,16 +129,16 @@ def decrypt_bigquery_credentials(credentials_json: str) -> str: def decrypt_sensitive_fields(data: Union[Dict[str, Any], list, Any]) -> Union[Dict[str, Any], list, Any]: """ Recursively decrypt sensitive fields in data structures. - + Args: data: The data to decrypt (dict, list, or primitive value) - + Returns: The data with sensitive fields decrypted """ if data is None: return data - + if isinstance(data, dict): return _decrypt_dict(data) elif isinstance(data, list): @@ -153,7 +153,7 @@ def _decrypt_dict(data: Dict[str, Any]) -> Dict[str, Any]: return data decrypted_data = data.copy() - + for key, value in data.items(): if isinstance(value, dict): # Recursively decrypt nested dictionaries @@ -215,10 +215,10 @@ def _decrypt_list(data: list) -> list: def decrypt_connection_data(connection_data: Dict[str, Any]) -> Dict[str, Any]: """ Decrypt sensitive fields in connection data. - + Args: connection_data: Connection data dictionary - + Returns: Connection data with sensitive fields decrypted """ @@ -228,10 +228,10 @@ def decrypt_connection_data(connection_data: Dict[str, Any]) -> Dict[str, Any]: def decrypt_request_data(request_data: Dict[str, Any]) -> Dict[str, Any]: """ Decrypt sensitive fields in request data. - + Args: request_data: Request data dictionary - + Returns: Request data with sensitive fields decrypted """ @@ -241,36 +241,36 @@ def decrypt_request_data(request_data: Dict[str, Any]) -> Dict[str, Any]: def is_encrypted_value(value: str) -> bool: """ Check if a value appears to be encrypted. - + Args: value: The value to check - + Returns: True if the value appears to be encrypted, False otherwise """ if not isinstance(value, str): return False - + # Check if it looks like base64 encoded encrypted data # Encrypted data is typically longer and contains base64 characters if len(value) > 100 and all(c in 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=' for c in value): return True - + return False def get_sensitive_fields_in_data(data: Union[Dict[str, Any], list]) -> list: """ Get list of sensitive fields found in the data. - + Args: data: The data to analyze - + Returns: List of sensitive field names found """ sensitive_fields = [] - + if isinstance(data, dict): for key, value in data.items(): if key.lower() in SENSITIVE_FIELDS: @@ -281,37 +281,37 @@ def get_sensitive_fields_in_data(data: Union[Dict[str, Any], list]) -> list: for item in data: if isinstance(item, (dict, list)): sensitive_fields.extend(get_sensitive_fields_in_data(item)) - + return list(set(sensitive_fields)) # Remove duplicates def decrypt_with_logging(data: Dict[str, Any], context: str = "unknown") -> Dict[str, Any]: """ Decrypt data with detailed logging for debugging. - + Args: data: The data to decrypt context: Context string for logging (e.g., "connection_creation", "test_connection") - + Returns: Decrypted data """ logging.info(f"Starting decryption for context: {context}") - + # Find sensitive fields before decryption sensitive_fields = get_sensitive_fields_in_data(data) if sensitive_fields: logging.info(f"Found sensitive fields in {context}: {sensitive_fields}") - + # Decrypt the data decrypted_data = decrypt_sensitive_fields(data) - + # Log decryption results for field in sensitive_fields: if field in data and field in decrypted_data: original_value = data[field] decrypted_value = decrypted_data[field] - + if is_encrypted_value(original_value): if original_value != decrypted_value: logging.info(f"Successfully decrypted field '{field}' in {context}") @@ -319,7 +319,7 @@ def decrypt_with_logging(data: Dict[str, Any], context: str = "unknown") -> Dict logging.warning(f"Failed to decrypt field '{field}' in {context}, using original value") else: logging.debug(f"Field '{field}' was not encrypted in {context}") - + logging.info(f"Completed decryption for context: {context}") return decrypted_data @@ -342,42 +342,42 @@ def decrypt_test_connection_data(connection_data: Dict[str, Any]) -> Dict[str, A def decrypt_environment_data(environment_data: Dict[str, Any]) -> Dict[str, Any]: """Decrypt data for environment creation/update.""" - return decrypt_with_logging(environment_data, "environment_management") + return decrypt_with_logging(environment_data, "environment_management") def decrypt_connection_details_safe(connection_details: Dict[str, Any]) -> Dict[str, Any]: """ Safely decrypt connection_details with detailed error reporting. - + Args: connection_details: Connection details dictionary - + Returns: Connection details with sensitive fields decrypted """ logging.info("Starting connection_details decryption...") - + if not connection_details: logging.warning("connection_details is empty or None") return connection_details - + logging.info(f"connection_details type: {type(connection_details)}") logging.info(f"connection_details keys: {list(connection_details.keys())}") - + try: # Find sensitive fields before decryption sensitive_fields = get_sensitive_fields_in_data(connection_details) logging.info(f"Found sensitive fields in connection_details: {sensitive_fields}") - + # Decrypt the data decrypted_data = decrypt_sensitive_fields(connection_details) - + # Log decryption results for field in sensitive_fields: if field in connection_details and field in decrypted_data: original_value = connection_details[field] decrypted_value = decrypted_data[field] - + if is_encrypted_value(original_value): if original_value != decrypted_value: logging.info(f"Successfully decrypted field '{field}' in connection_details") @@ -385,49 +385,49 @@ def decrypt_connection_details_safe(connection_details: Dict[str, Any]) -> Dict[ logging.warning(f"Failed to decrypt field '{field}' in connection_details, using original value") else: logging.debug(f"Field '{field}' was not encrypted in connection_details") - + logging.info("Completed connection_details decryption") return decrypted_data - + except Exception as e: logging.error(f"Error during connection_details decryption: {e}") logging.error(f"connection_details content: {connection_details}") import traceback logging.error(f"Traceback: {traceback.format_exc()}") # Return original data on error - return connection_details + return connection_details def decrypt_connection_details_robust(connection_details: Dict[str, Any]) -> Dict[str, Any]: """ Robustly decrypt connection_details with comprehensive error handling. - + This function handles various scenarios: - Fully encrypted sensitive fields - Partially encrypted sensitive fields - Non-encrypted sensitive fields (backward compatibility) - Malformed encrypted data - BigQuery credentials with nested sensitive fields - + Args: connection_details: Connection details dictionary - + Returns: Connection details with sensitive fields decrypted """ logging.info("Starting robust connection_details decryption...") - + if not connection_details: logging.warning("connection_details is empty or None") return connection_details - + logging.info(f"connection_details type: {type(connection_details)}") logging.info(f"connection_details keys: {list(connection_details.keys())}") - + try: # Create a copy to avoid modifying the original decrypted_data = connection_details.copy() - + # Special handling for BigQuery credentials field if "credentials" in connection_details and isinstance(connection_details["credentials"], str): try: @@ -457,26 +457,26 @@ def decrypt_connection_details_robust(connection_details: Dict[str, Any]) -> Dic except Exception as e: logging.error(f"Error decrypting BigQuery credentials: {e}") decrypted_data["credentials"] = connection_details["credentials"] - + # Find other sensitive fields sensitive_fields = get_sensitive_fields_in_data(decrypted_data) logging.info(f"Found sensitive fields: {sensitive_fields}") - + # Process each sensitive field (excluding credentials which was handled above) for field in sensitive_fields: if field in decrypted_data and field != "credentials": # Skip credentials as it's already handled original_value = decrypted_data[field] - + if isinstance(original_value, str): # Validate the encrypted data validation = validate_encrypted_data(original_value) if validation["errors"]: logging.warning(f"Field '{field}' validation errors: {validation['errors']}") - + # Check if it appears to be encrypted if is_encrypted_value(original_value): logging.info(f"Field '{field}' appears to be encrypted, attempting decryption...") - + # Log detailed debug info for problematic fields if validation["warnings"] or not validation["is_valid"]: logging.debug(get_encryption_debug_info(original_value)) @@ -500,10 +500,10 @@ def decrypt_connection_details_robust(connection_details: Dict[str, Any]) -> Dic else: logging.debug(f"Field '{field}' is not a string, keeping as-is") decrypted_data[field] = original_value - + logging.info("Completed robust connection_details decryption") return decrypted_data - + except Exception as e: logging.error(f"❌ Critical error during connection_details decryption: {e}") logging.error(f"connection_details content: {connection_details}") @@ -516,25 +516,25 @@ def decrypt_connection_details_robust(connection_details: Dict[str, Any]) -> Dic def is_valid_encrypted_data(value: str) -> bool: """ Check if a value is valid encrypted data. - + Args: value: The value to check - + Returns: True if the value appears to be valid encrypted data """ if not isinstance(value, str): return False - + # Check if it's a reasonable length for encrypted data if len(value) < 100: return False - + # Check if it contains only base64 characters valid_chars = set('ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=') if not all(c in valid_chars for c in value): return False - + # Check if it's properly padded base64 try: # Try to decode as base64 @@ -547,25 +547,25 @@ def is_valid_encrypted_data(value: str) -> bool: def decrypt_field_safely(field_name: str, field_value: str) -> str: """ Safely decrypt a single field with comprehensive error handling. - + Args: field_name: Name of the field being decrypted field_value: Value to decrypt - + Returns: Decrypted value or original value if decryption fails """ logging.debug(f"Attempting to decrypt field '{field_name}'") - + if not isinstance(field_value, str): logging.debug(f"Field '{field_name}' is not a string, returning as-is") return field_value - + # Check if it looks like encrypted data if not is_valid_encrypted_data(field_value): logging.debug(f"Field '{field_name}' does not appear to be valid encrypted data") return field_value - + # Try to decrypt try: decrypted_value = decrypt_with_private_key(field_value) @@ -577,4 +577,4 @@ def decrypt_field_safely(field_name: str, field_value: str) -> str: return field_value except Exception as e: logging.error(f"❌ Error decrypting field '{field_name}': {e}") - return field_value \ No newline at end of file + return field_value diff --git a/backend/backend/utils/rsa_encryption.py b/backend/backend/utils/rsa_encryption.py index 8d534c1..73aea78 100644 --- a/backend/backend/utils/rsa_encryption.py +++ b/backend/backend/utils/rsa_encryption.py @@ -117,25 +117,25 @@ def generate_rsa_key_pair() -> Tuple[str, str]: key_size=RSA_KEY_SIZE, backend=default_backend() ) - + # Get public key public_key = private_key.public_key() - + # Convert to PEM format private_pem = private_key.private_bytes( encoding=serialization.Encoding.PEM, format=serialization.PrivateFormat.PKCS8, encryption_algorithm=serialization.NoEncryption() ).decode('utf-8') - + public_pem = public_key.public_bytes( encoding=serialization.Encoding.PEM, format=serialization.PublicFormat.SubjectPublicKeyInfo ).decode('utf-8') - + logger.info("RSA key pair generated successfully") return private_pem, public_pem - + except Exception as e: logger.error(f"Error generating RSA key pair: {e}") raise @@ -148,15 +148,15 @@ def encrypt_with_public_key(data: str) -> Optional[str]: if not public_key: logger.error("Cannot encrypt: RSA public key not available") return None - + # Convert string to bytes data_bytes = data.encode('utf-8') - + # Check data size if len(data_bytes) > MAX_RSA_ENCRYPT_SIZE: logger.error(f"Data too large for RSA encryption: {len(data_bytes)} bytes") return None - + # Encrypt data encrypted_bytes = public_key.encrypt( data_bytes, @@ -166,12 +166,12 @@ def encrypt_with_public_key(data: str) -> Optional[str]: label=None ) ) - + # Convert to base64 for safe transmission encrypted_b64 = base64.b64encode(encrypted_bytes).decode('utf-8') logger.debug(f"Data encrypted successfully: {len(data_bytes)} bytes") return encrypted_b64 - + except Exception as e: logger.error(f"Error encrypting data with RSA public key: {e}") return None @@ -184,16 +184,16 @@ def decrypt_with_private_key(encrypted_data: str) -> Optional[str]: if not private_key: logger.error("Cannot decrypt: RSA private key not available") return None - + # Validate input if not isinstance(encrypted_data, str): logger.error(f"Invalid input type: {type(encrypted_data)}, expected str") return None - + if not encrypted_data.strip(): logger.error("Empty encrypted data") return None - + # Check if it looks like base64 data try: # Convert from base64 @@ -203,16 +203,16 @@ def decrypt_with_private_key(encrypted_data: str) -> Optional[str]: logger.error(f"Failed to decode base64: {e}") logger.debug(f"Encrypted data preview: {encrypted_data[:100]}...") return None - + # Check if the data size is reasonable for RSA if len(encrypted_bytes) != 256: # 2048-bit RSA produces 256-byte output logger.warning(f"Unexpected encrypted data size: {len(encrypted_bytes)} bytes (expected 256 for RSA-2048)") logger.debug(f"This might indicate the data is not properly encrypted") - + # Try different padding schemes from cryptography.hazmat.primitives import hashes from cryptography.hazmat.primitives.asymmetric import padding - + # Method 1: OAEP with SHA256 (original) try: logger.debug("Attempting decryption with OAEP SHA256...") @@ -229,7 +229,7 @@ def decrypt_with_private_key(encrypted_data: str) -> Optional[str]: return decrypted_data except Exception as e: logger.debug(f"OAEP SHA256 decryption failed: {e}") - + # Method 2: OAEP with SHA1 try: logger.debug("Attempting decryption with OAEP SHA1...") @@ -246,7 +246,7 @@ def decrypt_with_private_key(encrypted_data: str) -> Optional[str]: return decrypted_data except Exception as e: logger.debug(f"OAEP SHA1 decryption failed: {e}") - + # Method 3: PKCS1v15 try: logger.debug("Attempting decryption with PKCS1v15...") @@ -259,15 +259,15 @@ def decrypt_with_private_key(encrypted_data: str) -> Optional[str]: return decrypted_data except Exception as e: logger.debug(f"PKCS1v15 decryption failed: {e}") - + # If all methods fail, log the error logger.error("All decryption methods failed") logger.debug(f"Encrypted data length: {len(encrypted_data)}") logger.debug(f"Encrypted data preview: {encrypted_data[:100]}...") logger.debug(f"Decoded bytes length: {len(encrypted_bytes)}") - + return None - + except Exception as e: logger.error(f"Error decrypting data with RSA private key: {e}") logger.debug(f"Encrypted data type: {type(encrypted_data)}") @@ -281,38 +281,38 @@ def validate_rsa_keys() -> bool: try: private_key = get_rsa_private_key() public_key = get_rsa_public_key() - + if not private_key or not public_key: logger.error("RSA keys validation failed: keys not available") return False - + # Test encryption/decryption test_data = "test_encryption" encrypted = encrypt_with_public_key(test_data) if not encrypted: logger.error("RSA keys validation failed: encryption failed") return False - + decrypted = decrypt_with_private_key(encrypted) if not decrypted or decrypted != test_data: logger.error("RSA keys validation failed: decryption failed") return False - + logger.info("RSA keys validation successful") return True - + except Exception as e: logger.error(f"RSA keys validation failed: {e}") - return False + return False def validate_encrypted_data(encrypted_data: str) -> dict: """ Validate encrypted data and provide detailed analysis. - + Args: encrypted_data: The encrypted data string to validate - + Returns: Dictionary with validation results and analysis """ @@ -322,67 +322,67 @@ def validate_encrypted_data(encrypted_data: str) -> dict: "warnings": [], "analysis": {} } - + try: # Check if it's a string if not isinstance(encrypted_data, str): result["errors"].append(f"Invalid type: {type(encrypted_data)}, expected str") return result - + # Check if it's empty if not encrypted_data.strip(): result["errors"].append("Empty encrypted data") return result - + # Check length result["analysis"]["length"] = len(encrypted_data) if len(encrypted_data) < 100: result["warnings"].append(f"Data seems too short for RSA encryption: {len(encrypted_data)} chars") - + # Check if it contains only base64 characters valid_chars = set('ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=') invalid_chars = set(encrypted_data) - valid_chars if invalid_chars: result["errors"].append(f"Contains invalid base64 characters: {invalid_chars}") return result - + # Try to decode as base64 try: decoded = base64.b64decode(encrypted_data) result["analysis"]["decoded_length"] = len(decoded) result["analysis"]["decoded_bytes"] = decoded[:10].hex() # First 10 bytes as hex - + # Check if it's the right size for RSA-2048 if len(decoded) == 256: result["analysis"]["rsa_size"] = "correct" else: result["warnings"].append(f"Unexpected size for RSA-2048: {len(decoded)} bytes (expected 256)") result["analysis"]["rsa_size"] = "incorrect" - + result["is_valid"] = True - + except Exception as e: result["errors"].append(f"Invalid base64: {e}") return result - + except Exception as e: result["errors"].append(f"Validation error: {e}") - + return result def get_encryption_debug_info(encrypted_data: str) -> str: """ Get detailed debug information about encrypted data. - + Args: encrypted_data: The encrypted data to analyze - + Returns: Formatted debug information string """ validation = validate_encrypted_data(encrypted_data) - + debug_info = f""" 🔍 Encrypted Data Analysis ======================== @@ -397,8 +397,8 @@ def get_encryption_debug_info(encrypted_data: str) -> str: Analysis: """ - + for key, value in validation['analysis'].items(): debug_info += f"- {key}: {value}\n" - - return debug_info \ No newline at end of file + + return debug_info diff --git a/backend/backend/utils/sample_project/dvd_rental/model_files/customer_details_with_address.json b/backend/backend/utils/sample_project/dvd_rental/model_files/customer_details_with_address.json index 5c455c5..98c3d57 100644 --- a/backend/backend/utils/sample_project/dvd_rental/model_files/customer_details_with_address.json +++ b/backend/backend/utils/sample_project/dvd_rental/model_files/customer_details_with_address.json @@ -302,4 +302,4 @@ "transformation_id": "sql" } ] -} \ No newline at end of file +} diff --git a/backend/backend/utils/sample_project/dvd_rental/model_files/customer_email_count.json b/backend/backend/utils/sample_project/dvd_rental/model_files/customer_email_count.json index 406a87b..5dcf956 100644 --- a/backend/backend/utils/sample_project/dvd_rental/model_files/customer_email_count.json +++ b/backend/backend/utils/sample_project/dvd_rental/model_files/customer_email_count.json @@ -27,4 +27,4 @@ "transformation_id": "sql" } ] -} \ No newline at end of file +} diff --git a/backend/backend/utils/sample_project/dvd_rental/model_files/customer_lifetime_value.json b/backend/backend/utils/sample_project/dvd_rental/model_files/customer_lifetime_value.json index b31c10b..40b889a 100644 --- a/backend/backend/utils/sample_project/dvd_rental/model_files/customer_lifetime_value.json +++ b/backend/backend/utils/sample_project/dvd_rental/model_files/customer_lifetime_value.json @@ -318,4 +318,4 @@ "transformation_id": "sql" } ] -} \ No newline at end of file +} diff --git a/backend/backend/utils/sample_project/dvd_rental/model_files/customer_rental_activity.json b/backend/backend/utils/sample_project/dvd_rental/model_files/customer_rental_activity.json index c82ec09..74c699c 100644 --- a/backend/backend/utils/sample_project/dvd_rental/model_files/customer_rental_activity.json +++ b/backend/backend/utils/sample_project/dvd_rental/model_files/customer_rental_activity.json @@ -80,4 +80,4 @@ "transformation_id": "sql" } ] -} \ No newline at end of file +} diff --git a/backend/backend/utils/sample_project/dvd_rental/model_files/film_replacement_cost_summary.json b/backend/backend/utils/sample_project/dvd_rental/model_files/film_replacement_cost_summary.json index aa370c8..6e722d2 100644 --- a/backend/backend/utils/sample_project/dvd_rental/model_files/film_replacement_cost_summary.json +++ b/backend/backend/utils/sample_project/dvd_rental/model_files/film_replacement_cost_summary.json @@ -27,4 +27,4 @@ "transformation_id": "sql" } ] -} \ No newline at end of file +} diff --git a/backend/backend/utils/sample_project/dvd_rental/model_files/payment_amount_summary.json b/backend/backend/utils/sample_project/dvd_rental/model_files/payment_amount_summary.json index 3236be7..4b9534c 100644 --- a/backend/backend/utils/sample_project/dvd_rental/model_files/payment_amount_summary.json +++ b/backend/backend/utils/sample_project/dvd_rental/model_files/payment_amount_summary.json @@ -27,4 +27,4 @@ "transformation_id": "sql" } ] -} \ No newline at end of file +} diff --git a/backend/backend/utils/sample_project/dvd_rental/model_files/staff_contact_info.json b/backend/backend/utils/sample_project/dvd_rental/model_files/staff_contact_info.json index 174cc72..a8f3e67 100644 --- a/backend/backend/utils/sample_project/dvd_rental/model_files/staff_contact_info.json +++ b/backend/backend/utils/sample_project/dvd_rental/model_files/staff_contact_info.json @@ -27,4 +27,4 @@ "transformation_id": "sql" } ] -} \ No newline at end of file +} diff --git a/backend/backend/utils/sample_project/dvd_rental/model_files/store_active_customer.json b/backend/backend/utils/sample_project/dvd_rental/model_files/store_active_customer.json index 5d28640..3280499 100644 --- a/backend/backend/utils/sample_project/dvd_rental/model_files/store_active_customer.json +++ b/backend/backend/utils/sample_project/dvd_rental/model_files/store_active_customer.json @@ -57,4 +57,4 @@ "transformation_id": "sql" } ] -} \ No newline at end of file +} diff --git a/backend/backend/utils/sample_project/dvd_rental/model_files/store_active_customer_counts.json b/backend/backend/utils/sample_project/dvd_rental/model_files/store_active_customer_counts.json index b209114..af67c25 100644 --- a/backend/backend/utils/sample_project/dvd_rental/model_files/store_active_customer_counts.json +++ b/backend/backend/utils/sample_project/dvd_rental/model_files/store_active_customer_counts.json @@ -76,4 +76,4 @@ "transformation_id": "sql" } ] -} \ No newline at end of file +} diff --git a/backend/backend/utils/sample_project/dvd_rental/model_files/store_inventory_category_cost_summary.json b/backend/backend/utils/sample_project/dvd_rental/model_files/store_inventory_category_cost_summary.json index 6e63f4e..dd09a94 100644 --- a/backend/backend/utils/sample_project/dvd_rental/model_files/store_inventory_category_cost_summary.json +++ b/backend/backend/utils/sample_project/dvd_rental/model_files/store_inventory_category_cost_summary.json @@ -528,4 +528,4 @@ "transformation_id": "sql" } ] -} \ No newline at end of file +} diff --git a/backend/backend/utils/sample_project/dvd_rental/model_files/store_inventory_counts.json b/backend/backend/utils/sample_project/dvd_rental/model_files/store_inventory_counts.json index 398a0d4..1b662a9 100644 --- a/backend/backend/utils/sample_project/dvd_rental/model_files/store_inventory_counts.json +++ b/backend/backend/utils/sample_project/dvd_rental/model_files/store_inventory_counts.json @@ -73,4 +73,4 @@ "transformation_id": "sql" } ] -} \ No newline at end of file +} diff --git a/backend/backend/utils/sample_project/dvd_rental/model_files/store_inventory_details.json b/backend/backend/utils/sample_project/dvd_rental/model_files/store_inventory_details.json index 98c0bd4..1b657a2 100644 --- a/backend/backend/utils/sample_project/dvd_rental/model_files/store_inventory_details.json +++ b/backend/backend/utils/sample_project/dvd_rental/model_files/store_inventory_details.json @@ -184,4 +184,4 @@ "transformation_id": "sql" } ] -} \ No newline at end of file +} diff --git a/backend/backend/utils/sample_project/dvd_rental/model_files/store_inventory_rating_summary.json b/backend/backend/utils/sample_project/dvd_rental/model_files/store_inventory_rating_summary.json index 837010f..a4bed7c 100644 --- a/backend/backend/utils/sample_project/dvd_rental/model_files/store_inventory_rating_summary.json +++ b/backend/backend/utils/sample_project/dvd_rental/model_files/store_inventory_rating_summary.json @@ -235,4 +235,4 @@ "transformation_id": "sql" } ] -} \ No newline at end of file +} diff --git a/backend/backend/utils/sample_project/dvd_rental/model_files/store_manager_locations.json b/backend/backend/utils/sample_project/dvd_rental/model_files/store_manager_locations.json index 113b90e..b593908 100644 --- a/backend/backend/utils/sample_project/dvd_rental/model_files/store_manager_locations.json +++ b/backend/backend/utils/sample_project/dvd_rental/model_files/store_manager_locations.json @@ -572,4 +572,4 @@ "transformation_id": "sql" } ] -} \ No newline at end of file +} diff --git a/backend/backend/utils/sample_project/dvd_rental/model_files/store_unique_film_count.json b/backend/backend/utils/sample_project/dvd_rental/model_files/store_unique_film_count.json index 11e6a57..b98c8bb 100644 --- a/backend/backend/utils/sample_project/dvd_rental/model_files/store_unique_film_count.json +++ b/backend/backend/utils/sample_project/dvd_rental/model_files/store_unique_film_count.json @@ -27,4 +27,4 @@ "transformation_id": "sql" } ] -} \ No newline at end of file +} diff --git a/backend/backend/utils/sample_project/dvd_rental/model_files/total_unique_film_categories.json b/backend/backend/utils/sample_project/dvd_rental/model_files/total_unique_film_categories.json index d7de539..d313f99 100644 --- a/backend/backend/utils/sample_project/dvd_rental/model_files/total_unique_film_categories.json +++ b/backend/backend/utils/sample_project/dvd_rental/model_files/total_unique_film_categories.json @@ -27,4 +27,4 @@ "transformation_id": "sql" } ] -} \ No newline at end of file +} diff --git a/backend/backend/utils/sample_project/jaffle_shop/model_files/dev_customers.json b/backend/backend/utils/sample_project/jaffle_shop/model_files/dev_customers.json index 3348ad2..edd62db 100644 --- a/backend/backend/utils/sample_project/jaffle_shop/model_files/dev_customers.json +++ b/backend/backend/utils/sample_project/jaffle_shop/model_files/dev_customers.json @@ -77,4 +77,4 @@ "transformation_id": "sql" } ] -} \ No newline at end of file +} diff --git a/backend/backend/utils/sample_project/jaffle_shop/model_files/dev_orders.json b/backend/backend/utils/sample_project/jaffle_shop/model_files/dev_orders.json index df39a4f..14d624b 100644 --- a/backend/backend/utils/sample_project/jaffle_shop/model_files/dev_orders.json +++ b/backend/backend/utils/sample_project/jaffle_shop/model_files/dev_orders.json @@ -88,4 +88,4 @@ "transformation_id": "sql" } ] -} \ No newline at end of file +} diff --git a/backend/backend/utils/sample_project/jaffle_shop/model_files/dev_payments.json b/backend/backend/utils/sample_project/jaffle_shop/model_files/dev_payments.json index c8205e6..72ba98b 100644 --- a/backend/backend/utils/sample_project/jaffle_shop/model_files/dev_payments.json +++ b/backend/backend/utils/sample_project/jaffle_shop/model_files/dev_payments.json @@ -132,4 +132,4 @@ "transformation_id": "sql" } ] -} \ No newline at end of file +} diff --git a/backend/backend/utils/sample_project/jaffle_shop/model_files/prod_customer_ltv.json b/backend/backend/utils/sample_project/jaffle_shop/model_files/prod_customer_ltv.json index eae5e4b..211bd4a 100644 --- a/backend/backend/utils/sample_project/jaffle_shop/model_files/prod_customer_ltv.json +++ b/backend/backend/utils/sample_project/jaffle_shop/model_files/prod_customer_ltv.json @@ -179,4 +179,4 @@ "transformation_id": "sql" } ] -} \ No newline at end of file +} diff --git a/backend/backend/utils/sample_project/jaffle_shop/model_files/prod_order_details.json b/backend/backend/utils/sample_project/jaffle_shop/model_files/prod_order_details.json index ff52b3c..38b6acf 100644 --- a/backend/backend/utils/sample_project/jaffle_shop/model_files/prod_order_details.json +++ b/backend/backend/utils/sample_project/jaffle_shop/model_files/prod_order_details.json @@ -80,4 +80,4 @@ "transformation_id": "sql" } ] -} \ No newline at end of file +} diff --git a/backend/backend/utils/sample_project/jaffle_shop/model_files/stg_aggr_order_payments.json b/backend/backend/utils/sample_project/jaffle_shop/model_files/stg_aggr_order_payments.json index 8e58a6c..2ec33c6 100644 --- a/backend/backend/utils/sample_project/jaffle_shop/model_files/stg_aggr_order_payments.json +++ b/backend/backend/utils/sample_project/jaffle_shop/model_files/stg_aggr_order_payments.json @@ -105,4 +105,4 @@ "transformation_id": "sql" } ] -} \ No newline at end of file +} diff --git a/backend/backend/utils/sample_project/jaffle_shop/model_files/stg_order_summaries.json b/backend/backend/utils/sample_project/jaffle_shop/model_files/stg_order_summaries.json index c209147..cb7aa56 100644 --- a/backend/backend/utils/sample_project/jaffle_shop/model_files/stg_order_summaries.json +++ b/backend/backend/utils/sample_project/jaffle_shop/model_files/stg_order_summaries.json @@ -71,4 +71,4 @@ "transformation_id": "sql" } ] -} \ No newline at end of file +} diff --git a/backend/backend/utils/sample_project/jaffle_shop/model_files/stg_payments_by_type.json b/backend/backend/utils/sample_project/jaffle_shop/model_files/stg_payments_by_type.json index 9b7ebbb..497cb2c 100644 --- a/backend/backend/utils/sample_project/jaffle_shop/model_files/stg_payments_by_type.json +++ b/backend/backend/utils/sample_project/jaffle_shop/model_files/stg_payments_by_type.json @@ -157,4 +157,4 @@ "transformation_id": "sql" } ] -} \ No newline at end of file +} diff --git a/backend/backend/utils/sample_project/jaffle_shop/seed_files/raw_orders.csv b/backend/backend/utils/sample_project/jaffle_shop/seed_files/raw_orders.csv index 7c2be07..c487062 100644 --- a/backend/backend/utils/sample_project/jaffle_shop/seed_files/raw_orders.csv +++ b/backend/backend/utils/sample_project/jaffle_shop/seed_files/raw_orders.csv @@ -1,100 +1,100 @@ -id,user_id,order_date,status -1,1,2018-01-01,returned -2,3,2018-01-02,completed -3,94,2018-01-04,completed -4,50,2018-01-05,completed -5,64,2018-01-05,completed -6,54,2018-01-07,completed -7,88,2018-01-09,completed -8,2,2018-01-11,returned -9,53,2018-01-12,completed -10,7,2018-01-14,completed -11,99,2018-01-14,completed -12,59,2018-01-15,completed -13,84,2018-01-17,completed -14,40,2018-01-17,returned -15,25,2018-01-17,completed -16,39,2018-01-18,completed -17,71,2018-01-18,completed -18,64,2018-01-20,returned -19,54,2018-01-22,completed -20,20,2018-01-23,completed -21,71,2018-01-23,completed -22,86,2018-01-24,completed -23,22,2018-01-26,return_pending -24,3,2018-01-27,completed -25,51,2018-01-28,completed -26,32,2018-01-28,completed -27,94,2018-01-29,completed -28,8,2018-01-29,completed -29,57,2018-01-31,completed -30,69,2018-02-02,completed -31,16,2018-02-02,completed -32,28,2018-02-04,completed -33,42,2018-02-04,completed -34,38,2018-02-06,completed -35,80,2018-02-08,completed -36,85,2018-02-10,completed -37,1,2018-02-10,completed -38,51,2018-02-10,completed -39,26,2018-02-11,completed -40,33,2018-02-13,completed -41,99,2018-02-14,completed -42,92,2018-02-16,completed -43,31,2018-02-17,completed -44,66,2018-02-17,completed -45,22,2018-02-17,completed -46,6,2018-02-19,completed -47,50,2018-02-20,completed -48,27,2018-02-21,completed -49,35,2018-02-21,completed -50,51,2018-02-23,completed -51,71,2018-02-24,completed -52,54,2018-02-25,return_pending -53,34,2018-02-26,completed -54,54,2018-02-26,completed -55,18,2018-02-27,completed -56,79,2018-02-28,completed -57,93,2018-03-01,completed -58,22,2018-03-01,completed -59,30,2018-03-02,completed -60,12,2018-03-03,completed -61,63,2018-03-03,completed -62,57,2018-03-05,completed -63,70,2018-03-06,completed -64,13,2018-03-07,completed -65,26,2018-03-08,completed -66,36,2018-03-10,completed -67,79,2018-03-11,completed -68,53,2018-03-11,completed -69,3,2018-03-11,completed -70,8,2018-03-12,completed -71,42,2018-03-12,shipped -72,30,2018-03-14,shipped -73,19,2018-03-16,completed -74,9,2018-03-17,shipped -75,69,2018-03-18,completed -76,25,2018-03-20,completed -77,35,2018-03-21,shipped -78,90,2018-03-23,shipped -79,52,2018-03-23,shipped -80,11,2018-03-23,shipped -81,76,2018-03-23,shipped -82,46,2018-03-24,shipped -83,54,2018-03-24,shipped -84,70,2018-03-26,placed -85,47,2018-03-26,shipped -86,68,2018-03-26,placed -87,46,2018-03-27,placed -88,91,2018-03-27,shipped -89,21,2018-03-28,placed -90,66,2018-03-30,shipped -91,47,2018-03-31,placed -92,84,2018-04-02,placed -93,66,2018-04-03,placed -94,63,2018-04-03,placed -95,27,2018-04-04,placed -96,90,2018-04-06,placed -97,89,2018-04-07,placed -98,41,2018-04-07,placed -99,85,2018-04-09,placed +id,user_id,order_date,status +1,1,2018-01-01,returned +2,3,2018-01-02,completed +3,94,2018-01-04,completed +4,50,2018-01-05,completed +5,64,2018-01-05,completed +6,54,2018-01-07,completed +7,88,2018-01-09,completed +8,2,2018-01-11,returned +9,53,2018-01-12,completed +10,7,2018-01-14,completed +11,99,2018-01-14,completed +12,59,2018-01-15,completed +13,84,2018-01-17,completed +14,40,2018-01-17,returned +15,25,2018-01-17,completed +16,39,2018-01-18,completed +17,71,2018-01-18,completed +18,64,2018-01-20,returned +19,54,2018-01-22,completed +20,20,2018-01-23,completed +21,71,2018-01-23,completed +22,86,2018-01-24,completed +23,22,2018-01-26,return_pending +24,3,2018-01-27,completed +25,51,2018-01-28,completed +26,32,2018-01-28,completed +27,94,2018-01-29,completed +28,8,2018-01-29,completed +29,57,2018-01-31,completed +30,69,2018-02-02,completed +31,16,2018-02-02,completed +32,28,2018-02-04,completed +33,42,2018-02-04,completed +34,38,2018-02-06,completed +35,80,2018-02-08,completed +36,85,2018-02-10,completed +37,1,2018-02-10,completed +38,51,2018-02-10,completed +39,26,2018-02-11,completed +40,33,2018-02-13,completed +41,99,2018-02-14,completed +42,92,2018-02-16,completed +43,31,2018-02-17,completed +44,66,2018-02-17,completed +45,22,2018-02-17,completed +46,6,2018-02-19,completed +47,50,2018-02-20,completed +48,27,2018-02-21,completed +49,35,2018-02-21,completed +50,51,2018-02-23,completed +51,71,2018-02-24,completed +52,54,2018-02-25,return_pending +53,34,2018-02-26,completed +54,54,2018-02-26,completed +55,18,2018-02-27,completed +56,79,2018-02-28,completed +57,93,2018-03-01,completed +58,22,2018-03-01,completed +59,30,2018-03-02,completed +60,12,2018-03-03,completed +61,63,2018-03-03,completed +62,57,2018-03-05,completed +63,70,2018-03-06,completed +64,13,2018-03-07,completed +65,26,2018-03-08,completed +66,36,2018-03-10,completed +67,79,2018-03-11,completed +68,53,2018-03-11,completed +69,3,2018-03-11,completed +70,8,2018-03-12,completed +71,42,2018-03-12,shipped +72,30,2018-03-14,shipped +73,19,2018-03-16,completed +74,9,2018-03-17,shipped +75,69,2018-03-18,completed +76,25,2018-03-20,completed +77,35,2018-03-21,shipped +78,90,2018-03-23,shipped +79,52,2018-03-23,shipped +80,11,2018-03-23,shipped +81,76,2018-03-23,shipped +82,46,2018-03-24,shipped +83,54,2018-03-24,shipped +84,70,2018-03-26,placed +85,47,2018-03-26,shipped +86,68,2018-03-26,placed +87,46,2018-03-27,placed +88,91,2018-03-27,shipped +89,21,2018-03-28,placed +90,66,2018-03-30,shipped +91,47,2018-03-31,placed +92,84,2018-04-02,placed +93,66,2018-04-03,placed +94,63,2018-04-03,placed +95,27,2018-04-04,placed +96,90,2018-04-06,placed +97,89,2018-04-07,placed +98,41,2018-04-07,placed +99,85,2018-04-09,placed diff --git a/backend/backend/utils/utils.py b/backend/backend/utils/utils.py index 95118ac..0996f65 100644 --- a/backend/backend/utils/utils.py +++ b/backend/backend/utils/utils.py @@ -73,4 +73,4 @@ def db_type_mapper() -> dict[str, str]: def convert_db_type_to_no_code_type(db_type: str) -> str: db_type = "".join([i for i in db_type if not i.isdigit() and i.isalnum()]) db_type = db_type.lower() - return db_type_mapper().get(db_type, "String") \ No newline at end of file + return db_type_mapper().get(db_type, "String") diff --git a/backend/celerybeat-schedule.db b/backend/celerybeat-schedule.db index bcb1312..f245f5b 100644 Binary files a/backend/celerybeat-schedule.db and b/backend/celerybeat-schedule.db differ diff --git a/backend/entrypoint.sh b/backend/entrypoint.sh index 97a3d96..af79c35 100755 --- a/backend/entrypoint.sh +++ b/backend/entrypoint.sh @@ -23,4 +23,3 @@ fi --reuse-port \ --backlog 1024 \ backend.server.wsgi:application - \ No newline at end of file diff --git a/backend/formulasql/examples/sample/geography.db b/backend/formulasql/examples/sample/geography.db index ffdacdb..32c9732 100644 Binary files a/backend/formulasql/examples/sample/geography.db and b/backend/formulasql/examples/sample/geography.db differ diff --git a/backend/formulasql/functions/logics.py b/backend/formulasql/functions/logics.py index 0c0bf8e..f93465d 100644 --- a/backend/formulasql/functions/logics.py +++ b/backend/formulasql/functions/logics.py @@ -43,7 +43,7 @@ def if_(table, node, data_types, inter_exps): e1 = FormulaSQLUtils.build_ibis_expression(table, data_types, inter_exps, params[0]) e2 = FormulaSQLUtils.build_ibis_expression(table, data_types, inter_exps, params[1]) e3 = FormulaSQLUtils.build_ibis_expression(table, data_types, inter_exps, params[2]) - + # Infer types where possible if e2 is not None and not e2.equals(null()): e3 = ensure_typed_null(e3, e2.type()) @@ -269,7 +269,7 @@ def false_(table, node, data_types, inter_exps): e = ibis.literal(False) data_types[node['outputs'][0]] = 'boolean' return e - + @staticmethod def between(table, node, data_types, inter_exps): params = node['inputs'] @@ -379,4 +379,4 @@ def coalesce(table, node, data_types, inter_exps): for inp in node['inputs']] e = ibis.coalesce(*exprs) data_types[node['outputs'][0]] = data_types.get(node['inputs'][0], 'string') - return e \ No newline at end of file + return e diff --git a/backend/formulasql/functions/text.py b/backend/formulasql/functions/text.py index 86197d8..dad7de2 100644 --- a/backend/formulasql/functions/text.py +++ b/backend/formulasql/functions/text.py @@ -86,7 +86,7 @@ def concatenate(table, node, data_types, inter_exps): inter_exps[node['outputs'][0]] = e data_types[node['outputs'][0]] = 'string' return e - + @staticmethod def concat(table, node, data_types, inter_exps): return Text.concatenate(table, node, data_types, inter_exps) @@ -252,7 +252,7 @@ def substitute(table, node, data_types, inter_exps): raise Exception("SUBSTITUTE function requires 3 parameters") data_types[node['outputs'][0]] = 'string' return e - + @staticmethod def trim(table, node, data_types, inter_exps): if node['inputs'].__len__() == 1: diff --git a/backend/formulasql/tests/conftest.py b/backend/formulasql/tests/conftest.py index 7e729d3..a09ee96 100644 --- a/backend/formulasql/tests/conftest.py +++ b/backend/formulasql/tests/conftest.py @@ -26,7 +26,7 @@ def mysql_sakila_db(): engine =sa.create_engine( f"mysql+pymysql://visitran:{mysql_password}@localhost:3307/sakila?charset=utf8mb4" ) - + mysqldata = ConnectionData( "localhost", 3307, @@ -40,7 +40,7 @@ def mysql_sakila_db(): ) yield mysqldata - + engine.dispose() diff --git a/backend/formulasql/tests/db_data/geography.db b/backend/formulasql/tests/db_data/geography.db index b001fa4..4fa2a2c 100644 Binary files a/backend/formulasql/tests/db_data/geography.db and b/backend/formulasql/tests/db_data/geography.db differ diff --git a/backend/formulasql/tests/db_data/sakila-schema.sql b/backend/formulasql/tests/db_data/sakila-schema.sql index 0cfcf98..112d408 100644 --- a/backend/formulasql/tests/db_data/sakila-schema.sql +++ b/backend/formulasql/tests/db_data/sakila-schema.sql @@ -185,9 +185,9 @@ CREATE TABLE film_category ( -- -- Table structure for table `film_text` --- +-- -- InnoDB added FULLTEXT support in 5.6.10. If you use an --- earlier version, then consider upgrading (recommended) or +-- earlier version, then consider upgrading (recommended) or -- changing InnoDB to MyISAM as the film_text engine -- diff --git a/backend/formulasql/tests/test_formulasql_datetime.py b/backend/formulasql/tests/test_formulasql_datetime.py index 26f1ec3..87ad58e 100644 --- a/backend/formulasql/tests/test_formulasql_datetime.py +++ b/backend/formulasql/tests/test_formulasql_datetime.py @@ -32,7 +32,7 @@ def setup(self,mysql_sakila_db): self.connection_mysql = ibis.mysql.connect(host=host, port=port, user=user, password=password, database='sakila') self.payment = self.connection_mysql.table('payment') - + # def test_uniq(self): # assert 1==1 diff --git a/backend/formulasql/tests/test_formulasql_logics.py b/backend/formulasql/tests/test_formulasql_logics.py index 945813f..3e6ea27 100644 --- a/backend/formulasql/tests/test_formulasql_logics.py +++ b/backend/formulasql/tests/test_formulasql_logics.py @@ -16,7 +16,7 @@ # 4 Anguilla NA 13254 102.0 class TestFormulaSQLLogics: - + @pytest.fixture(autouse=True) def setup(self): self.connection = ibis.sqlite.connect('formulasql/tests/db_data/geography.db') diff --git a/backend/formulasql/tests/test_formulasql_text.py b/backend/formulasql/tests/test_formulasql_text.py index 362d151..7ae46f8 100644 --- a/backend/formulasql/tests/test_formulasql_text.py +++ b/backend/formulasql/tests/test_formulasql_text.py @@ -31,7 +31,7 @@ def setup(self,mysql_sakila_db): database='sakila') self.payment = self.connection_mysql.table('payment') - + def test_numbervalue(self): formula = FormulaSQL(self.countries, 'test_col1', '=NUMBERVALUE("84000")') @@ -58,7 +58,7 @@ def test_code(self): countries_x = self.countries.mutate(formula.ibis_column()) row = countries_x['name', 'continent', 'population', 'area_km2', 'test_col1'].head().execute().iloc[0] assert (row['test_col1']== 581) - + def test_concatenate(self): formula = FormulaSQL(self.countries, 'test_col1', '=CONCATENATE("Visitran", " says hello world!")') countries_x = self.countries.mutate(formula.ibis_column()) diff --git a/backend/formulasql/tests/validate_new_formulas.py b/backend/formulasql/tests/validate_new_formulas.py old mode 100644 new mode 100755 diff --git a/backend/rbac/factory.py b/backend/rbac/factory.py index 057d738..7339310 100644 --- a/backend/rbac/factory.py +++ b/backend/rbac/factory.py @@ -50,4 +50,4 @@ def wrapped_view(view_or_request, *args, **kwargs): else: return view_func(*args, **kwargs) - return wrapped_view \ No newline at end of file + return wrapped_view diff --git a/backend/tests/integration_tests/data/invalid_csv/staff.csv b/backend/tests/integration_tests/data/invalid_csv/staff.csv index 2455ebc..2ad72df 100644 Binary files a/backend/tests/integration_tests/data/invalid_csv/staff.csv and b/backend/tests/integration_tests/data/invalid_csv/staff.csv differ diff --git a/backend/tests/integration_tests/data/sakila/film.csv b/backend/tests/integration_tests/data/sakila/film.csv index f371dfe..b487f1e 100644 --- a/backend/tests/integration_tests/data/sakila/film.csv +++ b/backend/tests/integration_tests/data/sakila/film.csv @@ -1,3 +1,11 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:14a79792a1386ad6a469ef5d50c49d2008adbbee8516e45cb8cb208faabe07ae -size 3526 +film_id,title,description,release_year,language_id,original_language_id,rental_duration,rental_rate,length,replacement_cost,rating,last_update,special_features,fulltext +1,ACADEMY DINOSAUR,An Epic Drama of a Feminist And a Mad Scientist who must Battle a Teacher in The Canadian Rockies,2006,1,,6,0.99,86,20.99,PG,2007-09-10 17:46:03.905795,"{""Deleted Scenes"",""Behind the Scenes""}",'academi':1 'battl':15 'canadian':20 'dinosaur':2 'drama':5 'epic':4 'feminist':8 'mad':11 'must':14 'rocki':21 'scientist':12 'teacher':17 +2,ACE GOLDFINGER,A Astounding Epistle of a Database Administrator And a Explorer who must Find a Car in Ancient China,2006,1,,3,4.99,48,12.99,G,2007-09-10 17:46:03.905795,"{Trailers,""Deleted Scenes""}",'ace':1 'administr':9 'ancient':19 'astound':4 'car':17 'china':20 'databas':8 'epistl':5 'explor':12 'find':15 'goldfing':2 'must':14 +3,ADAPTATION HOLES,A Astounding Reflection of a Lumberjack And a Car who must Sink a Lumberjack in A Baloon Factory,2006,1,,7,2.99,50,18.99,NC-17,2007-09-10 17:46:03.905795,"{Trailers,""Deleted Scenes""}","'adapt':1 'astound':4 'baloon':19 'car':11 'factori':20 'hole':2 'lumberjack':8,16 'must':13 'reflect':5 'sink':14" +4,AFFAIR PREJUDICE,A Fanciful Documentary of a Frisbee And a Lumberjack who must Chase a Monkey in A Shark Tank,2006,1,,5,2.99,117,26.99,G,2007-09-10 17:46:03.905795,"{Commentaries,""Behind the Scenes""}",'affair':1 'chase':14 'documentari':5 'fanci':4 'frisbe':8 'lumberjack':11 'monkey':16 'must':13 'prejudic':2 'shark':19 'tank':20 +5,AFRICAN EGG,A Fast-Paced Documentary of a Pastry Chef And a Dentist who must Pursue a Forensic Psychologist in The Gulf of Mexico,2006,1,,6,2.99,130,22.99,G,2007-09-10 17:46:03.905795,"{""Deleted Scenes""}",'african':1 'chef':11 'dentist':14 'documentari':7 'egg':2 'fast':5 'fast-pac':4 'forens':19 'gulf':23 'mexico':25 'must':16 'pace':6 'pastri':10 'psychologist':20 'pursu':17 +6,AGENT TRUMAN,A Intrepid Panorama of a Robot And a Boy who must Escape a Sumo Wrestler in Ancient China,2006,1,,3,2.99,169,17.99,PG,2007-09-10 17:46:03.905795,"{""Deleted Scenes""}",'agent':1 'ancient':19 'boy':11 'china':20 'escap':14 'intrepid':4 'must':13 'panorama':5 'robot':8 'sumo':16 'truman':2 'wrestler':17 +7,AIRPLANE SIERRA,A Touching Saga of a Hunter And a Butler who must Discover a Butler in A Jet Boat,2006,1,,6,4.99,62,28.99,PG-13,2007-09-10 17:46:03.905795,"{Trailers,""Deleted Scenes""}","'airplan':1 'boat':20 'butler':11,16 'discov':14 'hunter':8 'jet':19 'must':13 'saga':5 'sierra':2 'touch':4" +8,AIRPORT POLLOCK,An Epic Tale of a Moose And a Girl who must Confront a Monkey in Ancient India,2006,1,,6,4.99,54,15.99,R,2007-09-10 17:46:03.905795,{Trailers},'airport':1 'ancient':18 'confront':14 'epic':4 'girl':11 'india':19 'monkey':16 'moos':8 'must':13 'pollock':2 'tale':5 +9,ALABAMA DEVIL,A Thoughtful Panorama of a Database Administrator And a Mad Scientist who must Outgun a Mad Scientist in A Jet Boat,2006,1,,3,2.99,114,21.99,PG-13,2007-09-10 17:46:03.905795,"{Trailers,""Deleted Scenes""}","'administr':9 'alabama':1 'boat':23 'databas':8 'devil':2 'jet':22 'mad':12,18 'must':15 'outgun':16 'panorama':5 'scientist':13,19 'thought':4" +10,ALADDIN CALENDAR,A Action-Packed Tale of a Man And a Lumberjack who must Reach a Feminist in Ancient China,2006,1,,6,4.99,63,24.99,NC-17,2007-09-10 17:46:03.905795,"{Trailers,""Deleted Scenes""}",'action':5 'action-pack':4 'aladdin':1 'ancient':20 'calendar':2 'china':21 'feminist':18 'lumberjack':13 'man':10 'must':15 'pack':6 'reach':16 'tale':7 diff --git a/backend/tests/integration_tests/data/sakila/inventory.csv b/backend/tests/integration_tests/data/sakila/inventory.csv index 513281a..d764cd5 100644 --- a/backend/tests/integration_tests/data/sakila/inventory.csv +++ b/backend/tests/integration_tests/data/sakila/inventory.csv @@ -1,3 +1,11 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:500bdc074b7456a4b702fdc50ce51fe5e5e887f4030468069c052b64ca60a430 -size 303 +inventory_id,film_id,store_id,last_update +1,1,1,2006-02-15 10:09:17 +2,1,1,2006-02-15 10:09:17 +3,1,1,2006-02-15 10:09:17 +4,1,1,2006-02-15 10:09:17 +5,1,2,2006-02-15 10:09:17 +6,1,2,2006-02-15 10:09:17 +7,1,2,2006-02-15 10:09:17 +8,1,2,2006-02-15 10:09:17 +9,2,2,2006-02-15 10:09:17 +10,2,2,2006-02-15 10:09:17 diff --git a/backend/tests/integration_tests/data/sakila/rental.csv b/backend/tests/integration_tests/data/sakila/rental.csv index 5ac1571..8af65ba 100644 --- a/backend/tests/integration_tests/data/sakila/rental.csv +++ b/backend/tests/integration_tests/data/sakila/rental.csv @@ -1,3 +1,1000 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:ab7de135256e69c0955f175d7631132f39a95717a8cac3a04681430d3ce31053 -size 74477 +rental_id,rental_date,inventory_id,customer_id,return_date,staff_id,last_update +1,2005-05-24 22:53:30,367,130,2005-05-26 22:04:30,1,2006-02-15 21:30:53 +2,2005-05-24 22:54:33,1525,459,2005-05-28 19:40:33,1,2006-02-16 02:30:53 +3,2005-05-24 23:03:39,1711,408,2005-06-01 22:12:39,1,2006-02-16 02:30:53 +4,2005-05-24 23:04:41,2452,333,2005-06-03 01:43:41,2,2006-02-16 02:30:53 +5,2005-05-24 23:05:21,2079,222,2005-06-02 04:33:21,1,2006-02-16 02:30:53 +6,2005-05-24 23:08:07,2792,549,2005-05-27 01:32:07,1,2006-02-16 02:30:53 +7,2005-05-24 23:11:53,3995,269,2005-05-29 20:34:53,2,2006-02-16 02:30:53 +8,2005-05-24 23:31:46,2346,239,2005-05-27 23:33:46,2,2006-02-16 02:30:53 +9,2005-05-25 00:00:40,2580,126,2005-05-28 00:22:40,1,2006-02-16 02:30:53 +10,2005-05-25 00:02:21,1824,399,2005-05-31 22:44:21,2,2006-02-16 02:30:53 +11,2005-05-25 00:09:02,4443,142,2005-06-02 20:56:02,2,2006-02-16 02:30:53 +12,2005-05-25 00:19:27,1584,261,2005-05-30 05:44:27,2,2006-02-16 02:30:53 +13,2005-05-25 00:22:55,2294,334,2005-05-30 04:28:55,1,2006-02-16 02:30:53 +14,2005-05-25 00:31:15,2701,446,2005-05-26 02:56:15,1,2006-02-16 02:30:53 +15,2005-05-25 00:39:22,3049,319,2005-06-03 03:30:22,1,2006-02-16 02:30:53 +16,2005-05-25 00:43:11,389,316,2005-05-26 04:42:11,2,2006-02-16 02:30:53 +17,2005-05-25 01:06:36,830,575,2005-05-27 00:43:36,1,2006-02-16 02:30:53 +18,2005-05-25 01:10:47,3376,19,2005-05-31 06:35:47,2,2006-02-16 02:30:53 +19,2005-05-25 01:17:24,1941,456,2005-05-31 06:00:24,1,2006-02-16 02:30:53 +20,2005-05-25 01:48:41,3517,185,2005-05-27 02:20:41,2,2006-02-16 02:30:53 +21,2005-05-25 01:59:46,146,388,2005-05-26 01:01:46,2,2006-02-16 02:30:53 +22,2005-05-25 02:19:23,727,509,2005-05-26 04:52:23,2,2006-02-16 02:30:53 +23,2005-05-25 02:40:21,4441,438,2005-05-29 06:34:21,1,2006-02-16 02:30:53 +24,2005-05-25 02:53:02,3273,350,2005-05-27 01:15:02,1,2006-02-16 02:30:53 +25,2005-05-25 03:21:20,3961,37,2005-05-27 21:25:20,2,2006-02-16 02:30:53 +26,2005-05-25 03:36:50,4371,371,2005-05-31 00:34:50,1,2006-02-16 02:30:53 +27,2005-05-25 03:41:50,1225,301,2005-05-30 01:13:50,2,2006-02-16 02:30:53 +28,2005-05-25 03:42:37,4068,232,2005-05-26 09:26:37,2,2006-02-16 02:30:53 +29,2005-05-25 03:47:12,611,44,2005-05-30 00:31:12,2,2006-02-16 02:30:53 +30,2005-05-25 04:01:32,3744,430,2005-05-30 03:12:32,1,2006-02-16 02:30:53 +31,2005-05-25 04:05:17,4482,369,2005-05-30 07:15:17,1,2006-02-16 02:30:53 +32,2005-05-25 04:06:21,3832,230,2005-05-25 23:55:21,1,2006-02-16 02:30:53 +33,2005-05-25 04:18:51,1681,272,2005-05-27 03:58:51,1,2006-02-16 02:30:53 +34,2005-05-25 04:19:28,2613,597,2005-05-29 00:10:28,2,2006-02-16 02:30:53 +35,2005-05-25 04:24:36,1286,484,2005-05-27 07:02:36,2,2006-02-16 02:30:53 +36,2005-05-25 04:36:26,1308,88,2005-05-29 00:31:26,1,2006-02-16 02:30:53 +37,2005-05-25 04:44:31,403,535,2005-05-29 01:03:31,1,2006-02-16 02:30:53 +38,2005-05-25 04:47:44,2540,302,2005-06-01 00:58:44,1,2006-02-16 02:30:53 +39,2005-05-25 04:51:46,4466,207,2005-05-31 03:14:46,2,2006-02-16 02:30:53 +40,2005-05-25 05:09:04,2638,413,2005-05-27 23:12:04,1,2006-02-16 02:30:53 +41,2005-05-25 05:12:29,1761,174,2005-06-02 00:28:29,1,2006-02-16 02:30:53 +42,2005-05-25 05:24:58,380,523,2005-05-31 02:47:58,2,2006-02-16 02:30:53 +43,2005-05-25 05:39:25,2578,532,2005-05-26 06:54:25,2,2006-02-16 02:30:53 +44,2005-05-25 05:53:23,3098,207,2005-05-29 10:56:23,2,2006-02-16 02:30:53 +45,2005-05-25 05:59:39,1853,436,2005-06-02 09:56:39,2,2006-02-16 02:30:53 +46,2005-05-25 06:04:08,3318,7,2005-06-02 08:18:08,2,2006-02-16 02:30:53 +47,2005-05-25 06:05:20,2211,35,2005-05-30 03:04:20,1,2006-02-16 02:30:53 +48,2005-05-25 06:20:46,1780,282,2005-06-02 05:42:46,1,2006-02-16 02:30:53 +49,2005-05-25 06:39:35,2965,498,2005-05-30 10:12:35,2,2006-02-16 02:30:53 +50,2005-05-25 06:44:53,1983,18,2005-05-28 11:28:53,2,2006-02-16 02:30:53 +51,2005-05-25 06:49:10,1257,256,2005-05-26 06:42:10,1,2006-02-16 02:30:53 +52,2005-05-25 06:51:29,4017,507,2005-05-31 01:27:29,2,2006-02-16 02:30:53 +53,2005-05-25 07:19:16,1255,569,2005-05-27 05:19:16,2,2006-02-16 02:30:53 +54,2005-05-25 07:23:25,2787,291,2005-06-01 05:05:25,2,2006-02-16 02:30:53 +55,2005-05-25 08:26:13,1139,131,2005-05-30 10:57:13,1,2006-02-16 02:30:53 +56,2005-05-25 08:28:11,1352,511,2005-05-26 14:21:11,1,2006-02-16 02:30:53 +57,2005-05-25 08:43:32,3938,6,2005-05-29 06:42:32,2,2006-02-16 02:30:53 +58,2005-05-25 08:53:14,3050,323,2005-05-28 14:40:14,1,2006-02-16 02:30:53 +59,2005-05-25 08:56:42,2884,408,2005-06-01 09:52:42,1,2006-02-16 02:30:53 +60,2005-05-25 08:58:25,330,470,2005-05-30 14:14:25,1,2006-02-16 02:30:53 +61,2005-05-25 09:01:57,4210,250,2005-06-02 07:22:57,2,2006-02-16 02:30:53 +62,2005-05-25 09:18:52,261,419,2005-05-30 10:55:52,1,2006-02-16 02:30:53 +63,2005-05-25 09:19:16,4008,383,2005-05-27 04:24:16,1,2006-02-16 02:30:53 +64,2005-05-25 09:21:29,79,368,2005-06-03 11:31:29,1,2006-02-16 02:30:53 +65,2005-05-25 09:32:03,3552,346,2005-05-29 14:21:03,1,2006-02-16 02:30:53 +66,2005-05-25 09:35:12,1162,86,2005-05-29 04:16:12,2,2006-02-16 02:30:53 +67,2005-05-25 09:41:01,239,119,2005-05-27 13:46:01,2,2006-02-16 02:30:53 +68,2005-05-25 09:47:31,4029,120,2005-05-31 10:20:31,2,2006-02-16 02:30:53 +69,2005-05-25 10:10:14,3207,305,2005-05-27 14:02:14,2,2006-02-16 02:30:53 +70,2005-05-25 10:15:23,2168,73,2005-05-27 05:56:23,2,2006-02-16 02:30:53 +71,2005-05-25 10:26:39,2408,100,2005-05-28 04:59:39,1,2006-02-16 02:30:53 +72,2005-05-25 10:52:13,2260,48,2005-05-28 05:52:13,2,2006-02-16 02:30:53 +73,2005-05-25 11:00:07,517,391,2005-06-01 13:56:07,2,2006-02-16 02:30:53 +74,2005-05-25 11:09:48,1744,265,2005-05-26 12:23:48,2,2006-02-16 02:30:53 +75,2005-05-25 11:13:34,3393,510,2005-06-03 12:58:34,1,2006-02-16 02:30:53 +76,2005-05-25 11:30:37,3021,1,2005-06-03 12:00:37,2,2006-02-16 02:30:53 +77,2005-05-25 11:31:59,1303,451,2005-05-26 16:53:59,2,2006-02-16 02:30:53 +78,2005-05-25 11:35:18,4067,135,2005-05-31 12:48:18,2,2006-02-16 02:30:53 +79,2005-05-25 12:11:07,3299,245,2005-06-03 10:54:07,2,2006-02-16 02:30:53 +80,2005-05-25 12:12:07,2478,314,2005-05-31 17:46:07,2,2006-02-16 02:30:53 +81,2005-05-25 12:15:19,2610,286,2005-06-02 14:08:19,2,2006-02-16 02:30:53 +82,2005-05-25 12:17:46,1388,427,2005-06-01 10:48:46,1,2006-02-16 02:30:53 +83,2005-05-25 12:30:15,466,131,2005-05-27 15:40:15,1,2006-02-16 02:30:53 +84,2005-05-25 12:36:30,1829,492,2005-05-29 18:33:30,1,2006-02-16 02:30:53 +85,2005-05-25 13:05:34,470,414,2005-05-29 16:53:34,1,2006-02-16 02:30:53 +86,2005-05-25 13:36:12,2275,266,2005-05-30 14:53:12,1,2006-02-16 02:30:53 +87,2005-05-25 13:52:43,1586,331,2005-05-29 11:12:43,2,2006-02-16 02:30:53 +88,2005-05-25 14:13:54,2221,53,2005-05-29 09:32:54,2,2006-02-16 02:30:53 +89,2005-05-25 14:28:29,2181,499,2005-05-29 14:33:29,1,2006-02-16 02:30:53 +90,2005-05-25 14:31:25,2984,25,2005-06-01 10:07:25,1,2006-02-16 02:30:53 +91,2005-05-25 14:57:22,139,267,2005-06-01 18:32:22,1,2006-02-16 02:30:53 +92,2005-05-25 15:38:46,775,302,2005-05-31 13:40:46,2,2006-02-16 02:30:53 +93,2005-05-25 15:54:16,4360,288,2005-06-03 20:18:16,1,2006-02-16 02:30:53 +94,2005-05-25 16:03:42,1675,197,2005-05-30 14:23:42,1,2006-02-16 02:30:53 +95,2005-05-25 16:12:52,178,400,2005-06-02 18:55:52,2,2006-02-16 02:30:53 +96,2005-05-25 16:32:19,3418,49,2005-05-30 10:47:19,2,2006-02-16 02:30:53 +97,2005-05-25 16:34:24,1283,263,2005-05-28 12:13:24,2,2006-02-16 02:30:53 +98,2005-05-25 16:48:24,2970,269,2005-05-27 11:29:24,2,2006-02-16 02:30:53 +99,2005-05-25 16:50:20,535,44,2005-05-28 18:52:20,1,2006-02-16 02:30:53 +100,2005-05-25 16:50:28,2599,208,2005-06-02 22:11:28,1,2006-02-16 02:30:53 +101,2005-05-25 17:17:04,617,468,2005-05-31 19:47:04,1,2006-02-16 02:30:53 +102,2005-05-25 17:22:10,373,343,2005-05-31 19:47:10,1,2006-02-16 02:30:53 +103,2005-05-25 17:30:42,3343,384,2005-06-03 22:36:42,1,2006-02-16 02:30:53 +104,2005-05-25 17:46:33,4281,310,2005-05-27 15:20:33,1,2006-02-16 02:30:53 +105,2005-05-25 17:54:12,794,108,2005-05-30 12:03:12,2,2006-02-16 02:30:53 +106,2005-05-25 18:18:19,3627,196,2005-06-04 00:01:19,2,2006-02-16 02:30:53 +107,2005-05-25 18:28:09,2833,317,2005-06-03 22:46:09,2,2006-02-16 02:30:53 +108,2005-05-25 18:30:05,3289,242,2005-05-30 19:40:05,1,2006-02-16 02:30:53 +109,2005-05-25 18:40:20,1044,503,2005-05-29 20:39:20,2,2006-02-16 02:30:53 +110,2005-05-25 18:43:49,4108,19,2005-06-03 18:13:49,2,2006-02-16 02:30:53 +111,2005-05-25 18:45:19,3725,227,2005-05-28 17:18:19,1,2006-02-16 02:30:53 +112,2005-05-25 18:57:24,2153,500,2005-06-02 20:44:24,1,2006-02-16 02:30:53 +113,2005-05-25 19:07:40,2963,93,2005-05-27 22:16:40,2,2006-02-16 02:30:53 +114,2005-05-25 19:12:42,4502,506,2005-06-01 23:10:42,1,2006-02-16 02:30:53 +115,2005-05-25 19:13:25,749,455,2005-05-29 20:17:25,1,2006-02-16 02:30:53 +116,2005-05-25 19:27:51,4453,18,2005-05-26 16:23:51,1,2006-02-16 02:30:53 +117,2005-05-25 19:30:46,4278,7,2005-05-31 23:59:46,2,2006-02-16 02:30:53 +118,2005-05-25 19:31:18,872,524,2005-05-31 15:00:18,1,2006-02-16 02:30:53 +119,2005-05-25 19:37:02,1359,51,2005-05-29 23:51:02,2,2006-02-16 02:30:53 +120,2005-05-25 19:37:47,37,365,2005-06-01 23:29:47,2,2006-02-16 02:30:53 +121,2005-05-25 19:41:29,1053,405,2005-05-29 21:31:29,1,2006-02-16 02:30:53 +122,2005-05-25 19:46:21,2908,273,2005-06-02 19:07:21,1,2006-02-16 02:30:53 +123,2005-05-25 20:26:42,1795,43,2005-05-26 19:41:42,1,2006-02-16 02:30:53 +124,2005-05-25 20:46:11,212,246,2005-05-30 00:47:11,2,2006-02-16 02:30:53 +125,2005-05-25 20:48:50,952,368,2005-06-02 21:39:50,1,2006-02-16 02:30:53 +126,2005-05-25 21:07:59,2047,439,2005-05-28 18:51:59,1,2006-02-16 02:30:53 +127,2005-05-25 21:10:40,2026,94,2005-06-02 21:38:40,1,2006-02-16 02:30:53 +128,2005-05-25 21:19:53,4322,40,2005-05-29 23:34:53,1,2006-02-16 02:30:53 +129,2005-05-25 21:20:03,4154,23,2005-06-04 01:25:03,2,2006-02-16 02:30:53 +130,2005-05-25 21:21:56,3990,56,2005-05-30 22:41:56,2,2006-02-16 02:30:53 +131,2005-05-25 21:42:46,815,325,2005-05-30 23:25:46,2,2006-02-16 02:30:53 +132,2005-05-25 21:46:54,3367,479,2005-05-31 21:02:54,1,2006-02-16 02:30:53 +133,2005-05-25 21:48:30,399,237,2005-05-30 00:26:30,2,2006-02-16 02:30:53 +134,2005-05-25 21:48:41,2272,222,2005-06-02 18:28:41,1,2006-02-16 02:30:53 +135,2005-05-25 21:58:58,103,304,2005-06-03 17:50:58,1,2006-02-16 02:30:53 +136,2005-05-25 22:02:30,2296,504,2005-05-31 18:06:30,1,2006-02-16 02:30:53 +137,2005-05-25 22:25:18,2591,560,2005-06-01 02:30:18,2,2006-02-16 02:30:53 +138,2005-05-25 22:48:22,4134,586,2005-05-29 20:21:22,2,2006-02-16 02:30:53 +139,2005-05-25 23:00:21,327,257,2005-05-29 17:12:21,1,2006-02-16 02:30:53 +140,2005-05-25 23:34:22,655,354,2005-05-27 01:10:22,1,2006-02-16 02:30:53 +141,2005-05-25 23:34:53,811,89,2005-06-02 01:57:53,1,2006-02-16 02:30:53 +142,2005-05-25 23:43:47,4407,472,2005-05-29 00:46:47,2,2006-02-16 02:30:53 +143,2005-05-25 23:45:52,847,297,2005-05-27 21:41:52,2,2006-02-16 02:30:53 +144,2005-05-25 23:49:56,1689,357,2005-06-01 21:41:56,2,2006-02-16 02:30:53 +145,2005-05-25 23:59:03,3905,82,2005-05-31 02:56:03,1,2006-02-16 02:30:53 +146,2005-05-26 00:07:11,1431,433,2005-06-04 00:20:11,2,2006-02-16 02:30:53 +147,2005-05-26 00:17:50,633,274,2005-05-29 23:21:50,2,2006-02-16 02:30:53 +148,2005-05-26 00:25:23,4252,142,2005-06-01 19:29:23,2,2006-02-16 02:30:53 +149,2005-05-26 00:28:05,1084,319,2005-06-02 21:30:05,2,2006-02-16 02:30:53 +150,2005-05-26 00:28:39,909,429,2005-06-01 02:10:39,2,2006-02-16 02:30:53 +151,2005-05-26 00:37:28,2942,14,2005-05-30 06:28:28,1,2006-02-16 02:30:53 +152,2005-05-26 00:41:10,2622,57,2005-06-03 06:05:10,1,2006-02-16 02:30:53 +153,2005-05-26 00:47:47,3888,348,2005-05-27 21:28:47,1,2006-02-16 02:30:53 +154,2005-05-26 00:55:56,1354,185,2005-05-29 23:18:56,2,2006-02-16 02:30:53 +155,2005-05-26 01:15:05,288,551,2005-06-01 00:03:05,1,2006-02-16 02:30:53 +156,2005-05-26 01:19:05,3193,462,2005-05-27 23:43:05,1,2006-02-16 02:30:53 +157,2005-05-26 01:25:21,887,344,2005-05-26 21:17:21,2,2006-02-16 02:30:53 +158,2005-05-26 01:27:11,2395,354,2005-06-03 00:30:11,2,2006-02-16 02:30:53 +159,2005-05-26 01:34:28,3453,505,2005-05-29 04:00:28,1,2006-02-16 02:30:53 +160,2005-05-26 01:46:20,1885,290,2005-06-01 05:45:20,1,2006-02-16 02:30:53 +161,2005-05-26 01:51:48,2941,182,2005-05-27 05:42:48,1,2006-02-16 02:30:53 +162,2005-05-26 02:02:05,1229,296,2005-05-27 03:38:05,2,2006-02-16 02:30:53 +163,2005-05-26 02:26:23,2306,104,2005-06-04 06:36:23,1,2006-02-16 02:30:53 +164,2005-05-26 02:26:49,1070,151,2005-05-28 00:32:49,1,2006-02-16 02:30:53 +165,2005-05-26 02:28:36,2735,33,2005-06-02 03:21:36,1,2006-02-16 02:30:53 +166,2005-05-26 02:49:11,3894,322,2005-05-31 01:28:11,1,2006-02-16 02:30:53 +167,2005-05-26 02:50:31,865,401,2005-05-27 03:07:31,1,2006-02-16 02:30:53 +168,2005-05-26 03:07:43,2714,469,2005-06-02 02:09:43,2,2006-02-16 02:30:53 +169,2005-05-26 03:09:30,1758,381,2005-05-27 01:37:30,2,2006-02-16 02:30:53 +170,2005-05-26 03:11:12,3688,107,2005-06-02 03:53:12,1,2006-02-16 02:30:53 +171,2005-05-26 03:14:15,4483,400,2005-06-03 00:24:15,2,2006-02-16 02:30:53 +172,2005-05-26 03:17:42,2873,176,2005-05-29 04:11:42,2,2006-02-16 02:30:53 +173,2005-05-26 03:42:10,3596,533,2005-05-28 01:37:10,2,2006-02-16 02:30:53 +174,2005-05-26 03:44:10,3954,552,2005-05-28 07:13:10,2,2006-02-16 02:30:53 +175,2005-05-26 03:46:26,4346,47,2005-06-03 06:01:26,2,2006-02-16 02:30:53 +176,2005-05-26 03:47:39,851,250,2005-06-01 02:36:39,2,2006-02-16 02:30:53 +177,2005-05-26 04:14:29,3545,548,2005-06-01 08:16:29,2,2006-02-16 02:30:53 +178,2005-05-26 04:21:46,1489,196,2005-06-04 07:09:46,2,2006-02-16 02:30:53 +179,2005-05-26 04:26:06,2575,19,2005-06-03 10:06:06,1,2006-02-16 02:30:53 +180,2005-05-26 04:46:23,2752,75,2005-06-01 09:58:23,1,2006-02-16 02:30:53 +181,2005-05-26 04:47:06,2417,587,2005-05-29 06:34:06,2,2006-02-16 02:30:53 +182,2005-05-26 04:49:17,4396,237,2005-06-01 05:43:17,2,2006-02-16 02:30:53 +183,2005-05-26 05:01:18,2877,254,2005-06-01 09:04:18,1,2006-02-16 02:30:53 +184,2005-05-26 05:29:49,1970,556,2005-05-28 10:10:49,1,2006-02-16 02:30:53 +185,2005-05-26 05:30:03,2598,125,2005-06-02 09:48:03,2,2006-02-16 02:30:53 +186,2005-05-26 05:32:52,1799,468,2005-06-03 07:19:52,2,2006-02-16 02:30:53 +187,2005-05-26 05:42:37,4004,515,2005-06-04 00:38:37,1,2006-02-16 02:30:53 +188,2005-05-26 05:47:12,3342,243,2005-05-26 23:48:12,1,2006-02-16 02:30:53 +189,2005-05-26 06:01:41,984,247,2005-05-27 06:11:41,1,2006-02-16 02:30:53 +190,2005-05-26 06:11:28,3962,533,2005-06-01 09:44:28,1,2006-02-16 02:30:53 +191,2005-05-26 06:14:06,4365,412,2005-05-28 05:33:06,1,2006-02-16 02:30:53 +192,2005-05-26 06:20:37,1897,437,2005-06-02 10:57:37,1,2006-02-16 02:30:53 +193,2005-05-26 06:41:48,3900,270,2005-05-30 06:21:48,2,2006-02-16 02:30:53 +194,2005-05-26 06:52:33,1337,29,2005-05-30 04:08:33,2,2006-02-16 02:30:53 +195,2005-05-26 06:52:36,506,564,2005-05-31 02:47:36,2,2006-02-16 02:30:53 +196,2005-05-26 06:55:58,190,184,2005-05-27 10:54:58,1,2006-02-16 02:30:53 +197,2005-05-26 06:59:21,4212,546,2005-06-03 05:04:21,2,2006-02-16 02:30:53 +198,2005-05-26 07:03:49,1789,54,2005-06-04 11:45:49,1,2006-02-16 02:30:53 +199,2005-05-26 07:11:58,2135,71,2005-05-28 09:06:58,1,2006-02-16 02:30:53 +200,2005-05-26 07:12:21,3926,321,2005-05-31 12:07:21,1,2006-02-16 02:30:53 +201,2005-05-26 07:13:45,776,444,2005-06-04 02:02:45,2,2006-02-16 02:30:53 +202,2005-05-26 07:27:36,674,20,2005-06-02 03:52:36,1,2006-02-16 02:30:53 +203,2005-05-26 07:27:57,3374,109,2005-06-03 12:52:57,1,2006-02-16 02:30:53 +204,2005-05-26 07:30:37,1842,528,2005-05-30 08:11:37,1,2006-02-16 02:30:53 +205,2005-05-26 07:59:37,303,114,2005-05-29 09:43:37,2,2006-02-16 02:30:53 +206,2005-05-26 08:01:54,1717,345,2005-05-27 06:26:54,1,2006-02-16 02:30:53 +207,2005-05-26 08:04:38,102,47,2005-05-27 09:32:38,2,2006-02-16 02:30:53 +208,2005-05-26 08:10:22,3669,274,2005-05-27 03:55:22,1,2006-02-16 02:30:53 +209,2005-05-26 08:14:01,729,379,2005-05-27 09:00:01,1,2006-02-16 02:30:53 +210,2005-05-26 08:14:15,1801,391,2005-05-27 12:12:15,2,2006-02-16 02:30:53 +211,2005-05-26 08:33:10,4005,170,2005-05-28 14:09:10,1,2006-02-16 02:30:53 +212,2005-05-26 08:34:41,764,59,2005-05-30 12:46:41,2,2006-02-16 02:30:53 +213,2005-05-26 08:44:08,1505,394,2005-05-31 12:33:08,2,2006-02-16 02:30:53 +214,2005-05-26 08:48:49,1453,98,2005-05-31 04:06:49,2,2006-02-16 02:30:53 +215,2005-05-26 09:02:47,679,197,2005-05-28 09:45:47,2,2006-02-16 02:30:53 +216,2005-05-26 09:17:43,1398,91,2005-06-03 08:21:43,1,2006-02-16 02:30:53 +217,2005-05-26 09:24:26,4395,121,2005-05-31 03:24:26,2,2006-02-16 02:30:53 +218,2005-05-26 09:27:09,2291,309,2005-06-04 11:53:09,2,2006-02-16 02:30:53 +219,2005-05-26 09:41:45,3074,489,2005-05-28 04:40:45,1,2006-02-16 02:30:53 +220,2005-05-26 10:06:49,1259,542,2005-06-01 07:43:49,1,2006-02-16 02:30:53 +221,2005-05-26 10:14:09,3578,143,2005-05-29 05:57:09,1,2006-02-16 02:30:53 +222,2005-05-26 10:14:38,2745,83,2005-05-31 08:36:38,2,2006-02-16 02:30:53 +223,2005-05-26 10:15:23,3121,460,2005-05-30 11:43:23,1,2006-02-16 02:30:53 +224,2005-05-26 10:18:27,4285,318,2005-06-04 06:59:27,1,2006-02-16 02:30:53 +225,2005-05-26 10:27:50,651,467,2005-06-01 07:01:50,2,2006-02-16 02:30:53 +226,2005-05-26 10:44:04,4181,221,2005-05-31 13:26:04,2,2006-02-16 02:30:53 +227,2005-05-26 10:51:46,214,301,2005-05-30 07:24:46,1,2006-02-16 02:30:53 +228,2005-05-26 10:54:28,511,571,2005-06-04 09:39:28,1,2006-02-16 02:30:53 +229,2005-05-26 11:19:20,1131,312,2005-05-31 11:56:20,2,2006-02-16 02:30:53 +230,2005-05-26 11:31:50,1085,58,2005-05-30 15:22:50,1,2006-02-16 02:30:53 +231,2005-05-26 11:31:59,4032,365,2005-05-27 07:27:59,1,2006-02-16 02:30:53 +232,2005-05-26 11:38:05,2945,256,2005-05-27 08:42:05,2,2006-02-16 02:30:53 +233,2005-05-26 11:43:44,715,531,2005-05-28 17:28:44,2,2006-02-16 02:30:53 +234,2005-05-26 11:47:20,1321,566,2005-06-03 10:39:20,2,2006-02-16 02:30:53 +235,2005-05-26 11:51:09,3537,119,2005-06-04 09:36:09,1,2006-02-16 02:30:53 +236,2005-05-26 11:53:49,1265,446,2005-05-28 13:55:49,1,2006-02-16 02:30:53 +237,2005-05-26 12:15:13,241,536,2005-05-29 18:10:13,1,2006-02-16 02:30:53 +238,2005-05-26 12:30:22,503,211,2005-05-27 06:49:22,1,2006-02-16 02:30:53 +239,2005-05-26 12:30:26,131,49,2005-06-01 13:26:26,2,2006-02-16 02:30:53 +240,2005-05-26 12:40:23,3420,103,2005-06-04 07:22:23,1,2006-02-16 02:30:53 +241,2005-05-26 12:49:01,4438,245,2005-05-28 11:43:01,2,2006-02-16 02:30:53 +242,2005-05-26 13:05:08,2095,214,2005-06-02 15:26:08,1,2006-02-16 02:30:53 +243,2005-05-26 13:06:05,1721,543,2005-06-03 17:28:05,2,2006-02-16 02:30:53 +244,2005-05-26 13:40:40,1041,257,2005-05-31 11:58:40,1,2006-02-16 02:30:53 +245,2005-05-26 13:46:59,3045,158,2005-05-27 09:58:59,2,2006-02-16 02:30:53 +246,2005-05-26 13:57:07,2829,240,2005-05-29 10:12:07,2,2006-02-16 02:30:53 +247,2005-05-26 14:01:05,4095,102,2005-05-28 13:38:05,2,2006-02-16 02:30:53 +248,2005-05-26 14:07:58,1913,545,2005-05-31 14:03:58,2,2006-02-16 02:30:53 +249,2005-05-26 14:19:09,2428,472,2005-05-28 17:47:09,2,2006-02-16 02:30:53 +250,2005-05-26 14:30:24,368,539,2005-05-27 08:50:24,1,2006-02-16 02:30:53 +251,2005-05-26 14:35:40,4352,204,2005-05-29 17:17:40,1,2006-02-16 02:30:53 +252,2005-05-26 14:39:53,1203,187,2005-06-02 14:48:53,1,2006-02-16 02:30:53 +253,2005-05-26 14:43:14,2969,416,2005-05-27 12:21:14,1,2006-02-16 02:30:53 +254,2005-05-26 14:43:48,1835,390,2005-05-31 09:19:48,2,2006-02-16 02:30:53 +255,2005-05-26 14:52:15,3264,114,2005-05-27 12:45:15,1,2006-02-16 02:30:53 +256,2005-05-26 15:20:58,3194,436,2005-05-31 15:58:58,1,2006-02-16 02:30:53 +257,2005-05-26 15:27:05,2570,373,2005-05-29 16:25:05,2,2006-02-16 02:30:53 +258,2005-05-26 15:28:14,3534,502,2005-05-30 18:38:14,2,2006-02-16 02:30:53 +259,2005-05-26 15:32:46,30,482,2005-06-04 15:27:46,2,2006-02-16 02:30:53 +260,2005-05-26 15:42:20,435,21,2005-05-31 13:21:20,2,2006-02-16 02:30:53 +261,2005-05-26 15:44:23,1369,414,2005-06-02 09:47:23,2,2006-02-16 02:30:53 +262,2005-05-26 15:46:56,4261,236,2005-05-28 15:49:56,2,2006-02-16 02:30:53 +263,2005-05-26 15:47:40,1160,449,2005-05-30 10:07:40,2,2006-02-16 02:30:53 +264,2005-05-26 16:00:49,2069,251,2005-05-27 10:12:49,2,2006-02-16 02:30:53 +265,2005-05-26 16:07:38,2276,303,2005-06-01 14:20:38,1,2006-02-16 02:30:53 +266,2005-05-26 16:08:05,3303,263,2005-05-27 10:55:05,2,2006-02-16 02:30:53 +267,2005-05-26 16:16:21,1206,417,2005-05-30 16:53:21,2,2006-02-16 02:30:53 +268,2005-05-26 16:19:08,1714,75,2005-05-27 14:35:08,1,2006-02-16 02:30:53 +269,2005-05-26 16:19:46,3501,322,2005-05-27 15:59:46,2,2006-02-16 02:30:53 +270,2005-05-26 16:20:56,207,200,2005-06-03 12:40:56,2,2006-02-16 02:30:53 +271,2005-05-26 16:22:01,2388,92,2005-06-03 17:30:01,2,2006-02-16 02:30:53 +272,2005-05-26 16:27:11,971,71,2005-06-03 13:10:11,2,2006-02-16 02:30:53 +273,2005-05-26 16:29:36,1590,193,2005-05-29 18:49:36,2,2006-02-16 02:30:53 +274,2005-05-26 16:48:51,656,311,2005-06-03 18:17:51,1,2006-02-16 02:30:53 +275,2005-05-26 17:09:53,1718,133,2005-06-04 22:35:53,1,2006-02-16 02:30:53 +276,2005-05-26 17:16:07,1221,58,2005-06-03 12:59:07,1,2006-02-16 02:30:53 +277,2005-05-26 17:32:11,1409,45,2005-05-28 22:54:11,1,2006-02-16 02:30:53 +278,2005-05-26 17:40:58,182,214,2005-06-02 16:43:58,2,2006-02-16 02:30:53 +279,2005-05-26 18:02:50,661,384,2005-06-03 18:48:50,2,2006-02-16 02:30:53 +280,2005-05-26 18:36:58,1896,167,2005-05-27 23:42:58,1,2006-02-16 02:30:53 +281,2005-05-26 18:49:35,1208,582,2005-05-27 18:11:35,2,2006-02-16 02:30:53 +282,2005-05-26 18:56:26,4486,282,2005-06-01 16:32:26,2,2006-02-16 02:30:53 +283,2005-05-26 19:05:05,3530,242,2005-05-31 19:19:05,1,2006-02-16 02:30:53 +284,2005-05-26 19:21:44,350,359,2005-06-04 14:18:44,2,2006-02-16 02:30:53 +285,2005-05-26 19:41:40,2486,162,2005-05-31 16:58:40,2,2006-02-16 02:30:53 +286,2005-05-26 19:44:51,314,371,2005-06-04 18:00:51,2,2006-02-16 02:30:53 +287,2005-05-26 19:44:54,3631,17,2005-06-02 01:10:54,1,2006-02-16 02:30:53 +288,2005-05-26 19:47:49,3546,82,2005-06-03 20:53:49,2,2006-02-16 02:30:53 +289,2005-05-26 20:01:09,2449,81,2005-05-28 15:09:09,1,2006-02-16 02:30:53 +290,2005-05-26 20:08:33,2776,429,2005-05-30 00:32:33,1,2006-02-16 02:30:53 +291,2005-05-26 20:20:47,485,577,2005-06-03 02:06:47,2,2006-02-16 02:30:53 +292,2005-05-26 20:22:12,4264,515,2005-06-05 00:58:12,1,2006-02-16 02:30:53 +293,2005-05-26 20:27:02,1828,158,2005-06-03 16:45:02,2,2006-02-16 02:30:53 +294,2005-05-26 20:29:57,2751,369,2005-05-28 17:20:57,1,2006-02-16 02:30:53 +295,2005-05-26 20:33:20,4030,65,2005-05-27 18:23:20,2,2006-02-16 02:30:53 +296,2005-05-26 20:35:19,3878,468,2005-06-04 02:31:19,2,2006-02-16 02:30:53 +297,2005-05-26 20:48:48,1594,48,2005-05-27 19:52:48,2,2006-02-16 02:30:53 +298,2005-05-26 20:52:26,1083,460,2005-05-29 22:08:26,2,2006-02-16 02:30:53 +299,2005-05-26 20:55:36,4376,448,2005-05-28 00:25:36,2,2006-02-16 02:30:53 +300,2005-05-26 20:57:00,249,47,2005-06-05 01:34:00,2,2006-02-16 02:30:53 +301,2005-05-26 21:06:14,3448,274,2005-06-01 01:54:14,2,2006-02-16 02:30:53 +302,2005-05-26 21:13:46,2921,387,2005-06-03 15:49:46,2,2006-02-16 02:30:53 +303,2005-05-26 21:16:52,1111,596,2005-05-27 23:41:52,2,2006-02-16 02:30:53 +304,2005-05-26 21:21:28,1701,534,2005-06-02 00:05:28,1,2006-02-16 02:30:53 +305,2005-05-26 21:22:07,2665,464,2005-06-02 22:33:07,2,2006-02-16 02:30:53 +306,2005-05-26 21:31:57,2781,547,2005-05-28 19:37:57,1,2006-02-16 02:30:53 +307,2005-05-26 21:48:13,1097,375,2005-06-04 22:24:13,1,2006-02-16 02:30:53 +308,2005-05-26 22:01:39,187,277,2005-06-04 20:24:39,2,2006-02-16 02:30:53 +309,2005-05-26 22:38:10,1946,251,2005-06-02 03:10:10,2,2006-02-16 02:30:53 +310,2005-05-26 22:41:07,593,409,2005-06-02 04:09:07,1,2006-02-16 02:30:53 +311,2005-05-26 22:51:37,2830,201,2005-06-01 00:02:37,1,2006-02-16 02:30:53 +312,2005-05-26 22:52:19,2008,143,2005-06-02 18:14:19,2,2006-02-16 02:30:53 +313,2005-05-26 22:56:19,4156,594,2005-05-29 01:29:19,2,2006-02-16 02:30:53 +314,2005-05-26 23:09:41,2851,203,2005-05-28 22:49:41,2,2006-02-16 02:30:53 +315,2005-05-26 23:12:55,2847,238,2005-05-29 23:33:55,1,2006-02-16 02:30:53 +316,2005-05-26 23:22:55,3828,249,2005-05-29 23:25:55,2,2006-02-16 02:30:53 +317,2005-05-26 23:23:56,26,391,2005-06-01 19:56:56,2,2006-02-16 02:30:53 +318,2005-05-26 23:37:39,2559,60,2005-06-03 04:31:39,2,2006-02-16 02:30:53 +319,2005-05-26 23:52:13,3024,77,2005-05-30 18:55:13,1,2006-02-16 02:30:53 +320,2005-05-27 00:09:24,1090,2,2005-05-28 04:30:24,2,2006-02-16 02:30:53 +322,2005-05-27 00:47:35,4556,496,2005-06-02 00:32:35,1,2006-02-16 02:30:53 +323,2005-05-27 00:49:27,2362,144,2005-05-30 03:12:27,1,2006-02-16 02:30:53 +324,2005-05-27 01:00:04,3364,292,2005-05-30 04:27:04,1,2006-02-16 02:30:53 +325,2005-05-27 01:09:55,2510,449,2005-05-31 07:01:55,2,2006-02-16 02:30:53 +326,2005-05-27 01:10:11,3979,432,2005-06-04 20:25:11,2,2006-02-16 02:30:53 +327,2005-05-27 01:18:57,2678,105,2005-06-04 04:06:57,1,2006-02-16 02:30:53 +328,2005-05-27 01:29:31,2524,451,2005-06-01 02:27:31,1,2006-02-16 02:30:53 +329,2005-05-27 01:57:14,2659,231,2005-05-31 04:19:14,2,2006-02-16 02:30:53 +330,2005-05-27 02:15:30,1536,248,2005-06-04 05:09:30,2,2006-02-16 02:30:53 +331,2005-05-27 02:22:26,1872,67,2005-06-05 00:25:26,1,2006-02-16 02:30:53 +332,2005-05-27 02:27:10,1529,299,2005-06-03 01:26:10,2,2006-02-16 02:30:53 +333,2005-05-27 02:52:21,4001,412,2005-06-01 00:55:21,2,2006-02-16 02:30:53 +334,2005-05-27 03:03:07,3973,194,2005-05-29 03:54:07,1,2006-02-16 02:30:53 +335,2005-05-27 03:07:10,1411,16,2005-06-05 00:15:10,2,2006-02-16 02:30:53 +336,2005-05-27 03:15:23,1811,275,2005-05-29 22:43:23,1,2006-02-16 02:30:53 +337,2005-05-27 03:22:30,751,19,2005-06-02 03:27:30,1,2006-02-16 02:30:53 +338,2005-05-27 03:42:52,2596,165,2005-06-01 05:23:52,2,2006-02-16 02:30:53 +339,2005-05-27 03:47:18,2410,516,2005-06-04 05:46:18,2,2006-02-16 02:30:53 +340,2005-05-27 03:55:25,946,209,2005-06-04 07:57:25,2,2006-02-16 02:30:53 +341,2005-05-27 04:01:42,4168,56,2005-06-05 08:51:42,1,2006-02-16 02:30:53 +342,2005-05-27 04:11:04,4019,539,2005-05-29 01:28:04,2,2006-02-16 02:30:53 +343,2005-05-27 04:13:41,3301,455,2005-05-28 08:34:41,1,2006-02-16 02:30:53 +344,2005-05-27 04:30:22,2327,236,2005-05-29 10:13:22,2,2006-02-16 02:30:53 +345,2005-05-27 04:32:25,1396,144,2005-05-31 09:50:25,1,2006-02-16 02:30:53 +346,2005-05-27 04:34:41,4319,14,2005-06-05 04:24:41,2,2006-02-16 02:30:53 +347,2005-05-27 04:40:33,1625,378,2005-05-28 09:56:33,2,2006-02-16 02:30:53 +348,2005-05-27 04:50:56,1825,473,2005-06-01 04:43:56,1,2006-02-16 02:30:53 +349,2005-05-27 04:53:11,2920,36,2005-05-28 06:33:11,2,2006-02-16 02:30:53 +350,2005-05-27 05:01:28,2756,9,2005-06-04 05:01:28,2,2006-02-16 02:30:53 +351,2005-05-27 05:39:03,3371,118,2005-06-01 11:10:03,1,2006-02-16 02:30:53 +352,2005-05-27 05:48:19,4369,157,2005-05-29 09:05:19,1,2006-02-16 02:30:53 +353,2005-05-27 06:03:39,3989,503,2005-06-03 04:39:39,2,2006-02-16 02:30:53 +354,2005-05-27 06:12:26,2058,452,2005-06-01 06:48:26,1,2006-02-16 02:30:53 +355,2005-05-27 06:15:33,141,446,2005-06-01 02:50:33,2,2006-02-16 02:30:53 +356,2005-05-27 06:32:30,2868,382,2005-05-30 06:24:30,2,2006-02-16 02:30:53 +357,2005-05-27 06:37:15,4417,198,2005-05-30 07:04:15,2,2006-02-16 02:30:53 +358,2005-05-27 06:43:59,1925,102,2005-05-29 11:28:59,2,2006-02-16 02:30:53 +359,2005-05-27 06:48:33,1156,152,2005-05-29 03:55:33,1,2006-02-16 02:30:53 +360,2005-05-27 06:51:14,3489,594,2005-06-03 01:58:14,1,2006-02-16 02:30:53 +361,2005-05-27 07:03:28,6,587,2005-05-31 08:01:28,1,2006-02-16 02:30:53 +362,2005-05-27 07:10:25,2324,147,2005-06-01 08:34:25,1,2006-02-16 02:30:53 +363,2005-05-27 07:14:00,4282,345,2005-05-28 12:22:00,2,2006-02-16 02:30:53 +364,2005-05-27 07:20:12,833,430,2005-05-31 10:44:12,2,2006-02-16 02:30:53 +365,2005-05-27 07:31:20,2887,167,2005-06-04 04:46:20,1,2006-02-16 02:30:53 +366,2005-05-27 07:33:54,360,134,2005-06-04 01:55:54,2,2006-02-16 02:30:53 +367,2005-05-27 07:37:02,3437,439,2005-05-30 05:43:02,2,2006-02-16 02:30:53 +368,2005-05-27 07:42:29,1247,361,2005-06-04 11:20:29,2,2006-02-16 02:30:53 +369,2005-05-27 07:46:49,944,508,2005-06-01 06:20:49,2,2006-02-16 02:30:53 +370,2005-05-27 07:49:43,3347,22,2005-06-05 06:39:43,2,2006-02-16 02:30:53 +371,2005-05-27 08:08:18,1235,295,2005-06-05 03:05:18,2,2006-02-16 02:30:53 +372,2005-05-27 08:13:58,4089,510,2005-06-04 03:50:58,2,2006-02-16 02:30:53 +373,2005-05-27 08:16:25,1649,464,2005-06-01 11:41:25,1,2006-02-16 02:30:53 +374,2005-05-27 08:26:30,4420,337,2005-06-05 07:13:30,1,2006-02-16 02:30:53 +375,2005-05-27 08:49:21,1815,306,2005-06-04 14:11:21,1,2006-02-16 02:30:53 +376,2005-05-27 08:58:15,3197,542,2005-06-02 04:48:15,1,2006-02-16 02:30:53 +377,2005-05-27 09:04:05,3012,170,2005-06-02 03:36:05,2,2006-02-16 02:30:53 +378,2005-05-27 09:23:22,2242,53,2005-05-29 15:20:22,1,2006-02-16 02:30:53 +379,2005-05-27 09:25:32,3462,584,2005-06-02 06:19:32,1,2006-02-16 02:30:53 +380,2005-05-27 09:34:39,1777,176,2005-06-04 11:45:39,1,2006-02-16 02:30:53 +381,2005-05-27 09:43:25,2748,371,2005-05-31 12:00:25,1,2006-02-16 02:30:53 +382,2005-05-27 10:12:00,4358,183,2005-05-31 15:03:00,1,2006-02-16 02:30:53 +383,2005-05-27 10:12:20,955,298,2005-06-03 10:37:20,1,2006-02-16 02:30:53 +384,2005-05-27 10:18:20,910,371,2005-06-02 09:21:20,2,2006-02-16 02:30:53 +385,2005-05-27 10:23:25,1565,213,2005-05-30 15:27:25,2,2006-02-16 02:30:53 +386,2005-05-27 10:26:31,1288,109,2005-05-30 08:32:31,1,2006-02-16 02:30:53 +387,2005-05-27 10:35:27,2684,506,2005-06-01 13:37:27,2,2006-02-16 02:30:53 +388,2005-05-27 10:37:27,434,28,2005-05-30 05:45:27,1,2006-02-16 02:30:53 +389,2005-05-27 10:45:41,691,500,2005-06-05 06:22:41,2,2006-02-16 02:30:53 +390,2005-05-27 11:02:26,3759,48,2005-06-02 16:09:26,2,2006-02-16 02:30:53 +391,2005-05-27 11:03:55,2193,197,2005-06-01 11:59:55,2,2006-02-16 02:30:53 +392,2005-05-27 11:14:42,263,359,2005-06-01 14:28:42,2,2006-02-16 02:30:53 +393,2005-05-27 11:18:25,145,251,2005-05-28 07:10:25,2,2006-02-16 02:30:53 +394,2005-05-27 11:26:11,1890,274,2005-06-03 16:44:11,2,2006-02-16 02:30:53 +395,2005-05-27 11:45:49,752,575,2005-05-31 13:42:49,1,2006-02-16 02:30:53 +396,2005-05-27 11:47:04,1020,112,2005-05-29 10:14:04,1,2006-02-16 02:30:53 +397,2005-05-27 12:29:02,4193,544,2005-05-28 17:36:02,2,2006-02-16 02:30:53 +398,2005-05-27 12:44:03,1686,422,2005-06-02 08:19:03,1,2006-02-16 02:30:53 +399,2005-05-27 12:48:38,553,204,2005-05-29 15:27:38,1,2006-02-16 02:30:53 +400,2005-05-27 12:51:44,258,249,2005-05-31 08:34:44,2,2006-02-16 02:30:53 +401,2005-05-27 12:57:55,2179,46,2005-05-29 17:55:55,2,2006-02-16 02:30:53 +402,2005-05-27 13:17:18,461,354,2005-05-30 08:53:18,2,2006-02-16 02:30:53 +403,2005-05-27 13:28:52,3983,424,2005-05-29 11:47:52,2,2006-02-16 02:30:53 +404,2005-05-27 13:31:51,1293,168,2005-05-30 16:58:51,1,2006-02-16 02:30:53 +405,2005-05-27 13:32:39,4090,272,2005-06-05 18:53:39,2,2006-02-16 02:30:53 +406,2005-05-27 13:46:46,2136,381,2005-05-30 12:43:46,1,2006-02-16 02:30:53 +407,2005-05-27 13:57:38,1077,44,2005-05-31 18:23:38,1,2006-02-16 02:30:53 +408,2005-05-27 13:57:39,1438,84,2005-05-28 11:57:39,1,2006-02-16 02:30:53 +409,2005-05-27 14:10:58,3652,220,2005-06-02 10:40:58,2,2006-02-16 02:30:53 +410,2005-05-27 14:11:22,4010,506,2005-06-02 20:06:22,2,2006-02-16 02:30:53 +411,2005-05-27 14:14:14,1434,388,2005-06-03 17:39:14,1,2006-02-16 02:30:53 +412,2005-05-27 14:17:23,1400,375,2005-05-29 15:07:23,2,2006-02-16 02:30:53 +413,2005-05-27 14:45:37,3516,307,2005-06-03 11:11:37,1,2006-02-16 02:30:53 +414,2005-05-27 14:48:20,1019,219,2005-05-31 14:39:20,2,2006-02-16 02:30:53 +415,2005-05-27 14:51:45,3698,304,2005-05-28 19:07:45,2,2006-02-16 02:30:53 +416,2005-05-27 15:02:10,2371,222,2005-05-29 10:34:10,2,2006-02-16 02:30:53 +417,2005-05-27 15:07:27,2253,475,2005-05-29 20:01:27,2,2006-02-16 02:30:53 +418,2005-05-27 15:13:17,3063,151,2005-06-04 12:05:17,2,2006-02-16 02:30:53 +419,2005-05-27 15:15:11,2514,77,2005-06-02 11:53:11,1,2006-02-16 02:30:53 +420,2005-05-27 15:19:38,619,93,2005-06-03 15:07:38,2,2006-02-16 02:30:53 +421,2005-05-27 15:30:13,2985,246,2005-06-04 13:19:13,2,2006-02-16 02:30:53 +422,2005-05-27 15:31:55,1152,150,2005-06-01 11:47:55,2,2006-02-16 02:30:53 +423,2005-05-27 15:32:57,1783,284,2005-06-02 19:03:57,1,2006-02-16 02:30:53 +424,2005-05-27 15:34:01,2815,35,2005-06-05 09:44:01,1,2006-02-16 02:30:53 +425,2005-05-27 15:51:30,1518,182,2005-06-03 16:52:30,2,2006-02-16 02:30:53 +426,2005-05-27 15:56:57,1103,522,2005-06-05 11:45:57,1,2006-02-16 02:30:53 +427,2005-05-27 16:10:04,1677,288,2005-06-05 13:22:04,2,2006-02-16 02:30:53 +428,2005-05-27 16:10:58,3349,161,2005-05-31 17:24:58,2,2006-02-16 02:30:53 +429,2005-05-27 16:21:26,129,498,2005-06-05 20:23:26,2,2006-02-16 02:30:53 +430,2005-05-27 16:22:10,1920,190,2005-06-05 13:10:10,1,2006-02-16 02:30:53 +431,2005-05-27 16:31:05,4507,334,2005-06-05 11:29:05,1,2006-02-16 02:30:53 +432,2005-05-27 16:40:29,1119,46,2005-05-29 16:20:29,1,2006-02-16 02:30:53 +433,2005-05-27 16:40:40,4364,574,2005-05-30 19:55:40,2,2006-02-16 02:30:53 +434,2005-05-27 16:54:27,3360,246,2005-06-04 22:26:27,1,2006-02-16 02:30:53 +435,2005-05-27 17:17:09,3328,3,2005-06-02 11:20:09,2,2006-02-16 02:30:53 +436,2005-05-27 17:21:04,4317,267,2005-05-30 21:26:04,2,2006-02-16 02:30:53 +437,2005-05-27 17:47:22,1800,525,2005-06-05 14:22:22,2,2006-02-16 02:30:53 +438,2005-05-27 17:52:34,4260,249,2005-06-05 22:23:34,2,2006-02-16 02:30:53 +439,2005-05-27 17:54:48,354,319,2005-06-02 23:01:48,2,2006-02-16 02:30:53 +440,2005-05-27 18:00:35,4452,314,2005-05-29 16:15:35,1,2006-02-16 02:30:53 +441,2005-05-27 18:11:05,1578,54,2005-05-30 22:45:05,1,2006-02-16 02:30:53 +442,2005-05-27 18:12:13,1457,403,2005-05-30 12:30:13,2,2006-02-16 02:30:53 +443,2005-05-27 18:35:20,2021,547,2005-06-04 18:58:20,1,2006-02-16 02:30:53 +444,2005-05-27 18:39:15,723,239,2005-06-01 15:56:15,1,2006-02-16 02:30:53 +445,2005-05-27 18:42:57,1757,293,2005-05-30 22:35:57,2,2006-02-16 02:30:53 +446,2005-05-27 18:48:41,1955,401,2005-06-03 16:42:41,2,2006-02-16 02:30:53 +447,2005-05-27 18:57:02,3890,133,2005-06-05 18:38:02,1,2006-02-16 02:30:53 +448,2005-05-27 19:03:08,2671,247,2005-06-03 20:28:08,2,2006-02-16 02:30:53 +449,2005-05-27 19:13:15,2469,172,2005-06-04 01:08:15,2,2006-02-16 02:30:53 +450,2005-05-27 19:18:54,1343,247,2005-06-05 23:52:54,1,2006-02-16 02:30:53 +451,2005-05-27 19:27:54,205,87,2005-05-29 01:07:54,2,2006-02-16 02:30:53 +452,2005-05-27 19:30:33,2993,127,2005-05-30 20:53:33,2,2006-02-16 02:30:53 +453,2005-05-27 19:31:16,4425,529,2005-05-29 23:06:16,1,2006-02-16 02:30:53 +454,2005-05-27 19:31:36,3499,575,2005-05-30 15:46:36,1,2006-02-16 02:30:53 +455,2005-05-27 19:43:29,3344,343,2005-06-04 23:40:29,2,2006-02-16 02:30:53 +456,2005-05-27 19:50:06,1699,92,2005-06-02 22:14:06,1,2006-02-16 02:30:53 +457,2005-05-27 19:52:29,2368,300,2005-06-02 17:17:29,2,2006-02-16 02:30:53 +458,2005-05-27 19:58:36,3350,565,2005-06-06 00:51:36,1,2006-02-16 02:30:53 +459,2005-05-27 20:00:04,597,468,2005-05-29 22:47:04,1,2006-02-16 02:30:53 +460,2005-05-27 20:02:03,4238,240,2005-05-28 16:14:03,1,2006-02-16 02:30:53 +461,2005-05-27 20:08:55,2077,447,2005-06-01 14:32:55,1,2006-02-16 02:30:53 +462,2005-05-27 20:10:36,2314,364,2005-06-03 21:12:36,2,2006-02-16 02:30:53 +463,2005-05-27 20:11:47,826,21,2005-06-04 21:18:47,1,2006-02-16 02:30:53 +464,2005-05-27 20:42:44,1313,193,2005-05-30 00:49:44,2,2006-02-16 02:30:53 +465,2005-05-27 20:44:36,20,261,2005-06-02 02:43:36,1,2006-02-16 02:30:53 +466,2005-05-27 20:57:07,1786,442,2005-05-29 15:52:07,1,2006-02-16 02:30:53 +467,2005-05-27 21:10:03,339,557,2005-06-01 16:08:03,1,2006-02-16 02:30:53 +468,2005-05-27 21:13:10,2656,101,2005-06-04 15:26:10,2,2006-02-16 02:30:53 +469,2005-05-27 21:14:26,4463,154,2005-06-05 21:51:26,1,2006-02-16 02:30:53 +470,2005-05-27 21:17:08,1613,504,2005-06-04 17:47:08,1,2006-02-16 02:30:53 +471,2005-05-27 21:32:42,2872,209,2005-05-31 00:39:42,2,2006-02-16 02:30:53 +472,2005-05-27 21:36:15,1338,528,2005-05-29 21:07:15,1,2006-02-16 02:30:53 +473,2005-05-27 21:36:34,802,105,2005-06-05 17:02:34,1,2006-02-16 02:30:53 +474,2005-05-27 22:11:56,1474,274,2005-05-31 19:07:56,1,2006-02-16 02:30:53 +475,2005-05-27 22:16:26,2520,159,2005-05-28 19:58:26,1,2006-02-16 02:30:53 +476,2005-05-27 22:31:36,2451,543,2005-06-03 19:12:36,1,2006-02-16 02:30:53 +477,2005-05-27 22:33:33,2437,161,2005-06-02 18:35:33,2,2006-02-16 02:30:53 +478,2005-05-27 22:38:20,424,557,2005-05-31 18:39:20,2,2006-02-16 02:30:53 +479,2005-05-27 22:39:10,2060,231,2005-06-05 22:46:10,2,2006-02-16 02:30:53 +480,2005-05-27 22:47:39,2108,220,2005-06-04 21:17:39,2,2006-02-16 02:30:53 +481,2005-05-27 22:49:27,72,445,2005-05-30 17:46:27,2,2006-02-16 02:30:53 +482,2005-05-27 22:53:02,4178,546,2005-06-01 22:53:02,2,2006-02-16 02:30:53 +483,2005-05-27 23:00:25,1510,32,2005-05-28 21:30:25,1,2006-02-16 02:30:53 +484,2005-05-27 23:26:45,3115,491,2005-05-29 21:16:45,2,2006-02-16 02:30:53 +485,2005-05-27 23:40:52,2392,105,2005-05-28 22:40:52,2,2006-02-16 02:30:53 +486,2005-05-27 23:51:12,1822,398,2005-05-28 20:26:12,1,2006-02-16 02:30:53 +487,2005-05-28 00:00:30,3774,569,2005-05-28 19:18:30,2,2006-02-16 02:30:53 +488,2005-05-28 00:07:50,393,168,2005-06-03 22:30:50,2,2006-02-16 02:30:53 +489,2005-05-28 00:09:12,1940,476,2005-05-31 04:44:12,2,2006-02-16 02:30:53 +490,2005-05-28 00:09:56,3524,95,2005-05-30 22:32:56,2,2006-02-16 02:30:53 +491,2005-05-28 00:13:35,1326,196,2005-05-29 00:11:35,2,2006-02-16 02:30:53 +492,2005-05-28 00:24:58,1999,228,2005-05-28 22:34:58,1,2006-02-16 02:30:53 +493,2005-05-28 00:34:11,184,501,2005-05-30 18:40:11,1,2006-02-16 02:30:53 +494,2005-05-28 00:39:31,1850,64,2005-06-02 19:35:31,1,2006-02-16 02:30:53 +495,2005-05-28 00:40:48,1007,526,2005-05-29 06:07:48,1,2006-02-16 02:30:53 +496,2005-05-28 00:43:41,1785,56,2005-06-04 03:56:41,1,2006-02-16 02:30:53 +497,2005-05-28 00:54:39,2636,20,2005-06-03 20:47:39,2,2006-02-16 02:30:53 +498,2005-05-28 01:01:21,458,287,2005-05-30 21:20:21,2,2006-02-16 02:30:53 +499,2005-05-28 01:05:07,2381,199,2005-06-05 19:54:07,2,2006-02-16 02:30:53 +500,2005-05-28 01:05:25,4500,145,2005-05-31 20:04:25,1,2006-02-16 02:30:53 +501,2005-05-28 01:09:36,601,162,2005-05-30 06:14:36,2,2006-02-16 02:30:53 +502,2005-05-28 01:34:43,3131,179,2005-05-31 01:02:43,2,2006-02-16 02:30:53 +503,2005-05-28 01:35:25,3005,288,2005-05-28 22:12:25,2,2006-02-16 02:30:53 +504,2005-05-28 02:05:34,2086,170,2005-05-30 23:03:34,1,2006-02-16 02:30:53 +505,2005-05-28 02:06:37,71,111,2005-05-29 06:57:37,1,2006-02-16 02:30:53 +506,2005-05-28 02:09:19,667,469,2005-06-05 20:34:19,1,2006-02-16 02:30:53 +507,2005-05-28 02:31:19,3621,421,2005-06-02 05:07:19,2,2006-02-16 02:30:53 +508,2005-05-28 02:40:50,4179,434,2005-06-05 03:05:50,1,2006-02-16 02:30:53 +509,2005-05-28 02:51:12,3416,147,2005-05-31 06:27:12,1,2006-02-16 02:30:53 +510,2005-05-28 02:52:14,4338,113,2005-05-30 21:20:14,2,2006-02-16 02:30:53 +511,2005-05-28 03:04:04,3827,296,2005-06-03 04:58:04,1,2006-02-16 02:30:53 +512,2005-05-28 03:07:50,2176,231,2005-06-05 02:12:50,2,2006-02-16 02:30:53 +513,2005-05-28 03:08:10,225,489,2005-05-29 07:22:10,1,2006-02-16 02:30:53 +514,2005-05-28 03:09:28,1697,597,2005-06-05 00:49:28,2,2006-02-16 02:30:53 +515,2005-05-28 03:10:10,3369,110,2005-06-04 02:18:10,2,2006-02-16 02:30:53 +516,2005-05-28 03:11:47,4357,400,2005-06-04 02:19:47,1,2006-02-16 02:30:53 +517,2005-05-28 03:17:57,234,403,2005-05-29 06:33:57,1,2006-02-16 02:30:53 +518,2005-05-28 03:18:02,4087,480,2005-05-30 05:32:02,1,2006-02-16 02:30:53 +519,2005-05-28 03:22:33,3564,245,2005-06-03 05:06:33,1,2006-02-16 02:30:53 +520,2005-05-28 03:27:37,3845,161,2005-06-04 05:47:37,1,2006-02-16 02:30:53 +521,2005-05-28 03:32:22,2397,374,2005-05-28 22:37:22,1,2006-02-16 02:30:53 +522,2005-05-28 03:33:20,3195,382,2005-05-31 04:23:20,1,2006-02-16 02:30:53 +523,2005-05-28 03:53:26,1905,138,2005-05-31 05:58:26,2,2006-02-16 02:30:53 +524,2005-05-28 03:57:28,1962,223,2005-05-31 05:20:28,1,2006-02-16 02:30:53 +525,2005-05-28 04:25:33,1817,14,2005-06-06 04:18:33,1,2006-02-16 02:30:53 +526,2005-05-28 04:27:37,1387,408,2005-05-30 07:52:37,1,2006-02-16 02:30:53 +527,2005-05-28 04:28:38,266,169,2005-06-02 08:19:38,1,2006-02-16 02:30:53 +528,2005-05-28 04:30:05,1655,359,2005-06-03 10:01:05,2,2006-02-16 02:30:53 +529,2005-05-28 04:34:17,2624,469,2005-05-30 00:35:17,1,2006-02-16 02:30:53 +530,2005-05-28 05:13:01,3332,312,2005-06-01 10:21:01,2,2006-02-16 02:30:53 +531,2005-05-28 05:23:38,1113,589,2005-05-29 08:00:38,2,2006-02-16 02:30:53 +532,2005-05-28 05:36:58,2793,120,2005-06-02 01:50:58,1,2006-02-16 02:30:53 +533,2005-05-28 06:14:46,4306,528,2005-06-01 06:26:46,2,2006-02-16 02:30:53 +534,2005-05-28 06:15:25,992,184,2005-06-06 07:51:25,1,2006-02-16 02:30:53 +535,2005-05-28 06:16:32,4209,307,2005-05-31 02:48:32,1,2006-02-16 02:30:53 +536,2005-05-28 06:17:33,2962,514,2005-06-03 10:02:33,2,2006-02-16 02:30:53 +537,2005-05-28 06:20:55,3095,315,2005-06-05 11:48:55,2,2006-02-16 02:30:53 +538,2005-05-28 06:21:05,2262,110,2005-06-02 01:22:05,2,2006-02-16 02:30:53 +539,2005-05-28 06:26:16,3427,161,2005-05-30 02:02:16,1,2006-02-16 02:30:53 +540,2005-05-28 06:40:25,3321,119,2005-06-06 00:47:25,1,2006-02-16 02:30:53 +541,2005-05-28 06:41:58,1662,535,2005-06-02 09:12:58,2,2006-02-16 02:30:53 +542,2005-05-28 06:42:13,4444,261,2005-06-03 09:05:13,1,2006-02-16 02:30:53 +543,2005-05-28 06:43:34,530,493,2005-06-06 07:16:34,2,2006-02-16 02:30:53 +544,2005-05-28 07:03:00,2964,311,2005-06-06 06:23:00,1,2006-02-16 02:30:53 +545,2005-05-28 07:10:20,1086,54,2005-06-04 01:47:20,2,2006-02-16 02:30:53 +546,2005-05-28 07:16:25,487,20,2005-06-01 08:36:25,1,2006-02-16 02:30:53 +547,2005-05-28 07:24:28,2065,506,2005-06-06 01:31:28,2,2006-02-16 02:30:53 +548,2005-05-28 07:34:56,3704,450,2005-06-05 03:14:56,2,2006-02-16 02:30:53 +549,2005-05-28 07:35:37,1818,159,2005-06-02 09:08:37,1,2006-02-16 02:30:53 +550,2005-05-28 07:39:16,3632,432,2005-06-06 12:20:16,2,2006-02-16 02:30:53 +551,2005-05-28 07:44:18,3119,315,2005-06-02 12:55:18,2,2006-02-16 02:30:53 +552,2005-05-28 07:53:38,23,106,2005-06-04 12:45:38,2,2006-02-16 02:30:53 +553,2005-05-28 08:14:44,1349,176,2005-06-02 03:01:44,2,2006-02-16 02:30:53 +554,2005-05-28 08:23:16,1951,376,2005-05-31 03:29:16,2,2006-02-16 02:30:53 +555,2005-05-28 08:31:14,4397,55,2005-05-30 07:34:14,2,2006-02-16 02:30:53 +556,2005-05-28 08:31:36,1814,22,2005-06-06 07:29:36,2,2006-02-16 02:30:53 +557,2005-05-28 08:36:22,158,444,2005-06-03 10:42:22,2,2006-02-16 02:30:53 +558,2005-05-28 08:38:43,4163,442,2005-06-06 13:52:43,1,2006-02-16 02:30:53 +559,2005-05-28 08:39:02,1227,572,2005-06-05 08:38:02,2,2006-02-16 02:30:53 +560,2005-05-28 08:53:02,644,463,2005-06-04 12:27:02,2,2006-02-16 02:30:53 +561,2005-05-28 08:54:06,928,77,2005-06-05 05:54:06,1,2006-02-16 02:30:53 +562,2005-05-28 09:01:21,3390,102,2005-06-02 05:26:21,2,2006-02-16 02:30:53 +563,2005-05-28 09:10:49,53,324,2005-06-06 11:32:49,1,2006-02-16 02:30:53 +564,2005-05-28 09:12:09,2973,282,2005-05-29 05:07:09,1,2006-02-16 02:30:53 +565,2005-05-28 09:26:31,1494,288,2005-06-01 07:28:31,1,2006-02-16 02:30:53 +566,2005-05-28 09:51:39,4330,253,2005-06-05 09:35:39,1,2006-02-16 02:30:53 +567,2005-05-28 09:56:20,3308,184,2005-06-01 06:41:20,2,2006-02-16 02:30:53 +568,2005-05-28 09:57:36,2232,155,2005-05-31 15:44:36,1,2006-02-16 02:30:53 +569,2005-05-28 10:12:41,4534,56,2005-06-03 10:08:41,2,2006-02-16 02:30:53 +570,2005-05-28 10:15:04,1122,21,2005-05-30 08:32:04,1,2006-02-16 02:30:53 +571,2005-05-28 10:17:41,4250,516,2005-06-05 07:56:41,1,2006-02-16 02:30:53 +572,2005-05-28 10:30:13,1899,337,2005-06-02 05:04:13,2,2006-02-16 02:30:53 +573,2005-05-28 10:35:23,4020,1,2005-06-03 06:32:23,1,2006-02-16 02:30:53 +574,2005-05-28 10:44:28,3883,76,2005-06-04 11:42:28,1,2006-02-16 02:30:53 +575,2005-05-28 10:56:09,4451,142,2005-06-05 15:39:09,1,2006-02-16 02:30:53 +576,2005-05-28 10:56:10,1866,588,2005-06-04 13:15:10,2,2006-02-16 02:30:53 +577,2005-05-28 11:09:14,375,6,2005-06-01 13:27:14,2,2006-02-16 02:30:53 +578,2005-05-28 11:15:48,2938,173,2005-06-02 09:59:48,1,2006-02-16 02:30:53 +579,2005-05-28 11:19:23,3481,181,2005-06-02 13:51:23,1,2006-02-16 02:30:53 +580,2005-05-28 11:19:53,3515,17,2005-06-01 10:44:53,2,2006-02-16 02:30:53 +581,2005-05-28 11:20:29,1380,186,2005-06-04 12:37:29,2,2006-02-16 02:30:53 +582,2005-05-28 11:33:46,4579,198,2005-05-29 08:33:46,1,2006-02-16 02:30:53 +583,2005-05-28 11:48:55,2679,386,2005-06-04 07:09:55,2,2006-02-16 02:30:53 +584,2005-05-28 11:49:00,1833,69,2005-06-01 11:54:00,1,2006-02-16 02:30:53 +585,2005-05-28 11:50:45,3544,490,2005-06-03 15:35:45,2,2006-02-16 02:30:53 +586,2005-05-28 12:03:00,898,77,2005-05-29 13:16:00,1,2006-02-16 02:30:53 +587,2005-05-28 12:05:33,1413,64,2005-05-30 13:45:33,2,2006-02-16 02:30:53 +588,2005-05-28 12:08:37,95,89,2005-05-29 16:25:37,2,2006-02-16 02:30:53 +589,2005-05-28 12:27:50,4231,308,2005-06-03 07:15:50,2,2006-02-16 02:30:53 +590,2005-05-28 13:06:50,473,462,2005-06-02 09:18:50,1,2006-02-16 02:30:53 +591,2005-05-28 13:11:04,377,19,2005-05-29 17:20:04,2,2006-02-16 02:30:53 +592,2005-05-28 13:21:08,638,244,2005-05-29 16:55:08,1,2006-02-16 02:30:53 +593,2005-05-28 13:33:23,1810,16,2005-05-30 17:10:23,2,2006-02-16 02:30:53 +594,2005-05-28 13:41:56,2766,538,2005-05-30 12:00:56,1,2006-02-16 02:30:53 +595,2005-05-28 13:59:54,595,294,2005-06-05 15:16:54,1,2006-02-16 02:30:53 +596,2005-05-28 14:00:03,821,589,2005-05-29 17:10:03,1,2006-02-16 02:30:53 +597,2005-05-28 14:01:02,4469,249,2005-06-06 19:06:02,2,2006-02-16 02:30:53 +598,2005-05-28 14:04:50,599,159,2005-06-03 18:00:50,2,2006-02-16 02:30:53 +599,2005-05-28 14:05:57,4136,393,2005-06-01 16:41:57,2,2006-02-16 02:30:53 +600,2005-05-28 14:08:19,1567,332,2005-06-03 11:57:19,2,2006-02-16 02:30:53 +601,2005-05-28 14:08:22,3225,429,2005-06-04 10:50:22,1,2006-02-16 02:30:53 +602,2005-05-28 14:15:54,1300,590,2005-06-05 15:16:54,2,2006-02-16 02:30:53 +603,2005-05-28 14:27:51,3248,537,2005-05-29 13:13:51,1,2006-02-16 02:30:53 +604,2005-05-28 14:37:07,1585,426,2005-06-03 11:03:07,2,2006-02-16 02:30:53 +605,2005-05-28 14:39:10,4232,501,2005-06-01 09:28:10,2,2006-02-16 02:30:53 +606,2005-05-28 14:48:39,3509,299,2005-06-04 09:44:39,2,2006-02-16 02:30:53 +607,2005-05-28 15:02:41,2561,554,2005-05-30 12:54:41,2,2006-02-16 02:30:53 +608,2005-05-28 15:03:44,4254,494,2005-06-04 17:14:44,2,2006-02-16 02:30:53 +609,2005-05-28 15:04:02,2944,150,2005-06-05 14:47:02,2,2006-02-16 02:30:53 +610,2005-05-28 15:15:25,3642,500,2005-06-02 12:30:25,2,2006-02-16 02:30:53 +611,2005-05-28 15:18:18,1230,580,2005-05-31 20:15:18,2,2006-02-16 02:30:53 +612,2005-05-28 15:24:54,2180,161,2005-05-30 14:22:54,2,2006-02-16 02:30:53 +613,2005-05-28 15:27:22,270,595,2005-06-02 20:01:22,1,2006-02-16 02:30:53 +614,2005-05-28 15:33:28,280,307,2005-06-04 12:27:28,2,2006-02-16 02:30:53 +615,2005-05-28 15:35:52,3397,533,2005-06-03 17:35:52,2,2006-02-16 02:30:53 +616,2005-05-28 15:45:39,989,471,2005-06-02 09:55:39,1,2006-02-16 02:30:53 +617,2005-05-28 15:49:14,4142,372,2005-05-31 14:29:14,2,2006-02-16 02:30:53 +618,2005-05-28 15:50:07,4445,248,2005-06-01 19:45:07,1,2006-02-16 02:30:53 +619,2005-05-28 15:52:26,2482,407,2005-06-06 17:55:26,2,2006-02-16 02:30:53 +620,2005-05-28 15:54:45,2444,321,2005-06-04 20:26:45,1,2006-02-16 02:30:53 +621,2005-05-28 15:58:12,1144,239,2005-05-30 21:54:12,1,2006-02-16 02:30:53 +622,2005-05-28 15:58:22,2363,109,2005-06-04 10:13:22,1,2006-02-16 02:30:53 +623,2005-05-28 16:01:28,1222,495,2005-05-30 11:19:28,1,2006-02-16 02:30:53 +624,2005-05-28 16:13:22,3660,569,2005-06-06 20:35:22,1,2006-02-16 02:30:53 +625,2005-05-28 16:35:46,2889,596,2005-06-01 14:19:46,1,2006-02-16 02:30:53 +626,2005-05-28 16:58:09,452,584,2005-06-01 14:02:09,2,2006-02-16 02:30:53 +627,2005-05-28 17:04:43,425,241,2005-06-04 19:58:43,2,2006-02-16 02:30:53 +628,2005-05-28 17:05:46,2513,173,2005-06-06 16:29:46,2,2006-02-16 02:30:53 +629,2005-05-28 17:19:15,1527,94,2005-06-02 20:01:15,2,2006-02-16 02:30:53 +630,2005-05-28 17:24:51,1254,417,2005-06-05 20:05:51,2,2006-02-16 02:30:53 +631,2005-05-28 17:36:32,2465,503,2005-06-03 14:56:32,2,2006-02-16 02:30:53 +632,2005-05-28 17:37:50,1287,442,2005-06-03 16:04:50,1,2006-02-16 02:30:53 +633,2005-05-28 17:37:59,58,360,2005-06-03 22:49:59,2,2006-02-16 02:30:53 +634,2005-05-28 17:40:35,2630,428,2005-06-05 16:18:35,2,2006-02-16 02:30:53 +635,2005-05-28 17:46:57,1648,42,2005-06-06 18:24:57,1,2006-02-16 02:30:53 +636,2005-05-28 17:47:58,4213,239,2005-06-04 16:32:58,1,2006-02-16 02:30:53 +637,2005-05-28 18:14:29,1581,250,2005-05-29 23:48:29,2,2006-02-16 02:30:53 +638,2005-05-28 18:24:43,2685,372,2005-06-02 19:03:43,2,2006-02-16 02:30:53 +639,2005-05-28 18:25:02,4204,198,2005-05-29 18:22:02,1,2006-02-16 02:30:53 +640,2005-05-28 18:43:26,495,465,2005-05-30 13:39:26,1,2006-02-16 02:30:53 +641,2005-05-28 18:45:47,3548,396,2005-06-04 15:24:47,1,2006-02-16 02:30:53 +642,2005-05-28 18:49:12,140,157,2005-06-01 20:50:12,2,2006-02-16 02:30:53 +643,2005-05-28 18:52:11,3105,240,2005-05-31 15:15:11,2,2006-02-16 02:30:53 +644,2005-05-28 18:59:12,4304,316,2005-06-04 18:06:12,1,2006-02-16 02:30:53 +645,2005-05-28 19:14:09,3128,505,2005-06-05 14:01:09,1,2006-02-16 02:30:53 +646,2005-05-28 19:16:14,1922,185,2005-05-31 16:50:14,2,2006-02-16 02:30:53 +647,2005-05-28 19:22:52,3435,569,2005-06-01 00:10:52,1,2006-02-16 02:30:53 +648,2005-05-28 19:25:54,3476,253,2005-06-03 15:57:54,2,2006-02-16 02:30:53 +649,2005-05-28 19:35:45,1781,197,2005-06-05 16:00:45,1,2006-02-16 02:30:53 +650,2005-05-28 19:45:40,4384,281,2005-05-29 21:02:40,1,2006-02-16 02:30:53 +651,2005-05-28 19:46:50,739,266,2005-05-30 16:29:50,1,2006-02-16 02:30:53 +652,2005-05-28 20:08:47,1201,43,2005-05-29 14:57:47,2,2006-02-16 02:30:53 +653,2005-05-28 20:12:20,126,327,2005-06-04 14:44:20,2,2006-02-16 02:30:53 +654,2005-05-28 20:15:30,2312,23,2005-05-30 22:02:30,2,2006-02-16 02:30:53 +655,2005-05-28 20:16:20,331,287,2005-05-31 16:46:20,2,2006-02-16 02:30:53 +656,2005-05-28 20:18:24,2846,437,2005-05-30 16:19:24,1,2006-02-16 02:30:53 +657,2005-05-28 20:23:09,848,65,2005-06-01 02:11:09,1,2006-02-16 02:30:53 +658,2005-05-28 20:23:23,3226,103,2005-06-06 19:31:23,2,2006-02-16 02:30:53 +659,2005-05-28 20:27:53,1382,207,2005-05-31 01:36:53,2,2006-02-16 02:30:53 +660,2005-05-28 20:53:31,1414,578,2005-05-30 15:26:31,1,2006-02-16 02:30:53 +661,2005-05-28 21:01:25,2247,51,2005-06-02 01:22:25,2,2006-02-16 02:30:53 +662,2005-05-28 21:09:31,2968,166,2005-06-01 19:00:31,2,2006-02-16 02:30:53 +663,2005-05-28 21:23:02,3997,176,2005-06-02 17:39:02,2,2006-02-16 02:30:53 +664,2005-05-28 21:31:08,87,523,2005-06-02 20:56:08,2,2006-02-16 02:30:53 +665,2005-05-28 21:38:39,1012,415,2005-05-29 21:37:39,1,2006-02-16 02:30:53 +666,2005-05-28 21:48:51,3075,437,2005-06-05 16:45:51,2,2006-02-16 02:30:53 +667,2005-05-28 21:49:02,797,596,2005-05-31 03:07:02,1,2006-02-16 02:30:53 +668,2005-05-28 21:54:45,3528,484,2005-05-29 22:32:45,1,2006-02-16 02:30:53 +669,2005-05-28 22:03:25,3677,313,2005-06-03 03:39:25,1,2006-02-16 02:30:53 +670,2005-05-28 22:04:03,227,201,2005-06-06 22:43:03,2,2006-02-16 02:30:53 +671,2005-05-28 22:04:30,1027,14,2005-06-03 01:21:30,2,2006-02-16 02:30:53 +672,2005-05-28 22:05:29,697,306,2005-06-06 02:10:29,2,2006-02-16 02:30:53 +673,2005-05-28 22:07:30,1769,468,2005-06-01 23:42:30,1,2006-02-16 02:30:53 +674,2005-05-28 22:11:35,1150,87,2005-06-01 23:58:35,2,2006-02-16 02:30:53 +675,2005-05-28 22:22:44,1273,338,2005-06-01 02:57:44,2,2006-02-16 02:30:53 +676,2005-05-28 22:27:51,2329,490,2005-05-29 20:36:51,2,2006-02-16 02:30:53 +677,2005-05-28 23:00:08,4558,194,2005-06-05 19:11:08,2,2006-02-16 02:30:53 +678,2005-05-28 23:15:48,3741,269,2005-06-03 04:43:48,2,2006-02-16 02:30:53 +679,2005-05-28 23:24:57,907,526,2005-06-06 21:59:57,2,2006-02-16 02:30:53 +680,2005-05-28 23:27:26,4147,482,2005-06-02 02:28:26,2,2006-02-16 02:30:53 +681,2005-05-28 23:39:44,3346,531,2005-06-01 01:42:44,1,2006-02-16 02:30:53 +682,2005-05-28 23:53:18,3160,148,2005-05-29 19:14:18,2,2006-02-16 02:30:53 +683,2005-05-29 00:09:48,2038,197,2005-06-02 04:27:48,1,2006-02-16 02:30:53 +684,2005-05-29 00:13:15,3242,461,2005-06-04 21:26:15,2,2006-02-16 02:30:53 +685,2005-05-29 00:17:51,1385,172,2005-06-05 05:32:51,2,2006-02-16 02:30:53 +686,2005-05-29 00:27:10,2441,411,2005-05-30 02:29:10,1,2006-02-16 02:30:53 +687,2005-05-29 00:32:09,1731,250,2005-05-31 23:53:09,1,2006-02-16 02:30:53 +688,2005-05-29 00:45:24,4135,162,2005-06-02 01:30:24,1,2006-02-16 02:30:53 +689,2005-05-29 00:46:53,742,571,2005-06-03 23:48:53,2,2006-02-16 02:30:53 +690,2005-05-29 00:54:53,2646,85,2005-06-06 00:45:53,1,2006-02-16 02:30:53 +691,2005-05-29 01:01:26,4034,433,2005-06-07 06:21:26,1,2006-02-16 02:30:53 +692,2005-05-29 01:32:10,800,18,2005-06-02 03:54:10,2,2006-02-16 02:30:53 +693,2005-05-29 01:42:31,635,190,2005-06-03 02:29:31,2,2006-02-16 02:30:53 +694,2005-05-29 01:49:43,592,399,2005-06-05 06:52:43,1,2006-02-16 02:30:53 +695,2005-05-29 01:50:53,4276,528,2005-06-03 02:28:53,1,2006-02-16 02:30:53 +696,2005-05-29 01:59:10,2076,19,2005-06-01 02:45:10,1,2006-02-16 02:30:53 +697,2005-05-29 02:04:04,3949,387,2005-06-04 00:47:04,2,2006-02-16 02:30:53 +698,2005-05-29 02:10:52,1412,109,2005-06-01 21:52:52,1,2006-02-16 02:30:53 +699,2005-05-29 02:11:44,130,246,2005-06-04 20:23:44,2,2006-02-16 02:30:53 +700,2005-05-29 02:18:54,500,117,2005-05-30 05:54:54,1,2006-02-16 02:30:53 +701,2005-05-29 02:26:27,372,112,2005-06-03 04:59:27,1,2006-02-16 02:30:53 +702,2005-05-29 02:27:30,2556,475,2005-05-30 01:52:30,2,2006-02-16 02:30:53 +703,2005-05-29 02:29:36,1123,269,2005-06-03 04:54:36,2,2006-02-16 02:30:53 +704,2005-05-29 02:44:43,2628,330,2005-06-06 01:51:43,2,2006-02-16 02:30:53 +705,2005-05-29 02:48:52,2809,257,2005-05-30 06:21:52,1,2006-02-16 02:30:53 +706,2005-05-29 03:05:49,2278,60,2005-06-04 22:48:49,1,2006-02-16 02:30:53 +707,2005-05-29 03:18:19,819,252,2005-05-30 02:45:19,1,2006-02-16 02:30:53 +708,2005-05-29 03:23:47,3133,127,2005-05-31 21:27:47,2,2006-02-16 02:30:53 +709,2005-05-29 03:48:01,2459,479,2005-06-06 05:21:01,1,2006-02-16 02:30:53 +710,2005-05-29 03:48:36,194,518,2005-06-03 05:03:36,1,2006-02-16 02:30:53 +711,2005-05-29 03:49:03,4581,215,2005-05-31 08:29:03,2,2006-02-16 02:30:53 +712,2005-05-29 04:02:24,4191,313,2005-05-30 03:09:24,2,2006-02-16 02:30:53 +713,2005-05-29 04:10:17,3664,507,2005-06-07 07:13:17,1,2006-02-16 02:30:53 +714,2005-05-29 04:15:21,2010,452,2005-06-01 23:05:21,2,2006-02-16 02:30:53 +715,2005-05-29 04:22:41,2030,545,2005-06-05 09:28:41,1,2006-02-16 02:30:53 +716,2005-05-29 04:35:29,85,36,2005-06-01 07:42:29,2,2006-02-16 02:30:53 +717,2005-05-29 04:37:44,1383,412,2005-05-30 05:48:44,2,2006-02-16 02:30:53 +718,2005-05-29 04:52:23,1736,498,2005-06-02 02:27:23,1,2006-02-16 02:30:53 +719,2005-05-29 05:16:05,267,245,2005-06-01 07:53:05,2,2006-02-16 02:30:53 +720,2005-05-29 05:17:30,3687,480,2005-06-06 02:47:30,2,2006-02-16 02:30:53 +721,2005-05-29 05:28:47,1116,44,2005-05-31 11:24:47,1,2006-02-16 02:30:53 +722,2005-05-29 05:30:31,4540,259,2005-06-06 04:51:31,1,2006-02-16 02:30:53 +723,2005-05-29 05:34:44,3407,309,2005-05-30 05:50:44,1,2006-02-16 02:30:53 +724,2005-05-29 05:53:23,3770,416,2005-06-05 04:01:23,2,2006-02-16 02:30:53 +725,2005-05-29 06:03:41,4088,245,2005-06-03 08:52:41,2,2006-02-16 02:30:53 +726,2005-05-29 06:05:29,933,452,2005-06-05 04:40:29,2,2006-02-16 02:30:53 +727,2005-05-29 06:08:15,1629,484,2005-05-30 07:16:15,1,2006-02-16 02:30:53 +728,2005-05-29 06:12:38,242,551,2005-06-03 07:41:38,1,2006-02-16 02:30:53 +729,2005-05-29 06:35:13,1688,323,2005-06-04 03:23:13,2,2006-02-16 02:30:53 +730,2005-05-29 07:00:59,3473,197,2005-06-06 01:17:59,1,2006-02-16 02:30:53 +731,2005-05-29 07:25:16,4124,5,2005-05-30 05:21:16,1,2006-02-16 02:30:53 +732,2005-05-29 07:32:51,2530,447,2005-05-30 10:08:51,2,2006-02-16 02:30:53 +733,2005-05-29 07:35:21,2951,363,2005-06-05 09:14:21,1,2006-02-16 02:30:53 +734,2005-05-29 07:38:52,3084,538,2005-06-03 10:17:52,2,2006-02-16 02:30:53 +735,2005-05-29 08:08:13,3421,454,2005-06-07 13:35:13,1,2006-02-16 02:30:53 +736,2005-05-29 08:10:07,3689,276,2005-06-05 10:21:07,2,2006-02-16 02:30:53 +737,2005-05-29 08:11:31,769,589,2005-06-04 11:18:31,2,2006-02-16 02:30:53 +738,2005-05-29 08:20:08,2284,256,2005-06-06 08:59:08,2,2006-02-16 02:30:53 +739,2005-05-29 08:28:18,1183,84,2005-06-06 09:21:18,2,2006-02-16 02:30:53 +740,2005-05-29 08:30:36,600,89,2005-06-04 12:47:36,2,2006-02-16 02:30:53 +741,2005-05-29 08:35:49,3189,495,2005-06-04 11:55:49,1,2006-02-16 02:30:53 +742,2005-05-29 08:36:30,273,483,2005-06-05 11:30:30,1,2006-02-16 02:30:53 +743,2005-05-29 08:39:02,2528,548,2005-06-06 08:42:02,2,2006-02-16 02:30:53 +744,2005-05-29 09:13:08,3722,420,2005-06-01 07:05:08,2,2006-02-16 02:30:53 +745,2005-05-29 09:22:57,581,152,2005-06-01 09:10:57,1,2006-02-16 02:30:53 +746,2005-05-29 09:25:10,4272,130,2005-06-02 04:20:10,2,2006-02-16 02:30:53 +747,2005-05-29 09:26:34,1993,291,2005-06-05 07:28:34,1,2006-02-16 02:30:53 +748,2005-05-29 09:27:00,2803,7,2005-06-03 04:25:00,1,2006-02-16 02:30:53 +749,2005-05-29 09:33:33,1146,375,2005-05-31 11:45:33,2,2006-02-16 02:30:53 +750,2005-05-29 09:41:40,730,269,2005-05-30 13:31:40,1,2006-02-16 02:30:53 +751,2005-05-29 09:55:43,2711,53,2005-06-02 04:54:43,1,2006-02-16 02:30:53 +752,2005-05-29 10:14:15,1720,126,2005-06-04 06:30:15,1,2006-02-16 02:30:53 +753,2005-05-29 10:16:42,1021,135,2005-06-05 08:52:42,2,2006-02-16 02:30:53 +754,2005-05-29 10:18:59,734,281,2005-06-04 05:03:59,2,2006-02-16 02:30:53 +755,2005-05-29 10:26:29,3090,576,2005-06-01 10:25:29,2,2006-02-16 02:30:53 +756,2005-05-29 10:28:45,3152,201,2005-06-04 12:50:45,1,2006-02-16 02:30:53 +757,2005-05-29 10:29:47,1067,435,2005-06-07 15:27:47,1,2006-02-16 02:30:53 +758,2005-05-29 10:31:56,1191,563,2005-06-01 14:53:56,2,2006-02-16 02:30:53 +759,2005-05-29 10:57:57,2367,179,2005-06-07 16:23:57,2,2006-02-16 02:30:53 +760,2005-05-29 11:07:25,3250,77,2005-06-02 14:16:25,1,2006-02-16 02:30:53 +761,2005-05-29 11:09:01,2342,58,2005-06-03 16:18:01,2,2006-02-16 02:30:53 +762,2005-05-29 11:15:51,3683,146,2005-06-06 07:48:51,1,2006-02-16 02:30:53 +763,2005-05-29 11:32:15,2022,50,2005-05-31 17:31:15,1,2006-02-16 02:30:53 +764,2005-05-29 11:37:35,1069,149,2005-05-31 16:47:35,1,2006-02-16 02:30:53 +765,2005-05-29 11:38:34,515,69,2005-06-02 17:04:34,1,2006-02-16 02:30:53 +766,2005-05-29 11:47:02,2154,383,2005-06-06 07:14:02,1,2006-02-16 02:30:53 +767,2005-05-29 12:20:19,687,67,2005-06-02 14:15:19,2,2006-02-16 02:30:53 +768,2005-05-29 12:30:46,2895,566,2005-06-07 09:00:46,2,2006-02-16 02:30:53 +769,2005-05-29 12:51:44,1523,575,2005-06-01 17:43:44,1,2006-02-16 02:30:53 +770,2005-05-29 12:56:50,2491,405,2005-06-07 15:54:50,2,2006-02-16 02:30:53 +771,2005-05-29 12:59:14,353,476,2005-06-01 16:05:14,2,2006-02-16 02:30:53 +772,2005-05-29 13:08:06,3319,556,2005-06-06 08:19:06,1,2006-02-16 02:30:53 +773,2005-05-29 13:18:05,245,563,2005-06-07 17:22:05,1,2006-02-16 02:30:53 +774,2005-05-29 13:19:43,1188,575,2005-06-01 18:51:43,1,2006-02-16 02:30:53 +775,2005-05-29 13:23:26,1197,124,2005-05-30 07:53:26,2,2006-02-16 02:30:53 +776,2005-05-29 13:35:35,4339,113,2005-06-03 17:33:35,1,2006-02-16 02:30:53 +777,2005-05-29 14:07:58,451,360,2005-06-03 08:41:58,2,2006-02-16 02:30:53 +778,2005-05-29 14:09:53,1816,535,2005-06-05 20:05:53,1,2006-02-16 02:30:53 +779,2005-05-29 14:17:17,533,105,2005-06-06 16:46:17,1,2006-02-16 02:30:53 +780,2005-05-29 14:18:32,1919,300,2005-06-06 20:14:32,1,2006-02-16 02:30:53 +781,2005-05-29 14:23:58,88,313,2005-05-30 17:44:58,1,2006-02-16 02:30:53 +782,2005-05-29 14:38:57,2255,596,2005-06-02 13:18:57,2,2006-02-16 02:30:53 +783,2005-05-29 14:41:18,3046,53,2005-06-06 10:39:18,2,2006-02-16 02:30:53 +784,2005-05-29 14:44:22,2936,352,2005-06-01 17:28:22,2,2006-02-16 02:30:53 +785,2005-05-29 15:08:41,39,72,2005-05-30 15:51:41,1,2006-02-16 02:30:53 +786,2005-05-29 15:17:28,2637,439,2005-06-07 10:07:28,2,2006-02-16 02:30:53 +787,2005-05-29 16:03:03,3919,27,2005-06-07 11:07:03,2,2006-02-16 02:30:53 +788,2005-05-29 16:13:55,763,562,2005-05-31 16:40:55,1,2006-02-16 02:30:53 +789,2005-05-29 16:17:07,708,553,2005-06-06 18:15:07,1,2006-02-16 02:30:53 +790,2005-05-29 16:19:29,2858,593,2005-06-02 17:22:29,2,2006-02-16 02:30:53 +791,2005-05-29 16:30:42,1554,284,2005-06-01 19:11:42,1,2006-02-16 02:30:53 +792,2005-05-29 16:32:10,2841,261,2005-05-31 18:01:10,1,2006-02-16 02:30:53 +793,2005-05-29 16:44:08,379,528,2005-06-06 19:21:08,2,2006-02-16 02:30:53 +794,2005-05-29 16:44:11,1995,50,2005-06-05 16:11:11,1,2006-02-16 02:30:53 +795,2005-05-29 16:57:39,609,551,2005-06-01 11:33:39,2,2006-02-16 02:30:53 +796,2005-05-29 16:59:44,2697,26,2005-06-03 16:22:44,2,2006-02-16 02:30:53 +797,2005-05-29 17:12:17,1446,244,2005-06-03 16:06:17,1,2006-02-16 02:30:53 +798,2005-05-29 17:23:43,1102,134,2005-06-01 13:06:43,2,2006-02-16 02:30:53 +799,2005-05-29 17:24:48,1713,429,2005-06-05 12:25:48,1,2006-02-16 02:30:53 +800,2005-05-29 17:28:12,441,472,2005-05-30 14:59:12,1,2006-02-16 02:30:53 +801,2005-05-29 17:35:50,1642,402,2005-06-04 17:05:50,2,2006-02-16 02:30:53 +802,2005-05-29 17:38:59,785,350,2005-05-31 22:42:59,2,2006-02-16 02:30:53 +803,2005-05-29 17:52:30,1602,32,2005-05-30 14:35:30,2,2006-02-16 02:30:53 +804,2005-05-29 18:10:24,3909,171,2005-06-06 22:53:24,1,2006-02-16 02:30:53 +805,2005-05-29 18:18:18,3132,232,2005-06-07 15:11:18,2,2006-02-16 02:30:53 +806,2005-05-29 18:31:30,2386,435,2005-05-31 00:18:30,2,2006-02-16 02:30:53 +807,2005-05-29 18:50:50,2195,235,2005-06-03 18:36:50,2,2006-02-16 02:30:53 +808,2005-05-29 19:08:20,1928,104,2005-06-06 20:32:20,2,2006-02-16 02:30:53 +809,2005-05-29 19:10:20,2114,222,2005-06-05 19:05:20,2,2006-02-16 02:30:53 +810,2005-05-29 19:12:04,2533,346,2005-06-04 21:12:04,2,2006-02-16 02:30:53 +811,2005-05-29 19:30:42,4419,401,2005-06-02 16:19:42,2,2006-02-16 02:30:53 +812,2005-05-29 20:00:30,1099,225,2005-05-30 19:43:30,2,2006-02-16 02:30:53 +813,2005-05-29 20:14:34,4554,344,2005-06-05 20:56:34,1,2006-02-16 02:30:53 +814,2005-05-29 20:16:12,1572,134,2005-06-07 17:47:12,1,2006-02-16 02:30:53 +815,2005-05-29 20:24:28,3757,14,2005-06-03 15:32:28,1,2006-02-16 02:30:53 +816,2005-05-29 20:26:39,630,474,2005-06-06 22:31:39,2,2006-02-16 02:30:53 +817,2005-05-29 20:39:14,186,554,2005-05-31 18:24:14,1,2006-02-16 02:30:53 +818,2005-05-29 20:47:53,4106,321,2005-06-02 23:18:53,2,2006-02-16 02:30:53 +819,2005-05-29 21:00:32,623,511,2005-06-02 15:15:32,2,2006-02-16 02:30:53 +820,2005-05-29 21:07:22,2584,22,2005-06-07 00:22:22,2,2006-02-16 02:30:53 +821,2005-05-29 21:31:12,3380,348,2005-06-04 22:49:12,1,2006-02-16 02:30:53 +822,2005-05-29 21:36:00,2634,480,2005-06-07 17:24:00,1,2006-02-16 02:30:53 +823,2005-05-29 21:39:37,3249,441,2005-05-30 22:06:37,1,2006-02-16 02:30:53 +824,2005-05-29 21:45:32,3518,357,2005-05-31 19:01:32,1,2006-02-16 02:30:53 +825,2005-05-29 21:49:41,712,371,2005-06-04 20:27:41,2,2006-02-16 02:30:53 +826,2005-05-29 21:56:15,2263,207,2005-06-08 03:18:15,1,2006-02-16 02:30:53 +827,2005-05-29 21:58:43,62,573,2005-06-06 00:54:43,1,2006-02-16 02:30:53 +828,2005-05-29 22:14:55,2468,217,2005-05-30 17:22:55,1,2006-02-16 02:30:53 +829,2005-05-29 22:16:42,1684,371,2005-06-06 01:38:42,1,2006-02-16 02:30:53 +830,2005-05-29 22:43:55,3464,3,2005-06-01 17:43:55,1,2006-02-16 02:30:53 +831,2005-05-29 22:50:25,3912,509,2005-06-06 02:27:25,1,2006-02-16 02:30:53 +832,2005-05-29 22:51:20,1381,159,2005-06-07 17:37:20,2,2006-02-16 02:30:53 +833,2005-05-29 23:21:56,2898,417,2005-06-02 18:40:56,1,2006-02-16 02:30:53 +834,2005-05-29 23:24:30,3628,84,2005-05-30 22:00:30,2,2006-02-16 02:30:53 +835,2005-05-29 23:37:00,299,381,2005-06-02 23:38:00,1,2006-02-16 02:30:53 +836,2005-05-29 23:56:42,3140,368,2005-05-31 04:11:42,2,2006-02-16 02:30:53 +837,2005-05-30 00:02:08,977,172,2005-06-02 05:31:08,2,2006-02-16 02:30:53 +838,2005-05-30 00:27:57,2859,504,2005-06-06 22:19:57,2,2006-02-16 02:30:53 +839,2005-05-30 00:28:12,1886,337,2005-06-08 02:43:12,1,2006-02-16 02:30:53 +840,2005-05-30 00:28:41,4049,79,2005-05-31 20:39:41,2,2006-02-16 02:30:53 +841,2005-05-30 00:31:17,4318,387,2005-06-02 19:14:17,1,2006-02-16 02:30:53 +842,2005-05-30 00:32:04,2328,238,2005-06-01 02:21:04,1,2006-02-16 02:30:53 +843,2005-05-30 00:44:24,2214,313,2005-05-31 00:58:24,2,2006-02-16 02:30:53 +844,2005-05-30 00:58:20,536,429,2005-06-01 00:38:20,1,2006-02-16 02:30:53 +845,2005-05-30 01:17:25,2001,72,2005-06-07 02:00:25,1,2006-02-16 02:30:53 +846,2005-05-30 01:17:45,938,49,2005-06-01 00:56:45,2,2006-02-16 02:30:53 +847,2005-05-30 01:18:15,4387,380,2005-06-06 20:20:15,2,2006-02-16 02:30:53 +848,2005-05-30 01:19:53,1363,436,2005-06-05 23:40:53,1,2006-02-16 02:30:53 +849,2005-05-30 01:23:07,2424,449,2005-06-07 01:50:07,1,2006-02-16 02:30:53 +850,2005-05-30 01:35:12,2390,517,2005-05-31 01:51:12,1,2006-02-16 02:30:53 +851,2005-05-30 01:35:15,2780,530,2005-06-06 07:27:15,1,2006-02-16 02:30:53 +852,2005-05-30 01:36:57,1622,549,2005-06-01 22:44:57,1,2006-02-16 02:30:53 +853,2005-05-30 01:43:31,3693,122,2005-06-01 02:05:31,1,2006-02-16 02:30:53 +854,2005-05-30 01:56:11,921,369,2005-06-01 06:34:11,2,2006-02-16 02:30:53 +855,2005-05-30 02:00:28,2527,406,2005-06-03 20:16:28,2,2006-02-16 02:30:53 +856,2005-05-30 02:01:21,3969,53,2005-06-07 03:25:21,1,2006-02-16 02:30:53 +857,2005-05-30 02:01:23,2569,204,2005-06-02 06:07:23,2,2006-02-16 02:30:53 +858,2005-05-30 02:10:32,1258,358,2005-06-01 04:42:32,1,2006-02-16 02:30:53 +859,2005-05-30 02:36:20,3032,79,2005-06-02 07:49:20,2,2006-02-16 02:30:53 +860,2005-05-30 02:45:16,578,276,2005-06-08 07:28:16,1,2006-02-16 02:30:53 +861,2005-05-30 02:48:32,3711,502,2005-06-06 05:43:32,1,2006-02-16 02:30:53 +862,2005-05-30 03:09:11,1186,328,2005-06-03 21:27:11,1,2006-02-16 02:30:53 +863,2005-05-30 03:14:59,3999,379,2005-06-05 04:34:59,2,2006-02-16 02:30:53 +864,2005-05-30 03:27:17,2777,544,2005-06-06 08:28:17,1,2006-02-16 02:30:53 +865,2005-05-30 03:39:44,3183,154,2005-06-07 08:10:44,2,2006-02-16 02:30:53 +866,2005-05-30 03:43:54,2867,8,2005-06-08 04:28:54,1,2006-02-16 02:30:53 +867,2005-05-30 03:54:43,3389,99,2005-06-01 22:59:43,1,2006-02-16 02:30:53 +868,2005-05-30 04:19:55,3604,28,2005-05-31 02:28:55,1,2006-02-16 02:30:53 +869,2005-05-30 04:22:06,3399,296,2005-06-03 09:18:06,2,2006-02-16 02:30:53 +870,2005-05-30 04:25:47,2903,391,2005-06-06 04:32:47,1,2006-02-16 02:30:53 +871,2005-05-30 05:01:30,4573,303,2005-06-04 06:22:30,2,2006-02-16 02:30:53 +872,2005-05-30 05:03:04,3904,548,2005-06-06 10:35:04,1,2006-02-16 02:30:53 +873,2005-05-30 05:15:20,4568,375,2005-06-07 00:49:20,2,2006-02-16 02:30:53 +874,2005-05-30 05:36:21,363,52,2005-06-01 09:32:21,1,2006-02-16 02:30:53 +875,2005-05-30 05:38:24,1428,326,2005-06-06 00:34:24,2,2006-02-16 02:30:53 +876,2005-05-30 05:41:22,1471,339,2005-06-07 09:06:22,2,2006-02-16 02:30:53 +877,2005-05-30 05:48:59,886,9,2005-06-02 09:30:59,1,2006-02-16 02:30:53 +878,2005-05-30 05:49:13,4265,323,2005-06-07 04:35:13,1,2006-02-16 02:30:53 +879,2005-05-30 05:49:42,4021,482,2005-06-05 01:45:42,2,2006-02-16 02:30:53 +880,2005-05-30 06:12:33,1819,460,2005-06-02 04:35:33,2,2006-02-16 02:30:53 +881,2005-05-30 06:15:36,602,242,2005-06-02 10:21:36,1,2006-02-16 02:30:53 +882,2005-05-30 06:16:06,3841,477,2005-06-02 11:57:06,1,2006-02-16 02:30:53 +883,2005-05-30 06:21:05,2271,399,2005-06-07 04:50:05,2,2006-02-16 02:30:53 +884,2005-05-30 06:41:32,4079,17,2005-05-31 07:39:32,1,2006-02-16 02:30:53 +885,2005-05-30 06:54:28,646,62,2005-06-03 07:03:28,2,2006-02-16 02:30:53 +886,2005-05-30 06:54:51,4356,393,2005-06-01 06:04:51,2,2006-02-16 02:30:53 +887,2005-05-30 07:10:00,2727,16,2005-06-01 06:48:00,2,2006-02-16 02:30:53 +888,2005-05-30 07:13:14,387,128,2005-06-06 09:50:14,1,2006-02-16 02:30:53 +889,2005-05-30 07:14:53,1299,114,2005-05-31 07:56:53,2,2006-02-16 02:30:53 +890,2005-05-30 07:43:04,1464,349,2005-06-01 11:26:04,1,2006-02-16 02:30:53 +891,2005-05-30 07:43:12,2611,391,2005-06-08 09:21:12,1,2006-02-16 02:30:53 +892,2005-05-30 08:02:56,471,274,2005-06-05 12:51:56,1,2006-02-16 02:30:53 +893,2005-05-30 08:06:59,3260,502,2005-06-07 08:23:59,2,2006-02-16 02:30:53 +894,2005-05-30 08:31:31,1118,400,2005-06-07 12:39:31,1,2006-02-16 02:30:53 +895,2005-05-30 08:50:43,2744,192,2005-06-05 10:58:43,1,2006-02-16 02:30:53 +896,2005-05-30 09:03:52,2817,207,2005-06-05 07:37:52,2,2006-02-16 02:30:53 +897,2005-05-30 09:10:01,1334,432,2005-06-08 03:43:01,1,2006-02-16 02:30:53 +898,2005-05-30 09:26:19,3497,384,2005-06-01 10:45:19,2,2006-02-16 02:30:53 +899,2005-05-30 09:29:30,1096,156,2005-06-06 12:39:30,2,2006-02-16 02:30:53 +900,2005-05-30 09:38:41,3543,586,2005-06-07 11:54:41,1,2006-02-16 02:30:53 +901,2005-05-30 09:40:40,760,259,2005-06-02 10:32:40,1,2006-02-16 02:30:53 +902,2005-05-30 09:53:36,1514,561,2005-06-07 12:10:36,1,2006-02-16 02:30:53 +903,2005-05-30 10:11:29,2423,197,2005-06-03 09:33:29,1,2006-02-16 02:30:53 +904,2005-05-30 10:19:42,2466,44,2005-06-05 04:58:42,2,2006-02-16 02:30:53 +905,2005-05-30 10:25:00,4372,50,2005-06-06 06:23:00,1,2006-02-16 02:30:53 +906,2005-05-30 10:30:38,1862,549,2005-06-07 06:44:38,2,2006-02-16 02:30:53 +907,2005-05-30 10:37:27,3320,506,2005-06-02 09:51:27,1,2006-02-16 02:30:53 +908,2005-05-30 10:38:37,4427,85,2005-06-03 09:56:37,1,2006-02-16 02:30:53 +909,2005-05-30 10:43:38,3775,486,2005-06-08 12:07:38,1,2006-02-16 02:30:53 +910,2005-05-30 10:46:16,2601,374,2005-06-04 13:32:16,1,2006-02-16 02:30:53 +911,2005-05-30 10:50:22,1404,366,2005-06-07 12:26:22,2,2006-02-16 02:30:53 +912,2005-05-30 10:58:33,3200,390,2005-05-31 09:31:33,2,2006-02-16 02:30:53 +913,2005-05-30 11:04:58,3213,369,2005-06-07 13:22:58,2,2006-02-16 02:30:53 +914,2005-05-30 11:06:00,1393,596,2005-06-04 06:07:00,2,2006-02-16 02:30:53 +915,2005-05-30 11:20:27,1859,115,2005-06-02 11:55:27,1,2006-02-16 02:30:53 +916,2005-05-30 11:25:01,1290,6,2005-05-31 09:06:01,1,2006-02-16 02:30:53 +917,2005-05-30 11:27:06,3629,385,2005-06-02 08:31:06,1,2006-02-16 02:30:53 +918,2005-05-30 11:32:24,818,197,2005-05-31 07:55:24,2,2006-02-16 02:30:53 +919,2005-05-30 11:35:06,4052,374,2005-06-02 13:16:06,2,2006-02-16 02:30:53 +920,2005-05-30 11:44:01,3860,584,2005-06-02 08:19:01,2,2006-02-16 02:30:53 +921,2005-05-30 11:53:09,1827,508,2005-06-03 10:00:09,2,2006-02-16 02:30:53 +922,2005-05-30 11:55:55,2442,550,2005-06-08 10:12:55,2,2006-02-16 02:30:53 +923,2005-05-30 11:58:50,1884,37,2005-06-05 09:57:50,1,2006-02-16 02:30:53 +924,2005-05-30 12:10:59,3279,293,2005-06-04 17:28:59,1,2006-02-16 02:30:53 +925,2005-05-30 12:13:52,3203,137,2005-06-02 14:41:52,2,2006-02-16 02:30:53 +926,2005-05-30 12:15:54,4327,76,2005-06-01 08:53:54,2,2006-02-16 02:30:53 +927,2005-05-30 12:16:40,1158,167,2005-05-31 16:20:40,2,2006-02-16 02:30:53 +928,2005-05-30 12:27:14,246,79,2005-06-05 13:56:14,2,2006-02-16 02:30:53 +929,2005-05-30 12:32:39,4296,536,2005-06-06 12:17:39,1,2006-02-16 02:30:53 +930,2005-05-30 12:44:57,2835,141,2005-06-04 10:53:57,2,2006-02-16 02:30:53 +931,2005-05-30 12:53:01,3384,421,2005-05-31 14:28:01,1,2006-02-16 02:30:53 +932,2005-05-30 12:55:36,719,198,2005-05-31 10:30:36,2,2006-02-16 02:30:53 +933,2005-05-30 13:08:45,3672,66,2005-06-01 18:56:45,1,2006-02-16 02:30:53 +934,2005-05-30 13:24:46,3595,60,2005-06-08 16:44:46,2,2006-02-16 02:30:53 +935,2005-05-30 13:29:36,2421,256,2005-06-02 11:08:36,1,2006-02-16 02:30:53 +936,2005-05-30 13:52:49,901,469,2005-06-07 16:56:49,1,2006-02-16 02:30:53 +937,2005-05-30 14:47:31,1054,304,2005-06-05 09:53:31,2,2006-02-16 02:30:53 +938,2005-05-30 14:47:31,1521,46,2005-06-04 10:10:31,2,2006-02-16 02:30:53 +939,2005-05-30 14:49:34,1314,367,2005-06-01 19:00:34,1,2006-02-16 02:30:53 +940,2005-05-30 15:01:02,1278,534,2005-06-01 18:26:02,1,2006-02-16 02:30:53 +941,2005-05-30 15:02:25,3630,562,2005-06-01 17:19:25,1,2006-02-16 02:30:53 +942,2005-05-30 15:05:47,4279,473,2005-06-08 15:59:47,2,2006-02-16 02:30:53 +943,2005-05-30 15:20:19,3737,57,2005-06-06 18:53:19,1,2006-02-16 02:30:53 +944,2005-05-30 15:26:24,151,131,2005-06-07 18:09:24,2,2006-02-16 02:30:53 +945,2005-05-30 15:33:17,1441,357,2005-06-02 15:02:17,2,2006-02-16 02:30:53 +946,2005-05-30 15:35:08,1264,486,2005-06-08 11:38:08,1,2006-02-16 02:30:53 +947,2005-05-30 15:36:57,4478,62,2005-06-04 18:48:57,1,2006-02-16 02:30:53 +948,2005-05-30 15:44:27,585,245,2005-06-08 17:30:27,2,2006-02-16 02:30:53 +949,2005-05-30 15:50:39,2202,368,2005-06-03 14:25:39,1,2006-02-16 02:30:53 +950,2005-05-30 16:06:08,491,83,2005-06-01 11:43:08,1,2006-02-16 02:30:53 +951,2005-05-30 16:10:35,1395,59,2005-05-31 19:01:35,2,2006-02-16 02:30:53 +952,2005-05-30 16:28:07,4389,311,2005-06-02 16:12:07,2,2006-02-16 02:30:53 +953,2005-05-30 16:34:02,2194,210,2005-05-31 20:34:02,1,2006-02-16 02:30:53 +954,2005-05-30 16:57:29,1231,297,2005-06-08 13:30:29,2,2006-02-16 02:30:53 +955,2005-05-30 16:59:03,4140,301,2005-05-31 11:58:03,2,2006-02-16 02:30:53 +956,2005-05-30 17:30:28,647,296,2005-06-07 13:54:28,2,2006-02-16 02:30:53 +957,2005-05-30 17:53:29,4428,440,2005-06-03 15:31:29,2,2006-02-16 02:30:53 +958,2005-05-30 17:58:03,548,186,2005-06-01 19:17:03,2,2006-02-16 02:30:53 +959,2005-05-30 18:07:00,3108,535,2005-06-02 14:37:00,2,2006-02-16 02:30:53 +960,2005-05-30 18:13:23,1966,445,2005-06-04 00:12:23,2,2006-02-16 02:30:53 +961,2005-05-30 18:16:44,3293,588,2005-06-04 23:40:44,2,2006-02-16 02:30:53 +962,2005-05-30 18:45:17,4535,520,2005-06-05 22:47:17,1,2006-02-16 02:30:53 +963,2005-05-30 18:52:53,1921,225,2005-06-07 16:19:53,2,2006-02-16 02:30:53 +964,2005-05-30 18:53:21,657,287,2005-06-04 22:32:21,2,2006-02-16 02:30:53 +965,2005-05-30 19:00:14,3363,502,2005-05-31 17:10:14,2,2006-02-16 02:30:53 +966,2005-05-30 19:00:37,1294,496,2005-05-31 23:51:37,1,2006-02-16 02:30:53 +967,2005-05-30 19:12:06,1954,330,2005-06-09 00:02:06,2,2006-02-16 02:30:53 +968,2005-05-30 19:20:03,119,576,2005-05-31 18:17:03,2,2006-02-16 02:30:53 +969,2005-05-30 19:23:48,443,551,2005-05-31 21:14:48,1,2006-02-16 02:30:53 +970,2005-05-30 19:50:28,1520,307,2005-06-09 01:19:28,1,2006-02-16 02:30:53 +971,2005-05-30 20:10:52,2911,561,2005-06-06 20:47:52,1,2006-02-16 02:30:53 +972,2005-05-30 20:21:07,2,411,2005-06-06 00:36:07,1,2006-02-16 02:30:53 +973,2005-05-30 20:27:45,1914,473,2005-06-08 22:47:45,2,2006-02-16 02:30:53 +974,2005-05-30 20:28:42,2617,596,2005-06-08 23:45:42,2,2006-02-16 02:30:53 +975,2005-05-30 21:07:15,3109,7,2005-06-03 01:48:15,2,2006-02-16 02:30:53 +976,2005-05-30 21:11:19,2290,581,2005-06-06 02:16:19,2,2006-02-16 02:30:53 +977,2005-05-30 21:22:26,2029,394,2005-06-04 22:32:26,2,2006-02-16 02:30:53 +978,2005-05-30 21:30:52,407,154,2005-06-07 16:22:52,1,2006-02-16 02:30:53 +979,2005-05-30 21:37:11,3917,279,2005-06-08 00:24:11,2,2006-02-16 02:30:53 +980,2005-05-30 21:45:19,4169,273,2005-06-01 20:32:19,1,2006-02-16 02:30:53 +981,2005-05-30 21:52:42,2913,326,2005-06-01 03:15:42,2,2006-02-16 02:30:53 +982,2005-05-30 22:15:24,3560,524,2005-06-02 16:18:24,1,2006-02-16 02:30:53 +983,2005-05-30 22:15:51,63,115,2005-06-02 22:56:51,1,2006-02-16 02:30:53 +984,2005-05-30 22:17:17,2305,262,2005-06-01 20:15:17,2,2006-02-16 02:30:53 +985,2005-05-30 22:18:35,1573,564,2005-06-04 23:36:35,1,2006-02-16 02:30:53 +986,2005-05-30 22:22:52,4045,253,2005-06-01 02:24:52,1,2006-02-16 02:30:53 +987,2005-05-30 22:59:12,390,11,2005-06-07 20:56:12,1,2006-02-16 02:30:53 +988,2005-05-30 23:08:03,1364,12,2005-06-07 00:22:03,1,2006-02-16 02:30:53 +989,2005-05-30 23:11:51,4388,83,2005-06-03 20:36:51,2,2006-02-16 02:30:53 +990,2005-05-30 23:25:14,4171,311,2005-06-06 18:41:14,2,2006-02-16 02:30:53 +991,2005-05-30 23:29:22,2863,593,2005-06-07 23:16:22,1,2006-02-16 02:30:53 +992,2005-05-30 23:47:56,3572,123,2005-06-05 19:01:56,1,2006-02-16 02:30:53 +993,2005-05-30 23:54:19,2080,513,2005-06-04 21:27:19,1,2006-02-16 02:30:53 +994,2005-05-30 23:55:36,2798,472,2005-06-04 01:00:36,2,2006-02-16 02:30:53 +995,2005-05-31 00:06:02,17,150,2005-06-06 02:30:02,2,2006-02-16 02:30:53 +996,2005-05-31 00:06:20,2075,331,2005-05-31 21:29:20,2,2006-02-16 02:30:53 +997,2005-05-31 00:08:25,4243,216,2005-06-02 00:17:25,2,2006-02-16 02:30:53 +998,2005-05-31 00:16:57,3395,389,2005-06-01 22:41:57,1,2006-02-16 02:30:53 +999,2005-05-31 00:25:10,4433,413,2005-06-03 06:05:10,2,2006-02-16 02:30:53 +1000,2005-05-31 00:25:56,1774,332,2005-06-08 19:42:56,2,2006-02-16 02:30:53 diff --git a/backend/tests/integration_tests/data/sakila/store.csv b/backend/tests/integration_tests/data/sakila/store.csv index 44d1d05..e930a62 100644 --- a/backend/tests/integration_tests/data/sakila/store.csv +++ b/backend/tests/integration_tests/data/sakila/store.csv @@ -1,3 +1,3 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:c7ff6d7c65544670e8ac0717a94933f7ca7a08529411b0e25278acb628bf7389 -size 101 +store_id,manager_staff_id,address_id,last_update +1,1,1,2006-02-15 09:57:12 +2,2,2,2006-02-15 09:57:12 diff --git a/backend/tests/integration_tests/how_to_add_more_tests.md b/backend/tests/integration_tests/how_to_add_more_tests.md index ec1f085..11b299b 100644 --- a/backend/tests/integration_tests/how_to_add_more_tests.md +++ b/backend/tests/integration_tests/how_to_add_more_tests.md @@ -7,7 +7,7 @@ execute the following commands: ```bash cd backend/ - uv run pytest -x -vv + uv run pytest -x -vv ``` This will run all the backend-related tests. @@ -53,7 +53,7 @@ If you're adding new CSV data, you'll need to update a few snapshot tests. To do this, navigate to the `visitran_backend` folder and run the following command: ```bash -uv run pytest --snapshot-update +uv run pytest --snapshot-update ``` This command will generate certain JSON files, which you are required to commit. @@ -89,7 +89,7 @@ Note: The first step is not required if [this issue](https://zipstack.atlassian. 2. Run the dummy test below to access the Visitran web UI opened on the test project: ```bash - TEST_UI_START=1 uv run pytest -x -vv -k "test_ui_start" + TEST_UI_START=1 uv run pytest -x -vv -k "test_ui_start" ``` You can then access the UI at `localhost:8000`. To end the session, press `Ctrl + C`. @@ -126,6 +126,6 @@ Inside the dictionary, add the `type` of transformation or refer to other transf 10. Run a snapshot update and then run the test. ```bash - uv run pytest --snapshot-update + uv run pytest --snapshot-update uv run pytest ``` diff --git a/backend/visitran/adapters/bigquery/connection.py b/backend/visitran/adapters/bigquery/connection.py index 64ce61f..6b45997 100644 --- a/backend/visitran/adapters/bigquery/connection.py +++ b/backend/visitran/adapters/bigquery/connection.py @@ -167,7 +167,7 @@ def create_or_replace_view(self, schema_name: str, table_name: str, select_state def is_table_exists(self, schema_name: str, table_name: str) -> bool: """Returns TRUE if table exists in DB. - + Falls back to BigQuery client API if Ibis fails (e.g., for tables with INTERVAL columns). """ with warnings.catch_warnings(): @@ -190,11 +190,11 @@ def is_table_exists(self, schema_name: str, table_name: str) -> bool: def _is_table_exists_via_client(self, schema_name: str, table_name: str) -> bool: """Check table existence using BigQuery client (bypasses Ibis schema parsing). - + This is a fallback for tables with INTERVAL columns that Ibis can't handle. """ from google.api_core.exceptions import NotFound - + try: client = bigquery.Client(project=self.project_id, credentials=self.credentials) table_ref = f"{self.project_id}.{schema_name}.{table_name}" @@ -207,7 +207,7 @@ def _is_table_exists_via_client(self, schema_name: str, table_name: str) -> bool def get_table_obj(self, schema_name: str, table_name: str): """Return table object, handling INTERVAL columns that Ibis can't parse. - + If Ibis fails due to INTERVAL columns, creates a temporary view that casts INTERVAL columns to STRING, allowing the table to be used in transformations. """ @@ -221,23 +221,23 @@ def get_table_obj(self, schema_name: str, table_name: str): def _get_table_obj_with_interval_workaround(self, schema_name: str, table_name: str): """Create a table object for tables with INTERVAL columns. - + Creates a temporary view that casts INTERVAL columns to STRING, allowing Ibis to work with the table. """ import logging import uuid - + logging.warning( f"Table '{schema_name}.{table_name}' has INTERVAL columns that Ibis can't handle. " "Creating a workaround view with INTERVAL columns cast to STRING." ) - + # Get table schema using BigQuery client client = bigquery.Client(project=self.project_id, credentials=self.credentials) table_ref = f"{self.project_id}.{schema_name}.{table_name}" bq_table = client.get_table(table_ref) - + qi = self.quote_identifier # Build SELECT with INTERVAL columns cast to STRING @@ -257,9 +257,9 @@ def _get_table_obj_with_interval_workaround(self, schema_name: str, table_name: OPTIONS(expiration_timestamp=TIMESTAMP_ADD(CURRENT_TIMESTAMP(), INTERVAL 1 HOUR)) AS SELECT {', '.join(select_columns)} FROM {qi(schema_name)}.{qi(table_name)} """ - + self.connection.raw_sql(create_view_sql) - + # Return the view as a table object return self.connection.table(temp_view_name, database=schema_name) @@ -271,12 +271,12 @@ def merge_into_table( primary_key: Union[str, list[str]] = None, ) -> None: """Efficient upsert using DELETE + INSERT for BigQuery. - + This approach is more efficient than MERGE for BigQuery because: 1. BigQuery is optimized for bulk operations 2. DELETE + INSERT performs better than UPDATE operations 3. Works better with BigQuery's partitioning strategy - + Args: primary_key: Can be a single column name (str) or list of column names for composite keys """ @@ -288,9 +288,9 @@ def merge_into_table( temp_table_name=f"{target_table_name}__temp", ) ) - - - + + + # 1. Create temporary table with incremental data (includes transformations) self.create_or_replace_table( schema_name=schema_name, @@ -301,10 +301,10 @@ def merge_into_table( # 2. Get target table columns target_columns = self.get_table_columns(schema_name=schema_name, table_name=target_table_name) - + if not target_columns: raise ValueError(f"No columns found in target table {schema_name}.{target_table_name}") - + qi = self.quote_identifier # 3. If primary key is provided, use efficient DELETE + INSERT @@ -373,7 +373,7 @@ def merge_into_table( self.connection.raw_sql(f"DROP TABLE IF EXISTS {qi(schema_name)}.{qi(target_table_name + '__temp')}") except Exception: pass # Ignore cleanup errors - + # Re-raise the original error with context raise Exception( f"BigQuery incremental upsert failed for {schema_name}.{target_table_name}: {str(e)}" @@ -381,7 +381,7 @@ def merge_into_table( - + def create_schema(self, schema_name: str) -> None: try: @@ -481,19 +481,19 @@ def validate(self) -> None: "dataset_id": self.dataset_id, "credentials": self.credentials, } - + for field, value in required.items(): if not value: raise ConnectionFieldMissingException(missing_fields=field) - + # Then, validate that the dataset exists in the project try: # Get the BigQuery client client = self.connection.client - + # Get the dataset reference dataset_ref = client.dataset(self.dataset_id, project=self.project_id) - + # Try to get the dataset - this will raise an exception if it doesn't exist client.get_dataset(dataset_ref) except Exception as e: diff --git a/backend/visitran/adapters/bigquery/db_reader.py b/backend/visitran/adapters/bigquery/db_reader.py index fff942b..06b6283 100644 --- a/backend/visitran/adapters/bigquery/db_reader.py +++ b/backend/visitran/adapters/bigquery/db_reader.py @@ -18,7 +18,7 @@ def __init__(self, db_connection: BigQueryConnection) -> None: def get_table_info(self, schema_name: str, table_name: str) -> tuple[str, dict[str, Any]]: """ Get table info, falling back to SQLAlchemy for tables Ibis can't handle. - + BigQuery's INTERVAL type doesn't specify precision/unit, causing Ibis to fail with "Interval precision is None". This override catches such errors and uses SQLAlchemy inspector as a fallback. @@ -40,12 +40,12 @@ def get_table_info(self, schema_name: str, table_name: str) -> tuple[str, dict[s def _get_table_info_via_sqlalchemy(self, schema_name: str, table_name: str) -> tuple[str, dict[str, Any]]: """ Fallback method using SQLAlchemy inspector for tables Ibis can't handle. - + This handles BigQuery tables with INTERVAL columns that cause Ibis to fail. """ columns = [] sqlalchemy_cols = self.inspector.get_columns(table_name, schema_name) - + for col in sqlalchemy_cols: columns.append({ "name": col["name"], @@ -55,21 +55,21 @@ def _get_table_info_via_sqlalchemy(self, schema_name: str, table_name: str) -> t "default": col.get("default"), "comment": col.get("comment", "") }) - + # Get constraints using inspector foreign_keys = self.inspector.get_foreign_keys(table_name, schema_name) primary_keys = self.inspector.get_pk_constraint(table_name, schema_name) - + try: unique_constraints = self.inspector.get_unique_constraints(table_name, schema_name) except Exception: unique_constraints = [] - + try: indexes = self.inspector.get_indexes(table_name, schema_name) except Exception: indexes = [] - + table_info = { "name": table_name, "schema_name": schema_name, @@ -79,5 +79,5 @@ def _get_table_info_via_sqlalchemy(self, schema_name: str, table_name: str) -> t "indexes": indexes, "columns": columns, } - + return table_name, table_info diff --git a/backend/visitran/adapters/bigquery/model.py b/backend/visitran/adapters/bigquery/model.py index cd8a155..c9d3f4e 100644 --- a/backend/visitran/adapters/bigquery/model.py +++ b/backend/visitran/adapters/bigquery/model.py @@ -76,22 +76,22 @@ def execute_incremental(self) -> None: self.model.destination_table_name, ) ) - + # Check for schema changes first if self._has_schema_changed(): logging.info(f"Schema change detected for {self.model.destination_schema_name}.{self.model.destination_table_name}, performing full refresh") self._full_refresh_table() - + else: # Continue with incremental logic if no schema changes self.model.select_statement = self.model.select_if_incremental() - + logging.info(f"No schema changes detected for {self.model.destination_schema_name}.{self.model.destination_table_name}, using incremental update") # Get primary key from model if available primary_key = getattr(self.model, 'primary_key', None) - + self.db_connection.merge_into_table( schema_name=self.model.destination_schema_name, target_table_name=self.model.destination_table_name, @@ -123,16 +123,16 @@ def _full_refresh_table(self) -> None: """Perform full refresh using existing table transformation methods.""" try: logging.info(f"Starting full refresh for {self.model.destination_schema_name}.{self.model.destination_table_name}") - + # Use BigQuery's create_or_replace_table which handles full refresh self.db_connection.create_or_replace_table( schema_name=self.model.destination_schema_name, table_name=self.model.destination_table_name, select_statement=self.model.select_statement, ) - + logging.info(f"Full refresh completed for {self.model.destination_schema_name}.{self.model.destination_table_name}") - + except Exception as e: logging.error(f"Full refresh failed for {self.model.destination_schema_name}.{self.model.destination_table_name}: {str(e)}") raise Exception( diff --git a/backend/visitran/adapters/model.py b/backend/visitran/adapters/model.py index c5990fd..2a7b46a 100644 --- a/backend/visitran/adapters/model.py +++ b/backend/visitran/adapters/model.py @@ -59,27 +59,27 @@ def execute_incremental(self) -> None: def _has_schema_changed(self) -> bool: """Detect if schema has changed significantly. - + This method compares the current table columns with the new SELECT statement columns to determine if a full refresh is needed due to schema changes. - + Returns: True if schema has changed significantly, False otherwise """ try: # Get current table columns current_columns = set(self.db_connection.get_table_columns( - schema_name=self.model.destination_schema_name, + schema_name=self.model.destination_schema_name, table_name=self.model.destination_table_name )) - + # Get new columns from SELECT statement new_columns = set(self.model.select_statement.columns) - + # Check for changes added_columns = new_columns - current_columns removed_columns = current_columns - new_columns - + # Log schema change details if added_columns or removed_columns: logging.info(f"Schema change detected for {self.model.destination_schema_name}.{self.model.destination_table_name}") @@ -88,9 +88,9 @@ def _has_schema_changed(self) -> bool: if removed_columns: logging.info(f" Removed columns: {list(removed_columns)}") return True - + return False - + except Exception as e: # If we can't determine schema, assume it changed (safe default) logging.warning(f"Could not determine schema for {self.model.destination_schema_name}.{self.model.destination_table_name}: {str(e)}") diff --git a/backend/visitran/adapters/postgres/connection.py b/backend/visitran/adapters/postgres/connection.py index b0a177f..beebff6 100644 --- a/backend/visitran/adapters/postgres/connection.py +++ b/backend/visitran/adapters/postgres/connection.py @@ -206,23 +206,23 @@ def upsert_into_table( primary_key: Union[str, list[str]], ) -> None: """Efficient upsert using PostgreSQL's INSERT ... ON CONFLICT. - + This approach is optimal for PostgreSQL because: 1. PostgreSQL's INSERT ... ON CONFLICT is highly efficient 2. No temporary tables needed 3. Atomic operation 4. Better performance than MERGE for PostgreSQL """ - + # Handle both single column and composite keys if isinstance(primary_key, str): key_columns = [primary_key] else: key_columns = primary_key - + # Get target table columns target_columns = self.get_table_columns(schema_name=schema_name, table_name=table_name) - + # Ensure unique constraint exists on primary key columns try: self._ensure_unique_constraint(schema_name, table_name, key_columns) @@ -233,7 +233,7 @@ def upsert_into_table( else: self._fallback_upsert(schema_name, table_name, select_statement, key_columns) return - + qi = self.quote_identifier # Build the ON CONFLICT clause @@ -254,14 +254,14 @@ def upsert_into_table( ON CONFLICT ({conflict_columns}) DO UPDATE SET {update_set_clause} """ - + # Execute the upsert self.connection.raw_sql(upsert_query) - - + + def _ensure_unique_constraint(self, schema_name: str, table_name: str, key_columns: list[str]) -> None: """Ensure a unique constraint exists on the specified columns.""" try: @@ -275,16 +275,16 @@ def _ensure_unique_constraint(self, schema_name: str, table_name: str, key_colum ALTER TABLE {qi(schema_name)}.{qi(table_name)} ADD CONSTRAINT {qi(constraint_name)} UNIQUE ({constraint_columns}) """ - + self.connection.raw_sql(add_constraint_sql) - + except Exception as e: # If constraint already exists, continue; otherwise bubble up for caller to handle if "already exists" in str(e).lower(): pass else: raise - + def _fallback_upsert(self, schema_name: str, table_name: str, select_statement: "Table", key_columns: list[str]) -> None: """Fallback upsert using DELETE + INSERT for tables without unique constraints.""" qi = self.quote_identifier @@ -313,6 +313,6 @@ def _fallback_upsert(self, schema_name: str, table_name: str, select_statement: ({', '.join([qi(col) for col in columns])}) {compiled_select}; """ - + # Execute the fallback upsert self.connection.raw_sql(fallback_query) diff --git a/backend/visitran/adapters/postgres/model.py b/backend/visitran/adapters/postgres/model.py index ecf2f67..d0abc94 100644 --- a/backend/visitran/adapters/postgres/model.py +++ b/backend/visitran/adapters/postgres/model.py @@ -78,10 +78,10 @@ def execute_incremental(self) -> None: self.model.select_statement = self.model.select_if_incremental() # Continue with incremental logic if no schema changes logging.info(f"No schema changes detected for {self.model.destination_schema_name}.{self.model.destination_table_name}, using incremental update") - + # Get primary key for upsert primary_key = getattr(self.model, 'primary_key', None) - + if primary_key: # MERGE mode: Upsert with primary key (updates existing, inserts new) logging.info(f"Incremental MERGE mode: upserting with primary_key={primary_key}") @@ -107,10 +107,10 @@ def execute_incremental(self) -> None: self.model.destination_table_name, ) ) - + # Get all data for first run self.model.select_statement = self.model.select() - + # Create table with all data self.db_connection.drop_table_if_exist( table_name=self.model.destination_table_name, @@ -135,13 +135,13 @@ def _full_refresh_table(self) -> None: """Perform full refresh using existing table transformation methods.""" try: logging.info(f"Starting full refresh for {self.model.destination_schema_name}.{self.model.destination_table_name}") - + # Drop existing table self.db_connection.drop_table_if_exist( schema_name=self.model.destination_schema_name, table_name=self.model.destination_table_name, ) - + # Create new table with current transformation logic # Note: create_table might already be populating data (CREATE TABLE ... AS SELECT ...) self.db_connection.create_table( @@ -149,9 +149,9 @@ def _full_refresh_table(self) -> None: table_name=self.model.destination_table_name, table_statement=self.model.select_statement, ) - + logging.info(f"Full refresh completed for {self.model.destination_schema_name}.{self.model.destination_table_name}") - + except Exception as e: logging.error(f"Full refresh failed for {self.model.destination_schema_name}.{self.model.destination_table_name}: {str(e)}") raise Exception( diff --git a/backend/visitran/adapters/seed.py b/backend/visitran/adapters/seed.py index c986c0b..a081d64 100644 --- a/backend/visitran/adapters/seed.py +++ b/backend/visitran/adapters/seed.py @@ -56,16 +56,16 @@ def get_csv_table(self) -> Table: cleaned_col = col.strip().replace(" ", "_") cleaned_col = cleaned_col.replace('"', "") cleaned_col = cleaned_col.replace("'", "") - + # Check if this looks like a date column (contains forward slashes) if "/" in cleaned_col: # For date-like columns, replace slashes with underscores to avoid duplicates # e.g., "1/22/20" becomes "1_22_20", "12/2/20" becomes "12_2_20" cleaned_col = cleaned_col.replace("/", "_") - + # Remove any remaining non-alphanumeric characters except underscores cleaned_col = re.sub(r"[^a-zA-Z0-9_]", "", cleaned_col).strip() - + if not cleaned_col or cleaned_col.strip() == "": raise InvalidCSVHeaders(csv_file_name=self.csv_file_name, column_name=col) diff --git a/backend/visitran/adapters/snowflake/connection.py b/backend/visitran/adapters/snowflake/connection.py index 9ea8469..d9df64b 100644 --- a/backend/visitran/adapters/snowflake/connection.py +++ b/backend/visitran/adapters/snowflake/connection.py @@ -126,7 +126,7 @@ def connection(self) -> Backend: def list_all_schemas(self) -> list[str]: sql_query = """ - SELECT + SELECT schema_name FROM information_schema.schemata WHERE schema_name NOT IN ('INFORMATION_SCHEMA') @@ -281,7 +281,7 @@ def upsert_into_table( primary_key: Union[str, list[str]], ) -> None: """Efficient upsert using Snowflake's MERGE INTO statement. - + This approach is optimal for Snowflake because: 1. MERGE INTO is natively supported and optimized 2. Atomic operation with ACID properties @@ -293,13 +293,13 @@ def upsert_into_table( key_columns = [primary_key] else: key_columns = primary_key - + # Get target table columns target_columns = self.get_table_columns(schema_name=schema_name, table_name=table_name) - + # Create temporary table name temp_table_name = f"{table_name}__temp" - + qi = self.quote_identifier try: diff --git a/backend/visitran/adapters/snowflake/db_reader.py b/backend/visitran/adapters/snowflake/db_reader.py index 28a64ba..255f869 100644 --- a/backend/visitran/adapters/snowflake/db_reader.py +++ b/backend/visitran/adapters/snowflake/db_reader.py @@ -31,33 +31,33 @@ def execute(self, existing_db_metadata: str = "") -> Dict[str, Any]: return self._cache logging.info("Building fresh database metadata tree...") - + try: # Use SQLAlchemy inspector for faster schema/table discovery schemas = self.inspector.get_schema_names() result = {"schemas": schemas, "tables": {}} - + # Process schemas in parallel for better performance with concurrent.futures.ThreadPoolExecutor(max_workers=4) as executor: # Submit schema scanning tasks future_to_schema = { - executor.submit(self._scan_schema_tables, schema): schema + executor.submit(self._scan_schema_tables, schema): schema for schema in schemas } - + # Collect results as they complete for future in concurrent.futures.as_completed(future_to_schema): schema, tables_info = future.result() if tables_info: result["tables"].update(tables_info) - + # Cache the result self._cache = result self._cache_timestamp = current_time - + logging.info(f"Database metadata tree built successfully: {len(schemas)} schemas, {len(result['tables'])} tables") return result - + except Exception as e: logging.error(f"Error building database metadata tree: {e}") # Fallback to base implementation if inspector fails @@ -71,24 +71,24 @@ def _scan_schema_tables(self, schema: str) -> tuple[str, Dict[str, Any]]: """ try: tables_info = {} - + # Get tables for this schema using inspector (faster) tables = self.inspector.get_table_names(schema=schema) - + for table in tables: try: # Get table info using optimized method table_info = self._get_optimized_table_info(schema, table) if table_info: tables_info[table] = table_info - + except Exception as table_error: logging.warning(f"Error getting info for table {schema}.{table}: {table_error}") # Continue with other tables continue - + return schema, tables_info - + except Exception as schema_error: logging.error(f"Error scanning schema {schema}: {schema_error}") return schema, {} @@ -101,11 +101,11 @@ def _get_optimized_table_info(self, schema: str, table: str) -> Dict[str, Any]: try: # Get columns using inspector (faster than raw SQL) columns_info = self.inspector.get_columns(table, schema=schema) - + # Get primary key info primary_keys = self.inspector.get_pk_constraint(table, schema=schema) pk_columns = primary_keys.get('constrained_columns', []) - + # Build column information columns = [] for col in columns_info: @@ -117,7 +117,7 @@ def _get_optimized_table_info(self, schema: str, table: str) -> Dict[str, Any]: "primary_key": col['name'] in pk_columns } columns.append(column_info) - + # Get table size info if available (optional) table_size = None try: @@ -130,7 +130,7 @@ def _get_optimized_table_info(self, schema: str, table: str) -> Dict[str, Any]: except: # Ignore size query errors, not critical pass - + return { "name": table, "schema_name": schema, @@ -139,7 +139,7 @@ def _get_optimized_table_info(self, schema: str, table: str) -> Dict[str, Any]: "row_count": table_size, "last_updated": time.time() } - + except Exception as e: logging.error(f"Error getting optimized table info for {schema}.{table}: {e}") # Fallback to base method if inspector fails @@ -163,16 +163,16 @@ def get_schema_summary(self) -> Dict[str, Any]: try: schemas = self.inspector.get_schema_names() summary = {"schemas": schemas, "table_counts": {}} - + for schema in schemas: try: tables = self.inspector.get_table_names(schema=schema) summary["table_counts"][schema] = len(tables) except: summary["table_counts"][schema] = 0 - + return summary - + except Exception as e: logging.error(f"Error getting schema summary: {e}") return {"schemas": [], "table_counts": {}} diff --git a/backend/visitran/adapters/snowflake/model.py b/backend/visitran/adapters/snowflake/model.py index 18f311b..3018f14 100644 --- a/backend/visitran/adapters/snowflake/model.py +++ b/backend/visitran/adapters/snowflake/model.py @@ -69,22 +69,22 @@ def execute_incremental(self) -> None: self.model.destination_table_name, ) ) - + # Get incremental data self.model.select_statement = self.model.select_if_incremental() - + # Check for schema changes first if self._has_schema_changed(): logging.info(f"Schema change detected for {self.model.destination_schema_name}.{self.model.destination_table_name}, performing full refresh") self._full_refresh_table() return - + # Continue with incremental logic if no schema changes logging.info(f"No schema changes detected for {self.model.destination_schema_name}.{self.model.destination_table_name}, using incremental update") - + # Get primary key for upsert primary_key = getattr(self.model, 'primary_key', None) - + if primary_key: # MERGE mode: Upsert with primary key (updates existing, inserts new) logging.info(f"Incremental MERGE mode: upserting with primary_key={primary_key}") @@ -110,10 +110,10 @@ def execute_incremental(self) -> None: self.model.destination_table_name, ) ) - + # Get all data for first run self.model.select_statement = self.model.select() - + # Create table with all data self.db_connection.drop_table_if_exist( table_name=self.model.destination_table_name, @@ -138,22 +138,22 @@ def _full_refresh_table(self) -> None: """Perform full refresh using existing table transformation methods.""" try: logging.info(f"Starting full refresh for {self.model.destination_schema_name}.{self.model.destination_table_name}") - + # Drop existing table self.db_connection.drop_table_if_exist( table_name=self.model.destination_table_name, schema_name=self.model.destination_schema_name, ) - + # Create new table with current transformation logic self.db_connection.create_table( table_name=self.model.destination_table_name, table_statement=self.model.select_statement, schema_name=self.model.destination_schema_name, ) - + logging.info(f"Full refresh completed for {self.model.destination_schema_name}.{self.model.destination_table_name}") - + except Exception as e: logging.error(f"Full refresh failed for {self.model.destination_schema_name}.{self.model.destination_table_name}: {str(e)}") raise Exception( diff --git a/backend/visitran/adapters/trino/model.py b/backend/visitran/adapters/trino/model.py index 13292bf..c7b045e 100644 --- a/backend/visitran/adapters/trino/model.py +++ b/backend/visitran/adapters/trino/model.py @@ -67,20 +67,20 @@ def execute_incremental(self) -> None: self.model.destination_table_name, ) ) - + # Check for schema changes first if self._has_schema_changed(): logging.info(f"Schema change detected for {self.model.destination_schema_name}.{self.model.destination_table_name}, performing full refresh") self._full_refresh_table() else: self.model.select_statement = self.model.select_if_incremental() - + # Continue with incremental logic if no schema changes logging.info(f"No schema changes detected for {self.model.destination_schema_name}.{self.model.destination_table_name}, using incremental update") - + # Get primary key for upsert primary_key = getattr(self.model, 'primary_key', None) - + if primary_key: # MERGE mode: Upsert with primary key (updates existing, inserts new) logging.info(f"Incremental MERGE mode: upserting with primary_key={primary_key}") @@ -106,10 +106,10 @@ def execute_incremental(self) -> None: self.model.destination_table_name, ) ) - + # Get all data for first run self.model.select_statement = self.model.select() - + # Create table with all data self.db_connection.drop_table_if_exist( schema_name=self.model.destination_schema_name, @@ -134,22 +134,22 @@ def _full_refresh_table(self) -> None: """Perform full refresh using existing table transformation methods.""" try: logging.info(f"Starting full refresh for {self.model.destination_schema_name}.{self.model.destination_table_name}") - + # Drop existing table self.db_connection.drop_table_if_exist( schema_name=self.model.destination_schema_name, table_name=self.model.destination_table_name, ) - + # Create new table with current transformation logic self.db_connection.create_table( schema_name=self.model.destination_schema_name, table_name=self.model.destination_table_name, table_statement=self.model.select_statement, ) - + logging.info(f"Full refresh completed for {self.model.destination_schema_name}.{self.model.destination_table_name}") - + except Exception as e: logging.error(f"Full refresh failed for {self.model.destination_schema_name}.{self.model.destination_table_name}: {str(e)}") raise Exception( diff --git a/backend/visitran/templates/delta_strategies.py b/backend/visitran/templates/delta_strategies.py index 46e8b3d..ee896db 100644 --- a/backend/visitran/templates/delta_strategies.py +++ b/backend/visitran/templates/delta_strategies.py @@ -13,9 +13,9 @@ class DeltaStrategy(ABC): """Abstract base class for delta detection strategies.""" - + @abstractmethod - def get_incremental_data(self, source_table: Table, destination_table: Table, + def get_incremental_data(self, source_table: Table, destination_table: Table, strategy_config: Dict[str, Any]) -> Table: """Return incremental data based on the strategy.""" pass @@ -23,87 +23,87 @@ def get_incremental_data(self, source_table: Table, destination_table: Table, class TimestampStrategy(DeltaStrategy): """Strategy using timestamp columns (e.g., updated_at, modified_at).""" - - def get_incremental_data(self, source_table: Table, destination_table: Table, + + def get_incremental_data(self, source_table: Table, destination_table: Table, strategy_config: Dict[str, Any]) -> Table: """Get records updated since the last run using timestamp column.""" timestamp_column = strategy_config.get("column", "updated_at") - + # Get the latest timestamp from destination table latest_timestamp = destination_table[timestamp_column].max().name("latest_timestamp") - + # Filter source table for records newer than the latest timestamp # Return the final incremental data ready for processing incremental_data = source_table.filter( source_table[timestamp_column] > latest_timestamp ) - + return incremental_data class DateStrategy(DeltaStrategy): """Strategy using date columns (e.g., created_date, snapshot_date).""" - - def get_incremental_data(self, source_table: Table, destination_table: Table, + + def get_incremental_data(self, source_table: Table, destination_table: Table, strategy_config: Dict[str, Any]) -> Table: """Get records for dates after the latest date in destination.""" date_column = strategy_config.get("column", "created_date") - + # Get the latest date from destination table latest_date = destination_table[date_column].max().name("latest_date") - + # Filter source table for records with dates after the latest date # Return the final incremental data ready for processing incremental_data = source_table.filter( source_table[date_column] > latest_date ) - + return incremental_data class SequenceStrategy(DeltaStrategy): """Strategy using sequence/ID columns (e.g., id, sequence_number).""" - - def get_incremental_data(self, source_table: Table, destination_table: Table, + + def get_incremental_data(self, source_table: Table, destination_table: Table, strategy_config: Dict[str, Any]) -> Table: """Get records with sequence numbers higher than the maximum in destination.""" sequence_column = strategy_config.get("column", "id") - + # Get the maximum sequence number from destination table max_sequence = destination_table[sequence_column].max().name("max_sequence") - + # Filter source table for records with higher sequence numbers # Return the final incremental data ready for processing incremental_data = source_table.filter( source_table[sequence_column] > max_sequence ) - + return incremental_data class ChecksumStrategy(DeltaStrategy): """Strategy using checksum/hash columns to detect changes.""" - - def get_incremental_data(self, source_table: Table, destination_table: Table, + + def get_incremental_data(self, source_table: Table, destination_table: Table, strategy_config: Dict[str, Any]) -> Table: """Get records where checksum differs from destination.""" checksum_column = strategy_config.get("column", "checksum") key_columns = strategy_config.get("key_columns", []) - + if not key_columns: raise ValueError("Checksum strategy requires key_columns configuration") - + # Join source and destination on key columns to compare checksums # This is a simplified version - in practice, you'd need more complex logic incremental_data = source_table - + return incremental_data class FullScanStrategy(DeltaStrategy): """Strategy that compares all records to detect changes (expensive but comprehensive).""" - - def get_incremental_data(self, source_table: Table, destination_table: Table, + + def get_incremental_data(self, source_table: Table, destination_table: Table, strategy_config: Dict[str, Any]) -> Table: """Get all records from source table for full comparison.""" # This strategy returns all source data for comparison @@ -113,22 +113,22 @@ def get_incremental_data(self, source_table: Table, destination_table: Table, class CustomStrategy(DeltaStrategy): """Strategy using custom logic provided by the user.""" - - def get_incremental_data(self, source_table: Table, destination_table: Table, + + def get_incremental_data(self, source_table: Table, destination_table: Table, strategy_config: Dict[str, Any]) -> Table: """Execute custom logic to determine incremental data.""" custom_logic = strategy_config.get("custom_logic") - + if not custom_logic or not callable(custom_logic): raise ValueError("Custom strategy requires a callable custom_logic function") - + # Execute custom logic return custom_logic(source_table, destination_table, strategy_config) class DeltaStrategyFactory: """Factory class for creating delta detection strategies.""" - + _strategies = { "timestamp": TimestampStrategy(), "date": DateStrategy(), @@ -137,15 +137,15 @@ class DeltaStrategyFactory: "full_scan": FullScanStrategy(), "custom": CustomStrategy(), } - + @classmethod def get_strategy(cls, strategy_type: str) -> DeltaStrategy: """Get a delta strategy by type.""" if strategy_type not in cls._strategies: raise ValueError(f"Unknown delta strategy: {strategy_type}") - + return cls._strategies[strategy_type] - + @classmethod def get_available_strategies(cls) -> list[str]: """Get list of available strategy types.""" diff --git a/backend/visitran/templates/model.py b/backend/visitran/templates/model.py index d296225..2526bd8 100644 --- a/backend/visitran/templates/model.py +++ b/backend/visitran/templates/model.py @@ -56,12 +56,12 @@ def __init__(self) -> None: # class with incremental materialization # this is get only self.destination_table_exists: bool = False - + # Primary key for efficient upserts (especially for BigQuery) # This should be set to the column name(s) that uniquely identify records # Can be a single column name (str) or list of column names for composite keys self.primary_key: Union[str, list[str]] = "" - + # Delta detection strategy for incremental processing # This defines how to identify new/changed records for incremental updates self.delta_strategy: dict[str, Any] = { @@ -144,32 +144,32 @@ def incremental_mode(self) -> str: if self.primary_key: return 'merge' return 'append' - + def _validate_delta_strategy_config(self) -> None: """Validate delta strategy configuration.""" strategy_type = self.delta_strategy.get("type") - + if strategy_type == "timestamp": if not self.delta_strategy.get("column"): raise ValueError( f"Timestamp strategy requires 'column' configuration. " f"Example: create_timestamp_strategy(column='updated_at')" ) - + elif strategy_type == "date": if not self.delta_strategy.get("column"): raise ValueError( f"Date strategy requires 'column' configuration. " f"Example: create_date_strategy(column='created_date')" ) - + elif strategy_type == "sequence": if not self.delta_strategy.get("column"): raise ValueError( f"Sequence strategy requires 'column' configuration. " f"Example: create_sequence_strategy(column='id')" ) - + elif strategy_type == "checksum": if not self.delta_strategy.get("column"): raise ValueError( @@ -181,7 +181,7 @@ def _validate_delta_strategy_config(self) -> None: f"Checksum strategy requires 'key_columns' configuration. " f"Example: create_checksum_strategy(checksum_column='content_hash', key_columns=['product_id'])" ) - + elif strategy_type == "custom": if not self.delta_strategy.get("custom_logic"): raise ValueError( @@ -192,41 +192,41 @@ def _validate_delta_strategy_config(self) -> None: raise ValueError( f"Custom strategy 'custom_logic' must be a callable function." ) - + elif strategy_type == "full_scan": # No additional validation needed for full scan pass - + else: raise ValueError( f"Unknown delta strategy type: {strategy_type}. " f"Available strategies: {DeltaStrategyFactory.get_available_strategies()}" ) - + def _execute_delta_strategy(self) -> Table: """Execute the configured delta strategy to get incremental data.""" if not self.destination_table_exists: # First run - return all data return self.select() - + # Get the delta strategy strategy_type = self.delta_strategy["type"] strategy = DeltaStrategyFactory.get_strategy(strategy_type) - + # Execute the strategy source_table = self.select() destination_table = self.destination_table_obj - + # Get incremental data from strategy incremental_data = strategy.get_incremental_data( source_table=source_table, destination_table=destination_table, strategy_config=self.delta_strategy ) - + # Return incremental data as-is (no additional transformation needed) return incremental_data - + def __str__(self) -> str: return f"{self.destination_schema_name}.{self.destination_table_name}" diff --git a/backend/visitran/visitran_context.py b/backend/visitran/visitran_context.py index 5a514bf..df02645 100644 --- a/backend/visitran/visitran_context.py +++ b/backend/visitran/visitran_context.py @@ -47,10 +47,10 @@ def __load_db_adapter(self) -> BaseAdapter: def __update_connection_details(self, env_data: dict[str, Any]): if self._db_type == "duckdb": return self._db_details.copy() - + # Create a copy to avoid modifying the original project config conn_details = self._db_details.copy() - + if env_data: conn_details.update(env_data) elif self.project_conf.get("project_schema"): diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index 63c0fb6..422ca4b 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -40,7 +40,8 @@ services: - postgres - redis healthcheck: - test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:8000/api/v1/health')"] + test: ["CMD", "python", "-c", + "import urllib.request; urllib.request.urlopen('http://localhost:8000/api/v1/health')"] interval: 10s timeout: 5s retries: 12 diff --git a/docker/dockerfiles/backend.Dockerfile.dockerignore b/docker/dockerfiles/backend.Dockerfile.dockerignore index b5964ef..35317a8 100644 --- a/docker/dockerfiles/backend.Dockerfile.dockerignore +++ b/docker/dockerfiles/backend.Dockerfile.dockerignore @@ -1,57 +1,57 @@ -**/__pycache__ -**/.pytest_cache -**/.python-version -**/.pyc -**/.pyo -**/.venv -**/.classpath -**/.dockerignore -**/.env -**/.git -**/.gitignore -**/.gitkeep -**/.project -**/.settings -**/.toolstarget -**/.vs -**/.vscode -**/*.*proj.user -**/*.dbmdl -**/*.jfm -**/bin -**/charts -**/docker-compose* -**/compose* -**/Dockerfile* -**/build -**/dist -**/node_modules -**/npm-debug.log -**/obj -**/secrets.dev.yaml -**/values.dev.yaml -**/.db -**/.sqlite3 -**/.log -**/*-log.txt -**/*.drawio -**/.tmp -**/.swp -**/.swo -**/.bak -*.idea -*.vscode -*.git -**/.pdm.toml -**/.pdm-build -**/.pdm-python -!LICENSE -*.md -!README.md -.jshintrc -.pre-commit-config.yaml -**/tests -test*.py - -tools - +**/__pycache__ +**/.pytest_cache +**/.python-version +**/.pyc +**/.pyo +**/.venv +**/.classpath +**/.dockerignore +**/.env +**/.git +**/.gitignore +**/.gitkeep +**/.project +**/.settings +**/.toolstarget +**/.vs +**/.vscode +**/*.*proj.user +**/*.dbmdl +**/*.jfm +**/bin +**/charts +**/docker-compose* +**/compose* +**/Dockerfile* +**/build +**/dist +**/node_modules +**/npm-debug.log +**/obj +**/secrets.dev.yaml +**/values.dev.yaml +**/.db +**/.sqlite3 +**/.log +**/*-log.txt +**/*.drawio +**/.tmp +**/.swp +**/.swo +**/.bak +*.idea +*.vscode +*.git +**/.pdm.toml +**/.pdm-build +**/.pdm-python +!LICENSE +*.md +!README.md +.jshintrc +.pre-commit-config.yaml +**/tests +test*.py + +tools + diff --git a/frontend/nginx.conf b/frontend/nginx.conf index 339d53c..a1a30be 100644 --- a/frontend/nginx.conf +++ b/frontend/nginx.conf @@ -28,7 +28,7 @@ http { keepalive_timeout 120; client_max_body_size 150M; - + gzip on; # Non-root user will not have access to default @@ -40,10 +40,10 @@ http { scgi_temp_path /tmp/scgi_temp 1 2; # Extend timeouts - proxy_connect_timeout 240s; - proxy_send_timeout 240s; + proxy_connect_timeout 240s; + proxy_send_timeout 240s; proxy_read_timeout 240s; - send_timeout 240s; + send_timeout 240s; server { listen 80; diff --git a/tests/unit/test_incremental_validation.py b/tests/unit/test_incremental_validation.py index eef5947..36ee53a 100644 --- a/tests/unit/test_incremental_validation.py +++ b/tests/unit/test_incremental_validation.py @@ -10,99 +10,99 @@ class ValidIncrementalModel(VisitranModel): """Valid incremental model with proper configuration.""" - + def __init__(self): super().__init__() self.materialization = Materialization.INCREMENTAL self.primary_key = "user_id" self.delta_strategy = create_timestamp_strategy(column="updated_at") - + def select(self): return None class InvalidIncrementalModelNoPrimaryKey(VisitranModel): """Invalid incremental model missing primary key.""" - + def __init__(self): super().__init__() self.materialization = Materialization.INCREMENTAL # Missing primary_key self.delta_strategy = create_timestamp_strategy(column="updated_at") - + def select(self): return None class InvalidIncrementalModelNoDeltaStrategy(VisitranModel): """Invalid incremental model missing delta strategy.""" - + def __init__(self): super().__init__() self.materialization = Materialization.INCREMENTAL self.primary_key = "user_id" # Missing delta_strategy - + def select(self): return None class InvalidIncrementalModelInvalidStrategy(VisitranModel): """Invalid incremental model with invalid delta strategy.""" - + def __init__(self): super().__init__() self.materialization = Materialization.INCREMENTAL self.primary_key = "user_id" self.delta_strategy = {"type": "invalid_strategy"} - + def select(self): return None class TestIncrementalValidation: """Test incremental model validation.""" - + def test_valid_incremental_model(self): """Test that valid incremental model passes validation.""" model = ValidIncrementalModel() # Should not raise any exceptions model._validate_incremental_config() - + def test_invalid_model_no_primary_key(self): """Test that model without primary key raises error.""" model = InvalidIncrementalModelNoPrimaryKey() - + with pytest.raises(ValueError) as exc_info: model._validate_incremental_config() - + assert "Primary key is required" in str(exc_info.value) assert "self.primary_key" in str(exc_info.value) - + def test_invalid_model_no_delta_strategy(self): """Test that model without delta strategy raises error.""" model = InvalidIncrementalModelNoDeltaStrategy() - + with pytest.raises(ValueError) as exc_info: model._validate_incremental_config() - + assert "Delta strategy is required" in str(exc_info.value) assert "self.delta_strategy" in str(exc_info.value) - + def test_invalid_model_invalid_strategy(self): """Test that model with invalid strategy type raises error.""" model = InvalidIncrementalModelInvalidStrategy() - + with pytest.raises(ValueError) as exc_info: model._validate_incremental_config() - + assert "Unknown delta strategy type" in str(exc_info.value) assert "invalid_strategy" in str(exc_info.value) - + def test_non_incremental_model_no_validation(self): """Test that non-incremental models don't require validation.""" model = VisitranModel() model.materialization = Materialization.TABLE # Not incremental - + # Should not raise any exceptions model._validate_incremental_config()