feat(package)!: Add Spider compression orchestration in docker compose package.

components/clp-package-utils/clp_package_utils/controller.py

@@ -13,6 +13,11 @@
 from clp_py_utils.clp_config import (
     API_SERVER_COMPONENT_NAME,
     AwsAuthType,
+    CLP_DB_PASS_ENV_VAR_NAME,
+    CLP_DB_USER_ENV_VAR_NAME,
+    CLP_QUEUE_PASS_ENV_VAR_NAME,
+    CLP_QUEUE_USER_ENV_VAR_NAME,
+    CLP_REDIS_PASS_ENV_VAR_NAME,
     ClpConfig,
     COMPRESSION_JOBS_TABLE_NAME,
     COMPRESSION_SCHEDULER_COMPONENT_NAME,
@@ -21,6 +26,7 @@
     DeploymentType,
     GARBAGE_COLLECTOR_COMPONENT_NAME,
     MCP_SERVER_COMPONENT_NAME,
+    OrchestrationType,
     QUERY_JOBS_TABLE_NAME,
     QUERY_SCHEDULER_COMPONENT_NAME,
     QUERY_WORKER_COMPONENT_NAME,
@@ -29,6 +35,10 @@
     REDIS_COMPONENT_NAME,
     REDUCER_COMPONENT_NAME,
     RESULTS_CACHE_COMPONENT_NAME,
+    SPIDER_DB_COMPONENT_NAME,
+    SPIDER_DB_PASS_ENV_VAR_NAME,
+    SPIDER_DB_USER_ENV_VAR_NAME,
+    SPIDER_SCHEDULER_COMPONENT_NAME,
     StorageEngine,
     StorageType,
     WEBUI_COMPONENT_NAME,
@@ -139,8 +149,8 @@ def _set_up_env_for_database(self) -> EnvVarsDict:
 
         # Credentials
         env_vars |= {
-            "CLP_DB_PASS": self._clp_config.database.password,
-            "CLP_DB_USER": self._clp_config.database.username,
+            CLP_DB_PASS_ENV_VAR_NAME: self._clp_config.database.password,
+            CLP_DB_USER_ENV_VAR_NAME: self._clp_config.database.username,
         }
 
         # Paths
@@ -185,8 +195,8 @@ def _set_up_env_for_queue(self) -> EnvVarsDict:
 
         # Credentials
         env_vars |= {
-            "CLP_QUEUE_PASS": self._clp_config.queue.password,
-            "CLP_QUEUE_USER": self._clp_config.queue.username,
+            CLP_QUEUE_PASS_ENV_VAR_NAME: self._clp_config.queue.password,
+            CLP_QUEUE_USER_ENV_VAR_NAME: self._clp_config.queue.username,
         }
 
         # Paths
@@ -235,7 +245,7 @@ def _set_up_env_for_redis(self) -> EnvVarsDict:
 
         # Credentials
         env_vars |= {
-            "CLP_REDIS_PASS": self._clp_config.redis.password,
+            CLP_REDIS_PASS_ENV_VAR_NAME: self._clp_config.redis.password,
         }
 
         # Paths
@@ -247,6 +257,47 @@ def _set_up_env_for_redis(self) -> EnvVarsDict:
 
         return env_vars
 
+    def _set_up_env_for_spider_db(self) -> EnvVarsDict:
+        """
+        Sets up environment variables for the Spider database component.
+
+        :return: Dictionary of environment variables necessary to launch the component.
+        """
+        component_name = SPIDER_DB_COMPONENT_NAME
+        logger.info(f"Setting up environment for {component_name}...")
+
+        env_vars = EnvVarsDict()
+
+        # Database
+        env_vars |= {"SPIDER_DB_NAME": self._clp_config.spider_db.name}
+
+        # Credentials
+        env_vars |= {
+            SPIDER_DB_PASS_ENV_VAR_NAME: self._clp_config.spider_db.password,
+            SPIDER_DB_USER_ENV_VAR_NAME: self._clp_config.spider_db.username,
+        }
+
+        return env_vars
+
+    def _set_up_env_for_spider_scheduler(self) -> EnvVarsDict:
+        """
+        Sets up environment variables for the Spider scheduler component.
+
+        :return: Dictionary of environment variables necessary to launch the component.
+        """
+        component_name = SPIDER_SCHEDULER_COMPONENT_NAME
+        logger.info(f"Setting up environment for {component_name}...")
+
+        env_vars = EnvVarsDict()
+
+        # Connection config
+        env_vars |= {
+            "SPIDER_SCHEDULER_HOST": _get_ip_from_hostname(self._clp_config.spider_scheduler.host),
+            "SPIDER_SCHEDULER_PORT": str(self._clp_config.spider_scheduler.port),
+        }
+
+        return env_vars
+
     def _set_up_env_for_results_cache(self) -> EnvVarsDict:
         """
         Sets up environment variables and directories for the results cache (MongoDB) component.
@@ -766,8 +817,13 @@ def set_up_env(self) -> None:
 
         # Component-specific config
         env_vars |= self._set_up_env_for_database()
-        env_vars |= self._set_up_env_for_queue()
-        env_vars |= self._set_up_env_for_redis()
+        if self._clp_config.queue is not None:
+            env_vars |= self._set_up_env_for_queue()
+        if self._clp_config.redis is not None:
+            env_vars |= self._set_up_env_for_redis()
+        if self._clp_config.compression_scheduler.type == OrchestrationType.spider:
+            env_vars |= self._set_up_env_for_spider_db()
+            env_vars |= self._set_up_env_for_spider_scheduler()
         env_vars |= self._set_up_env_for_results_cache()
         env_vars |= self._set_up_env_for_compression_scheduler()
         env_vars |= self._set_up_env_for_query_scheduler()
@@ -855,8 +911,13 @@ def _get_docker_file_name(self) -> str:
         :return: The Docker Compose file name to use based on the config.
         """
         deployment_type = self._clp_config.get_deployment_type()
+        compression_scheduler_type = self._clp_config.compression_scheduler.type
         if deployment_type == DeploymentType.BASE:
-            return "docker-compose-base.yaml"
+            if compression_scheduler_type == OrchestrationType.spider:
+                return "docker-compose-spider-base.yaml"
+            return "docker-compose.base.yaml"
+        if compression_scheduler_type == OrchestrationType.spider:
+            return "docker-compose-spider.yaml"
         return "docker-compose.yaml"
 
 

components/clp-package-utils/clp_package_utils/general.py

@@ -25,6 +25,7 @@
     REDIS_COMPONENT_NAME,
     REDUCER_COMPONENT_NAME,
     RESULTS_CACHE_COMPONENT_NAME,
+    SPIDER_DB_COMPONENT_NAME,
     StorageType,
     WEBUI_COMPONENT_NAME,
     WorkerConfig,
@@ -454,6 +455,7 @@ def generate_credentials_file(credentials_file_path: pathlib.Path):
         DB_COMPONENT_NAME: {"username": "clp-user", "password": secrets.token_urlsafe(8)},
         QUEUE_COMPONENT_NAME: {"username": "clp-user", "password": secrets.token_urlsafe(8)},
         REDIS_COMPONENT_NAME: {"password": secrets.token_urlsafe(16)},
+        SPIDER_DB_COMPONENT_NAME: {"username": "spider-user", "password": secrets.token_urlsafe(8)},
     }
 
     with open(credentials_file_path, "w") as f:
@@ -499,6 +501,13 @@ def validate_and_load_redis_credentials_file(
     clp_config.redis.load_credentials_from_file(clp_config.credentials_file_path)
 
 
+def validate_and_load_spider_db_credentials_file(
+    clp_config: ClpConfig, clp_home: pathlib.Path, generate_default_file: bool
+):
+    validate_credentials_file_path(clp_config, clp_home, generate_default_file)
+    clp_config.spider_db.load_credentials_from_file(clp_config.credentials_file_path)
+
+
 def validate_db_config(
     clp_config: ClpConfig,
     component_config: pathlib.Path,

components/clp-package-utils/clp_package_utils/scripts/start_clp.py

@@ -3,7 +3,7 @@
 import pathlib
 import sys
 
-from clp_py_utils.clp_config import CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH
+from clp_py_utils.clp_config import CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH, OrchestrationType
 from clp_py_utils.core import resolve_host_path_in_container
 
 from clp_package_utils.controller import DockerComposeController, get_or_create_instance_id
@@ -13,6 +13,7 @@
     validate_and_load_db_credentials_file,
     validate_and_load_queue_credentials_file,
     validate_and_load_redis_credentials_file,
+    validate_and_load_spider_db_credentials_file,
     validate_output_storage_config,
     validate_retention_config,
 )
@@ -60,8 +61,12 @@ def main(argv):
         )
 
         validate_and_load_db_credentials_file(clp_config, clp_home, True)
-        validate_and_load_queue_credentials_file(clp_config, clp_home, True)
-        validate_and_load_redis_credentials_file(clp_config, clp_home, True)
+        if clp_config.queue is not None:
+            validate_and_load_queue_credentials_file(clp_config, clp_home, True)
+        if clp_config.redis is not None:
+            validate_and_load_redis_credentials_file(clp_config, clp_home, True)
+        if clp_config.compression_scheduler.type == OrchestrationType.spider:
+            validate_and_load_spider_db_credentials_file(clp_config, clp_home, True)
         clp_config.validate_logs_input_config(True)
         validate_output_storage_config(clp_config)
         validate_retention_config(clp_config)