diff --git a/components/clp-package-utils/clp_package_utils/controller.py b/components/clp-package-utils/clp_package_utils/controller.py index ed8bbfbae1..7328bd4319 100644 --- a/components/clp-package-utils/clp_package_utils/controller.py +++ b/components/clp-package-utils/clp_package_utils/controller.py @@ -14,10 +14,16 @@ API_SERVER_COMPONENT_NAME, AwsAuthType, BundledService, + CLP_DB_PASS_ENV_VAR_NAME, + CLP_DB_ROOT_PASS_ENV_VAR_NAME, + CLP_DB_ROOT_USER_ENV_VAR_NAME, + CLP_DB_USER_ENV_VAR_NAME, ClpConfig, + ClpDbUserType, COMPRESSION_JOBS_TABLE_NAME, COMPRESSION_SCHEDULER_COMPONENT_NAME, COMPRESSION_WORKER_COMPONENT_NAME, + DatabaseEngine, DB_COMPONENT_NAME, DeploymentType, GARBAGE_COLLECTOR_COMPONENT_NAME, @@ -150,7 +156,9 @@ def _set_up_env_for_database_bundling(self) -> EnvVarsDict: # Runtime config env_vars |= { "CLP_DB_CONTAINER_IMAGE_REF": ( - "mysql:8.0.23" if self._clp_config.database.type == "mysql" else "mariadb:10-jammy" + "mysql:8.0.23" + if self._clp_config.database.type == DatabaseEngine.MYSQL + else "mariadb:10-jammy" ), } @@ -175,9 +183,12 @@ def _set_up_env_for_database(self) -> EnvVarsDict: } # Credentials + credentials = self._clp_config.database.credentials env_vars |= { - "CLP_DB_PASS": self._clp_config.database.password, - "CLP_DB_USER": self._clp_config.database.username, + CLP_DB_PASS_ENV_VAR_NAME: credentials[ClpDbUserType.CLP].password, + CLP_DB_ROOT_PASS_ENV_VAR_NAME: credentials[ClpDbUserType.ROOT].password, + CLP_DB_ROOT_USER_ENV_VAR_NAME: credentials[ClpDbUserType.ROOT].username, + CLP_DB_USER_ENV_VAR_NAME: credentials[ClpDbUserType.CLP].username, } return env_vars diff --git a/components/clp-package-utils/clp_package_utils/general.py b/components/clp-package-utils/clp_package_utils/general.py index e7a47d9381..9e325ea593 100644 --- a/components/clp-package-utils/clp_package_utils/general.py +++ b/components/clp-package-utils/clp_package_utils/general.py @@ -451,7 +451,12 @@ def load_config_file( def generate_credentials_file(credentials_file_path: pathlib.Path): credentials = { - DB_COMPONENT_NAME: {"username": "clp-user", "password": secrets.token_urlsafe(8)}, + DB_COMPONENT_NAME: { + "username": "clp-user", + "password": secrets.token_urlsafe(8), + "root_username": "root", + "root_password": secrets.token_urlsafe(8), + }, QUEUE_COMPONENT_NAME: {"username": "clp-user", "password": secrets.token_urlsafe(8)}, REDIS_COMPONENT_NAME: {"password": secrets.token_urlsafe(16)}, } diff --git a/components/clp-package-utils/clp_package_utils/scripts/archive_manager.py b/components/clp-package-utils/clp_package_utils/scripts/archive_manager.py index e048fb793e..6f7a6a4251 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/archive_manager.py +++ b/components/clp-package-utils/clp_package_utils/scripts/archive_manager.py @@ -11,6 +11,7 @@ CLP_DB_USER_ENV_VAR_NAME, CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH, CLP_DEFAULT_DATASET_NAME, + ClpDbUserType, StorageEngine, StorageType, ) @@ -226,9 +227,10 @@ def main(argv: list[str]) -> int: mounts.logs_dir, mounts.archives_output_dir, ] + credentials = clp_config.database.credentials extra_env_vars = { - CLP_DB_USER_ENV_VAR_NAME: clp_config.database.username, - CLP_DB_PASS_ENV_VAR_NAME: clp_config.database.password, + CLP_DB_PASS_ENV_VAR_NAME: credentials[ClpDbUserType.CLP].password, + CLP_DB_USER_ENV_VAR_NAME: credentials[ClpDbUserType.CLP].username, } container_start_cmd: list[str] = generate_container_start_cmd( container_name, necessary_mounts, clp_config.container_image_ref, extra_env_vars diff --git a/components/clp-package-utils/clp_package_utils/scripts/compress.py b/components/clp-package-utils/clp_package_utils/scripts/compress.py index 089a3d0a3e..e93d086eb4 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/compress.py +++ b/components/clp-package-utils/clp_package_utils/scripts/compress.py @@ -11,6 +11,7 @@ CLP_DB_USER_ENV_VAR_NAME, CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH, CLP_DEFAULT_DATASET_NAME, + ClpDbUserType, StorageEngine, StorageType, ) @@ -252,9 +253,10 @@ def main(argv): logger.error("No filesystem paths given for compression.") return -1 + credentials = clp_config.database.credentials extra_env_vars = { - CLP_DB_USER_ENV_VAR_NAME: clp_config.database.username, - CLP_DB_PASS_ENV_VAR_NAME: clp_config.database.password, + CLP_DB_PASS_ENV_VAR_NAME: credentials[ClpDbUserType.CLP].password, + CLP_DB_USER_ENV_VAR_NAME: credentials[ClpDbUserType.CLP].username, } container_start_cmd = generate_container_start_cmd( container_name, necessary_mounts, clp_config.container_image_ref, extra_env_vars diff --git a/components/clp-package-utils/clp_package_utils/scripts/compress_from_s3.py b/components/clp-package-utils/clp_package_utils/scripts/compress_from_s3.py index 2469e07ef1..bc64716374 100644 --- a/components/clp-package-utils/clp_package_utils/scripts/compress_from_s3.py +++ b/components/clp-package-utils/clp_package_utils/scripts/compress_from_s3.py @@ -11,6 +11,7 @@ CLP_DB_USER_ENV_VAR_NAME, CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH, CLP_DEFAULT_DATASET_NAME, + ClpDbUserType, StorageEngine, StorageType, ) @@ -306,9 +307,10 @@ def main(argv): logger.error("No S3 URLs given for compression.") return -1 + credentials = clp_config.database.credentials extra_env_vars = { - CLP_DB_USER_ENV_VAR_NAME: clp_config.database.username, - CLP_DB_PASS_ENV_VAR_NAME: clp_config.database.password, + CLP_DB_PASS_ENV_VAR_NAME: credentials[ClpDbUserType.CLP].password, + CLP_DB_USER_ENV_VAR_NAME: credentials[ClpDbUserType.CLP].username, } container_start_cmd = generate_container_start_cmd( container_name, necessary_mounts, clp_config.container_image_ref, extra_env_vars diff --git a/components/clp-package-utils/clp_package_utils/scripts/dataset_manager.py b/components/clp-package-utils/clp_package_utils/scripts/dataset_manager.py index 9bbb4080ea..2737d937d7 100644 --- a/components/clp-package-utils/clp_package_utils/scripts/dataset_manager.py +++ b/components/clp-package-utils/clp_package_utils/scripts/dataset_manager.py @@ -11,6 +11,7 @@ CLP_DB_PASS_ENV_VAR_NAME, CLP_DB_USER_ENV_VAR_NAME, CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH, + ClpDbUserType, StorageEngine, StorageType, ) @@ -155,9 +156,10 @@ def main(argv: list[str]) -> int: if aws_mount: necessary_mounts.append(mounts.aws_config_dir) + credentials = clp_config.database.credentials extra_env_vars = { - CLP_DB_USER_ENV_VAR_NAME: clp_config.database.username, - CLP_DB_PASS_ENV_VAR_NAME: clp_config.database.password, + CLP_DB_PASS_ENV_VAR_NAME: credentials[ClpDbUserType.CLP].password, + CLP_DB_USER_ENV_VAR_NAME: credentials[ClpDbUserType.CLP].username, } container_start_cmd = generate_container_start_cmd( container_name, necessary_mounts, clp_config.container_image_ref, extra_env_vars diff --git a/components/clp-package-utils/clp_package_utils/scripts/decompress.py b/components/clp-package-utils/clp_package_utils/scripts/decompress.py index 43fff95620..f0aa6e4657 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/decompress.py +++ b/components/clp-package-utils/clp_package_utils/scripts/decompress.py @@ -11,6 +11,7 @@ CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH, CLP_DEFAULT_DATASET_NAME, ClpConfig, + ClpDbUserType, StorageEngine, StorageType, ) @@ -132,9 +133,10 @@ def handle_extract_file_cmd( ) ) + credentials = clp_config.database.credentials extra_env_vars = { - CLP_DB_USER_ENV_VAR_NAME: clp_config.database.username, - CLP_DB_PASS_ENV_VAR_NAME: clp_config.database.password, + CLP_DB_PASS_ENV_VAR_NAME: credentials[ClpDbUserType.CLP].password, + CLP_DB_USER_ENV_VAR_NAME: credentials[ClpDbUserType.CLP].username, } container_start_cmd = generate_container_start_cmd( container_name, necessary_mounts, clp_config.container_image_ref, extra_env_vars @@ -214,9 +216,10 @@ def handle_extract_stream_cmd( container_clp_config, clp_config, get_container_config_filename(container_name) ) necessary_mounts = [mounts.logs_dir] + credentials = clp_config.database.credentials extra_env_vars = { - CLP_DB_USER_ENV_VAR_NAME: clp_config.database.username, - CLP_DB_PASS_ENV_VAR_NAME: clp_config.database.password, + CLP_DB_PASS_ENV_VAR_NAME: credentials[ClpDbUserType.CLP].password, + CLP_DB_USER_ENV_VAR_NAME: credentials[ClpDbUserType.CLP].username, } container_start_cmd = generate_container_start_cmd( container_name, necessary_mounts, clp_config.container_image_ref, extra_env_vars diff --git a/components/clp-package-utils/clp_package_utils/scripts/native/decompress.py b/components/clp-package-utils/clp_package_utils/scripts/native/decompress.py index 4b1a06fc95..e00f61699d 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/native/decompress.py +++ b/components/clp-package-utils/clp_package_utils/scripts/native/decompress.py @@ -13,6 +13,7 @@ CLP_DB_USER_ENV_VAR_NAME, CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH, ClpConfig, + ClpDbUserType, Database, ) from clp_py_utils.clp_metadata_db_utils import get_files_table_name @@ -245,10 +246,11 @@ def handle_extract_file_cmd( "--db-table-prefix", clp_db_connection_params["table_prefix"], ] # fmt: on + credentials = clp_db_connection_params["credentials"] extract_env = { **os.environ, - CLP_DB_USER_ENV_VAR_NAME: clp_db_connection_params["username"], - CLP_DB_PASS_ENV_VAR_NAME: clp_db_connection_params["password"], + CLP_DB_USER_ENV_VAR_NAME: credentials[ClpDbUserType.CLP].username, + CLP_DB_PASS_ENV_VAR_NAME: credentials[ClpDbUserType.CLP].password, } files_to_extract_list_path = None diff --git a/components/clp-package-utils/clp_package_utils/scripts/search.py b/components/clp-package-utils/clp_package_utils/scripts/search.py index 528009448b..5d117f0b3b 100755 --- a/components/clp-package-utils/clp_package_utils/scripts/search.py +++ b/components/clp-package-utils/clp_package_utils/scripts/search.py @@ -10,6 +10,7 @@ CLP_DB_USER_ENV_VAR_NAME, CLP_DEFAULT_CONFIG_FILE_RELATIVE_PATH, CLP_DEFAULT_DATASET_NAME, + ClpDbUserType, StorageEngine, StorageType, ) @@ -134,9 +135,10 @@ def main(argv): container_clp_config, clp_config, get_container_config_filename(container_name) ) necessary_mounts = [mounts.logs_dir] + credentials = clp_config.database.credentials extra_env_vars = { - CLP_DB_USER_ENV_VAR_NAME: clp_config.database.username, - CLP_DB_PASS_ENV_VAR_NAME: clp_config.database.password, + CLP_DB_PASS_ENV_VAR_NAME: credentials[ClpDbUserType.CLP].password, + CLP_DB_USER_ENV_VAR_NAME: credentials[ClpDbUserType.CLP].username, } container_start_cmd = generate_container_start_cmd( container_name, necessary_mounts, clp_config.container_image_ref, extra_env_vars diff --git a/components/clp-py-utils/clp_py_utils/clp_config.py b/components/clp-py-utils/clp_py_utils/clp_config.py index 540d75d52a..e312e21261 100644 --- a/components/clp-py-utils/clp_py_utils/clp_config.py +++ b/components/clp-py-utils/clp_py_utils/clp_config.py @@ -69,6 +69,8 @@ CLP_VERSION_FILE_PATH = pathlib.Path("VERSION") # Environment variable names +CLP_DB_ROOT_USER_ENV_VAR_NAME = "CLP_DB_ROOT_USER" +CLP_DB_ROOT_PASS_ENV_VAR_NAME = "CLP_DB_ROOT_PASS" CLP_DB_USER_ENV_VAR_NAME = "CLP_DB_USER" CLP_DB_PASS_ENV_VAR_NAME = "CLP_DB_PASS" CLP_QUEUE_USER_ENV_VAR_NAME = "CLP_QUEUE_USER" @@ -171,6 +173,20 @@ def validate_query_engine_package_compatibility(self): return self +class ClpDbUserType(KebabCaseStrEnum): + """Database user types used by CLP components.""" + + CLP = auto() + ROOT = auto() + + +class DbUserCredentials(BaseModel): + """Credentials for a database user.""" + + username: NonEmptyStr | None = None + password: NonEmptyStr | None = None + + class Database(BaseModel): DEFAULT_PORT: ClassVar[int] = 3306 @@ -182,15 +198,39 @@ class Database(BaseModel): auto_commit: bool = False compress: bool = True - username: NonEmptyStr | None = None - password: NonEmptyStr | None = None + credentials: dict[ClpDbUserType, DbUserCredentials] = { + ClpDbUserType.CLP: DbUserCredentials(), + ClpDbUserType.ROOT: DbUserCredentials(), + } - def ensure_credentials_loaded(self): - if self.username is None or self.password is None: - raise ValueError("Credentials not loaded.") + def ensure_credentials_loaded(self, user_type: ClpDbUserType) -> None: + """ + Ensures that credentials for the given `user_type` are loaded. + + :param user_type: + :raise ValueError: If credentials for the given `user_type` are not loaded. + """ + if ( + self.credentials[user_type].username is None + or self.credentials[user_type].password is None + ): + err_msg = f"Credentials for user type '{user_type}' are not loaded." + raise ValueError(err_msg) + + def get_mysql_connection_params( + self, + disable_localhost_socket_connection: bool = False, + user_type: ClpDbUserType = ClpDbUserType.CLP, + ) -> dict[str, Any]: + """ + Returns a dictionary of connection parameters to be used by mysql's or mariadb's `connect()` + method, ensuring only credentials for the given `user_type` are loaded. - def get_mysql_connection_params(self, disable_localhost_socket_connection: bool = False): - self.ensure_credentials_loaded() + :param disable_localhost_socket_connection: If true, force TCP connections. + :param user_type: User type whose credentials should be included. + :return: Dictionary of MySQL connection parameters. + """ + self.ensure_credentials_loaded(user_type) host = self.host if disable_localhost_socket_connection and "localhost" == self.host: @@ -200,8 +240,8 @@ def get_mysql_connection_params(self, disable_localhost_socket_connection: bool connection_params = { "host": host, "port": self.port, - "user": self.username, - "password": self.password, + "user": self.credentials[user_type].username, + "password": self.credentials[user_type].password, "database": self.name, "compress": self.compress, "autocommit": self.auto_commit, @@ -210,51 +250,87 @@ def get_mysql_connection_params(self, disable_localhost_socket_connection: bool connection_params["ssl_cert"] = self.ssl_cert return connection_params - def get_clp_connection_params_and_type(self, disable_localhost_socket_connection: bool = False): - self.ensure_credentials_loaded() + def get_clp_connection_params_and_type( + self, + disable_localhost_socket_connection: bool = False, + user_type: ClpDbUserType = ClpDbUserType.CLP, + ) -> dict[str, Any]: + """ + Returns a dictionary of connection parameters to be used by CLP components and ensures only + credentials for the given `user_type` are loaded. + + :param disable_localhost_socket_connection: If true, force TCP connections. + :param user_type: User type whose credentials should be included. + :return: Dictionary of CLP connection parameters. + """ + self.ensure_credentials_loaded(user_type) host = self.host if disable_localhost_socket_connection and "localhost" == self.host: host = "127.0.0.1" - connection_params_and_type = { - # NOTE: clp-core does not distinguish between mysql and mariadb - "type": DatabaseEngine.MYSQL.value, - "host": host, - "port": self.port, - "username": self.username, - "password": self.password, - "name": self.name, - "table_prefix": CLP_METADATA_TABLE_PREFIX, - "compress": self.compress, - "autocommit": self.auto_commit, - } - if self.ssl_cert: - connection_params_and_type["ssl_cert"] = self.ssl_cert - return connection_params_and_type + d = self.dump_to_primitive_dict() + + d["credentials"] = {user_type: self.credentials[user_type].model_dump()} + d["host"] = host + d["table_prefix"] = CLP_METADATA_TABLE_PREFIX + # NOTE: clp-core does not distinguish between mysql and mariadb + d["type"] = DatabaseEngine.MYSQL.value - def dump_to_primitive_dict(self): - d = self.model_dump(exclude={"username", "password"}) return d + def dump_to_primitive_dict(self) -> dict[str, Any]: + """:return: A dictionary representation of this model, excluding credentials.""" + return self.model_dump(exclude={"credentials"}) + def load_credentials_from_file(self, credentials_file_path: pathlib.Path): + """ + Loads database credentials from a YAML file. + + :param credentials_file_path: + :raise ValueError: If the file is empty or does not contain the expected keys. + """ config = read_yaml_config_file(credentials_file_path) if config is None: raise ValueError(f"Credentials file '{credentials_file_path}' is empty.") try: - self.username = get_config_value(config, f"{DB_COMPONENT_NAME}.username") - self.password = get_config_value(config, f"{DB_COMPONENT_NAME}.password") + self.credentials[ClpDbUserType.CLP].username = get_config_value( + config, f"{DB_COMPONENT_NAME}.username" + ) + self.credentials[ClpDbUserType.CLP].password = get_config_value( + config, f"{DB_COMPONENT_NAME}.password" + ) + self.credentials[ClpDbUserType.ROOT].username = get_config_value( + config, f"{DB_COMPONENT_NAME}.root_username" + ) + self.credentials[ClpDbUserType.ROOT].password = get_config_value( + config, f"{DB_COMPONENT_NAME}.root_password" + ) except KeyError as ex: raise ValueError( f"Credentials file '{credentials_file_path}' does not contain key '{ex}'." ) - def load_credentials_from_env(self): + def load_credentials_from_env(self, user_type: ClpDbUserType = ClpDbUserType.CLP): """ - :raise ValueError: if any expected environment variable is not set. + Loads database credentials from environment variables for the given user type. + + :param user_type: + :raise ValueError: If the user type is not supported. + :raise ValueError: Propagates `_get_env_var`'s exceptions. """ - self.username = _get_env_var(CLP_DB_USER_ENV_VAR_NAME) - self.password = _get_env_var(CLP_DB_PASS_ENV_VAR_NAME) + if user_type == ClpDbUserType.CLP: + user_env_var = CLP_DB_USER_ENV_VAR_NAME + pass_env_var = CLP_DB_PASS_ENV_VAR_NAME + elif user_type == ClpDbUserType.ROOT: + user_env_var = CLP_DB_ROOT_USER_ENV_VAR_NAME + pass_env_var = CLP_DB_ROOT_PASS_ENV_VAR_NAME + else: + err_msg = f"Unsupported user type '{user_type}'." + raise ValueError(err_msg) + + self.credentials[user_type].username = _get_env_var(user_env_var) + self.credentials[user_type].password = _get_env_var(pass_env_var) def transform_for_container(self): self.host = DB_COMPONENT_NAME diff --git a/components/clp-py-utils/clp_py_utils/sql_adapter.py b/components/clp-py-utils/clp_py_utils/sql_adapter.py index 2f7e3f98f8..9d1076490b 100644 --- a/components/clp-py-utils/clp_py_utils/sql_adapter.py +++ b/components/clp-py-utils/clp_py_utils/sql_adapter.py @@ -1,3 +1,5 @@ +"""SQL adapter for connecting to a MySQL-compatible database.""" + import contextlib import logging import time @@ -8,7 +10,7 @@ from sqlalchemy import pool from sqlalchemy.dialects.mysql import mariadbconnector, mysqlconnector -from clp_py_utils.clp_config import Database +from clp_py_utils.clp_config import ClpDbUserType, Database, DatabaseEngine class DummyCloseableObject: @@ -58,16 +60,78 @@ def alive(self): class SqlAdapter: + """SQL adapter for connecting to a MySQL-compatible database.""" + def __init__(self, database_config: Database): + """Initializes the SqlAdapter with the CLP database config model.""" self.database_config = database_config - def create_mysql_connection( - self, disable_localhost_socket_connection: bool = False - ) -> mysql.connector.MySQLConnection: + def create_connection( + self, + disable_localhost_socket_connection: bool = False, + user_type: ClpDbUserType = ClpDbUserType.CLP, + ) -> mysql.connector.abstracts.MySQLConnectionAbstract | mariadb.connection: + """ + Creates a connection to the database. + + :param disable_localhost_socket_connection: If true, force TCP connections. + :param user_type: User type whose credentials should be used to connect. + :return: The connection. + """ + if self.database_config.type == DatabaseEngine.MYSQL: + return self._create_mysql_connection(disable_localhost_socket_connection, user_type) + if self.database_config.type == DatabaseEngine.MARIADB: + return self._create_mariadb_connection(disable_localhost_socket_connection, user_type) + raise NotImplementedError + + def create_connection_pool( + self, + logger: logging.Logger, + pool_size: int, + disable_localhost_socket_connection: bool = False, + user_type: ClpDbUserType = ClpDbUserType.CLP, + ) -> ConnectionPoolWrapper: + """ + Creates a connection pool to the database. + + :param logger: The logger to use for logging connection pool errors. + :param pool_size: The size of the connection pool. + :param disable_localhost_socket_connection: If true, force TCP connections. + :param user_type: User type whose credentials should be used to connect. + :return: The connection pool. + """ + + def _create_connection(): + return self.create_connection(disable_localhost_socket_connection, user_type) + + if self.database_config.type == DatabaseEngine.MYSQL: + dialect = mysqlconnector.dialect() + elif self.database_config.type == DatabaseEngine.MARIADB: + dialect = mariadbconnector.dialect() + else: + raise NotImplementedError( + f"Database type '{self.database_config.type}' is not supported." + ) + return ConnectionPoolWrapper( + pool.QueuePool( + _create_connection, + pool_size=pool_size, + dialect=dialect, + max_overflow=0, + pre_ping=True, + ), + logger, + ) + + def _create_mysql_connection( + self, + disable_localhost_socket_connection: bool = False, + user_type: ClpDbUserType = ClpDbUserType.CLP, + ) -> mysql.connector.abstracts.MySQLConnectionAbstract: try: connection = mysql.connector.connect( **self.database_config.get_mysql_connection_params( - disable_localhost_socket_connection + disable_localhost_socket_connection, user_type ) ) except mysql.connector.Error as err: @@ -83,13 +147,15 @@ def create_mysql_connection( else: return connection - def create_mariadb_connection( - self, disable_localhost_socket_connection: bool = False + def _create_mariadb_connection( + self, + disable_localhost_socket_connection: bool = False, + user_type: ClpDbUserType = ClpDbUserType.CLP, ) -> mariadb.connection: try: connection = mariadb.connect( **self.database_config.get_mysql_connection_params( - disable_localhost_socket_connection + disable_localhost_socket_connection, user_type ) ) except mariadb.Error as err: @@ -97,36 +163,3 @@ def create_mariadb_connection( raise err else: return connection - - def create_connection(self, disable_localhost_socket_connection: bool = False): - if "mysql" == self.database_config.type: - return self.create_mysql_connection(disable_localhost_socket_connection) - if "mariadb" == self.database_config.type: - return self.create_mariadb_connection(disable_localhost_socket_connection) - raise NotImplementedError - - def create_connection_pool( - self, - logger: logging.Logger, - pool_size: int, - disable_localhost_socket_connection: bool = False, - ): - def create_connection(): - return self.create_connection(disable_localhost_socket_connection) - - if "mysql" == self.database_config.type: - dialect = mysqlconnector.dialect() - elif "mariadb" == self.database_config.type: - dialect = mariadbconnector.dialect() - else: - raise NotImplementedError - return ConnectionPoolWrapper( - pool.QueuePool( - create_connection, - pool_size=pool_size, - dialect=dialect, - max_overflow=0, - pre_ping=True, - ), - logger, - ) diff --git a/components/job-orchestration/job_orchestration/executor/compress/compression_task.py b/components/job-orchestration/job_orchestration/executor/compress/compression_task.py index ff10e13b48..66eda1f2c7 100644 --- a/components/job-orchestration/job_orchestration/executor/compress/compression_task.py +++ b/components/job-orchestration/job_orchestration/executor/compress/compression_task.py @@ -10,6 +10,7 @@ from clp_py_utils.clp_config import ( CLP_DB_PASS_ENV_VAR_NAME, CLP_DB_USER_ENV_VAR_NAME, + ClpDbUserType, COMPRESSION_JOBS_TABLE_NAME, COMPRESSION_TASKS_TABLE_NAME, Database, @@ -208,9 +209,10 @@ def _get_db_connection_env_vars_for_clp_cmd( :param clp_metadata_db_connection_config: :return: Dictionary of database connection environment variables for the clp command. """ + credentials = Database.model_validate(clp_metadata_db_connection_config).credentials return { - CLP_DB_USER_ENV_VAR_NAME: clp_metadata_db_connection_config["username"], - CLP_DB_PASS_ENV_VAR_NAME: clp_metadata_db_connection_config["password"], + CLP_DB_USER_ENV_VAR_NAME: credentials[ClpDbUserType.CLP].username, + CLP_DB_PASS_ENV_VAR_NAME: credentials[ClpDbUserType.CLP].password, } diff --git a/components/job-orchestration/job_orchestration/scheduler/utils.py b/components/job-orchestration/job_orchestration/scheduler/utils.py index b3284a4e82..3735113a33 100644 --- a/components/job-orchestration/job_orchestration/scheduler/utils.py +++ b/components/job-orchestration/job_orchestration/scheduler/utils.py @@ -38,7 +38,7 @@ def kill_hanging_jobs(sql_adapter: SqlAdapter, scheduler_type: str) -> list[int] raise ValueError(f"Unexpected scheduler type {scheduler_type}") with ( - closing(sql_adapter.create_mysql_connection()) as db_conn, + closing(sql_adapter.create_connection()) as db_conn, closing(db_conn.cursor(dictionary=True)) as db_cursor, ): db_cursor.execute( diff --git a/components/package-template/src/etc/credentials.template.yaml b/components/package-template/src/etc/credentials.template.yaml index eb2f9b73f7..dea9957ae4 100644 --- a/components/package-template/src/etc/credentials.template.yaml +++ b/components/package-template/src/etc/credentials.template.yaml @@ -2,6 +2,8 @@ #database: # username: "clp-user" # password: "pass" +# root_username: "root" +# root_password: "root-pass" # ## Queue credentials #queue: diff --git a/tools/deployment/package/docker-compose-all.yaml b/tools/deployment/package/docker-compose-all.yaml index 72704a085e..f68136f79e 100644 --- a/tools/deployment/package/docker-compose-all.yaml +++ b/tools/deployment/package/docker-compose-all.yaml @@ -52,7 +52,7 @@ services: environment: MYSQL_DATABASE: "${CLP_DB_NAME:-clp-db}" MYSQL_PASSWORD: "${CLP_DB_PASS:?Please set a value.}" - MYSQL_ROOT_PASSWORD: "${CLP_DB_PASS:?Please set a value.}" + MYSQL_ROOT_PASSWORD: "${CLP_DB_ROOT_PASS:?Please set a value.}" MYSQL_USER: "${CLP_DB_USER:?Please set a value.}" ports: - host_ip: "${CLP_DB_HOST:-127.0.0.1}" @@ -85,6 +85,8 @@ services: hostname: "db_table_creator" environment: CLP_DB_PASS: "${CLP_DB_PASS:?Please set a value.}" + CLP_DB_ROOT_PASS: "${CLP_DB_ROOT_PASS:?Please set a value.}" + CLP_DB_ROOT_USER: "${CLP_DB_ROOT_USER:-root}" CLP_DB_USER: "${CLP_DB_USER:?Please set a value.}" PYTHONPATH: "/opt/clp/lib/python3/site-packages" volumes: @@ -470,8 +472,8 @@ services: environment: CLP_LOGGING_LEVEL: "${CLP_MCP_LOGGING_LEVEL:-INFO}" CLP_LOGS_DIR: "/var/log/mcp_server" - CLP_DB_USER: "${CLP_DB_USER}" - CLP_DB_PASS: "${CLP_DB_PASS}" + CLP_DB_PASS: "${CLP_DB_PASS:?Please set a value.}" + CLP_DB_USER: "${CLP_DB_USER:?Please set a value.}" PYTHONPATH: "/opt/clp/lib/python3/site-packages" ports: - host_ip: "${CLP_MCP_HOST:-127.0.0.1}"